diff --git a/engram/dist/engram.c b/engram/dist/engram.c index 78c8663..ee8b408 100644 --- a/engram/dist/engram.c +++ b/engram/dist/engram.c @@ -11,6 +11,7 @@ el_val_t query_int(el_val_t path, el_val_t key, el_val_t default_val); el_val_t extract_id(el_val_t path, el_val_t prefix); el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body); el_val_t route_act_stats(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_text_health(el_val_t method, el_val_t path, el_val_t body); el_val_t persist_canonical(void); el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body); el_val_t route_get_node(el_val_t method, el_val_t path, el_val_t body); @@ -125,6 +126,11 @@ el_val_t route_act_stats(el_val_t method, el_val_t path, el_val_t body) { return 0; } +el_val_t route_text_health(el_val_t method, el_val_t path, el_val_t body) { + return engram_text_health_json(); + return 0; +} + el_val_t persist_canonical(void) { el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); el_val_t dir = ({ el_val_t _if_result_1 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_1 = (EL_STR("/tmp/engram")); } else { _if_result_1 = (dir_raw); } _if_result_1; }); @@ -458,6 +464,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) { if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/act-stats")) || str_eq(clean, EL_STR("/act-stats")))) { return route_act_stats(method, path, body); } + if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/text-health")) || str_eq(clean, EL_STR("/text-health")))) { + return route_text_health(method, path, body); + } if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/nodes")) || str_eq(clean, EL_STR("/nodes")))) { return route_create_node(method, path, body); } diff --git a/engram/src/server.el b/engram/src/server.el index e6303cd..e5bab58 100644 --- a/engram/src/server.el +++ b/engram/src/server.el @@ -90,6 +90,24 @@ fn route_act_stats(method: String, path: String, body: String) -> String { engram_act_stats_json() } +// route_text_health — GET /api/text-health +// (2026-08-08 self-review) The daily census half of the text-integrity gauge. +// Today's review found that the JSON parser had been replacing every \uXXXX +// escape with a literal '?' for at least two months: 3,119 of 4,081 +// non-telemetry nodes (76%) were damaged, including the self traversal root +// and every values node, and NOTHING detected it — because every gauge in the +// system measured whether the machinery was running, and none measured whether +// the text it carried was intact. No snapshot on disk predates the damage, so +// it cannot be undone; it can only be made impossible to repeat quietly. +// +// The parser is fixed. This route is the standing check: `damaged` should now +// hold flat at its historical floor and never climb. `write_damaged` (also on +// the heartbeat as txt_damaged) is the live regression signal — non-zero means +// a write path is mangling text right now. +fn route_text_health(method: String, path: String, body: String) -> String { + engram_text_health_json() +} + // (2026-07-18 self-review) Scoping sweep: `let` inside an if-block creates an // inner scope only — it does NOT mutate the outer binding (documented with // evidence in awareness.el, 2026-05-25). Every default/reassignment below used @@ -573,6 +591,9 @@ fn handle_request(method: String, path: String, body: String) -> String { if str_eq(method, "GET") && (str_eq(clean, "/api/act-stats") || str_eq(clean, "/act-stats")) { return route_act_stats(method, path, body) } + if str_eq(method, "GET") && (str_eq(clean, "/api/text-health") || str_eq(clean, "/text-health")) { + return route_text_health(method, path, body) + } // Nodes if str_eq(method, "POST") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes")) { diff --git a/lang/releases/v1.0.0-20260501/el_runtime.c b/lang/releases/v1.0.0-20260501/el_runtime.c index a15bb15..e52d157 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.c +++ b/lang/releases/v1.0.0-20260501/el_runtime.c @@ -2938,10 +2938,95 @@ static char* jp_parse_string_raw(JsonParser* jp) { case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'u': { - /* Skip 4 hex digits; emit '?' as a placeholder */ - for (int i = 0; i < 4 && jp->p < jp->end; i++) jp->p++; - c = '?'; - break; + /* Decode \uXXXX (with surrogate pairs) to UTF-8. + * + * (2026-08-08 self-review) This used to skip the 4 hex + * digits and emit a literal '?'. That is a LOSSY, silent, + * irreversible transform on every JSON string entering the + * runtime — and JSON writers escape non-ASCII by default + * (Python json.dumps ships ensure_ascii=True; most MCP + * clients do the same). So every em dash, curly quote, + * accented letter, and emoji arriving over MCP or HTTP was + * replaced by one question mark on the way in, with no + * error and no counter. + * + * Measured on the live store before the fix: 3,119 of + * 4,081 non-telemetry nodes (76%) carried the damage, + * including the self traversal root and all 13 values + * nodes ("Value ? Constraints as Freedom"). Node contents + * split cleanly into fully-clean or fully-mangled with + * zero overlap — the tell that this was one write path, + * not gradual rot. The corruption is unrecoverable in + * place (3 bytes collapse to 1), so the only real fix is + * to stop producing it; historical repair has to come from + * each node's upstream source. + * + * Malformed escapes keep the old '?' behaviour rather than + * failing the parse: a truncated body should not take down + * a route that previously tolerated it. */ + unsigned cp = 0; + int ok = 1; + for (int i = 0; i < 4; i++) { + if (jp->p >= jp->end) { ok = 0; break; } + char h = *jp->p++; + unsigned d; + if (h >= '0' && h <= '9') d = (unsigned)(h - '0'); + else if (h >= 'a' && h <= 'f') d = (unsigned)(h - 'a' + 10); + else if (h >= 'A' && h <= 'F') d = (unsigned)(h - 'A' + 10); + else { ok = 0; break; } + cp = (cp << 4) | d; + } + if (!ok) { c = '?'; break; } + /* High surrogate: pair it with the following low surrogate + * so astral-plane codepoints (emoji) survive. If the next + * token is not a valid low surrogate, rewind so it is + * parsed on its own terms rather than swallowed. */ + if (cp >= 0xD800 && cp <= 0xDBFF && + (size_t)(jp->end - jp->p) >= 6 && + jp->p[0] == '\\' && jp->p[1] == 'u') { + const char* save = jp->p; + unsigned lo = 0; int ok2 = 1; + jp->p += 2; + for (int i = 0; i < 4; i++) { + char h = *jp->p++; + unsigned d; + if (h >= '0' && h <= '9') d = (unsigned)(h - '0'); + else if (h >= 'a' && h <= 'f') d = (unsigned)(h - 'a' + 10); + else if (h >= 'A' && h <= 'F') d = (unsigned)(h - 'A' + 10); + else { ok2 = 0; break; } + lo = (lo << 4) | d; + } + if (ok2 && lo >= 0xDC00 && lo <= 0xDFFF) + cp = 0x10000u + ((cp - 0xD800u) << 10) + (lo - 0xDC00u); + else jp->p = save; + } + /* Lone surrogate → U+FFFD (WHATWG / serde_json behaviour): + * emitting a raw surrogate would produce invalid UTF-8. */ + if (cp >= 0xD800 && cp <= 0xDFFF) cp = 0xFFFD; + + char ub[4]; int un; + if (cp < 0x80) { + ub[0] = (char)cp; un = 1; + } else if (cp < 0x800) { + ub[0] = (char)(0xC0 | (cp >> 6)); + ub[1] = (char)(0x80 | (cp & 0x3F)); un = 2; + } else if (cp < 0x10000) { + ub[0] = (char)(0xE0 | (cp >> 12)); + ub[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); + ub[2] = (char)(0x80 | (cp & 0x3F)); un = 3; + } else { + ub[0] = (char)(0xF0 | (cp >> 18)); + ub[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); + ub[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); + ub[3] = (char)(0x80 | (cp & 0x3F)); un = 4; + } + while (len + (size_t)un >= cap) { + cap *= 2; + out = realloc(out, cap); + if (!out) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + } + for (int i = 0; i < un; i++) out[len++] = ub[i]; + continue; /* bytes already appended */ } default: c = esc; break; } @@ -7214,6 +7299,91 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) { return el_wrap_str(el_strdup(n->id)); } +/* ── Text-integrity instrumentation (2026-08-08 self-review) ─────────────── + * + * WHY THIS EXISTS. The JSON parser silently replaced every \uXXXX escape with + * '?' for at least two months (see jp_parse_string_raw). 3,119 of 4,081 + * non-telemetry nodes — 76%, including the self traversal root and all 13 + * values nodes — were damaged before anything noticed, and nothing noticed + * because nothing measured. Working memory, Hebbian potentiation, embedding + * coverage, sync age, and the breaker were all instrumented to four decimal + * places; the actual TEXT was not instrumented at all. Every gauge answered + * "is the machinery running" and none answered "is what it carries intact." + * + * The damage is unrecoverable in place (a 3-byte codepoint collapses to one + * byte), and no snapshot on disk predates it, so this cannot be undone. What + * it can be is *impossible to repeat quietly*. Two numbers, split by the + * question each answers: + * + * stock — engram_text_health_json(), a full O(total bytes) census. Too + * expensive for the 60s heartbeat, exactly right for the daily + * self-review. Answers "how much damage is in the store". + * flow — _eg_txt_write_damaged, incremented per damaged node at creation. + * O(len) on a path that already copies the string, so it is free. + * Rides the heartbeat. Answers "is a write path damaging things + * RIGHT NOW" — which is the regression question, and the one that + * would have caught this in a day instead of two months. + * + * SIGNATURE. Conservative on purpose — a false alarm that cries corruption + * over ordinary punctuation is worse than useless. Two patterns, both of + * which are essentially absent from well-formed English prose: + * (a) alnum '?' alnum — "na?ve", "caf?s", "don?t". A real question mark + * never sits between two word characters. + * (b) ' ? ' followed by a lowercase letter — a lost em/en dash. A real + * question mark is not preceded by a space, and + * what follows one starts a new sentence. + * Deliberately NOT flagged: a trailing '?' after a word, '? ' before a + * capital, or '?' at end of string — all legitimate. This under-counts (it + * cannot see a mangled 'café ' where the '?' landed before a space), so the + * census is a floor on the damage, never an exaggeration of it. */ +static int eg_text_loss_signature(const char* s) { + if (!s) return 0; + for (const char* p = s; *p; p++) { + if (*p != '?') continue; + unsigned char prev = (p == s) ? 0 : (unsigned char)p[-1]; + unsigned char next = (unsigned char)p[1]; + /* (a) sandwiched between word characters. */ + if (isalnum(prev) && isalnum(next)) return 1; + /* (b) spaced, with lowercase continuation — a lost dash. */ + if (prev == ' ' && next == ' ' && islower((unsigned char)p[2])) return 1; + } + return 0; +} + +/* Damaged-node creations since process start. See the block comment above. */ +static int64_t _eg_txt_write_damaged = 0; + +/* engram_text_health_json — full text-integrity census over the store. + * O(total content bytes); call it on demand (daily self-review / a route), + * never per heartbeat. `damaged` counts nodes carrying the loss signature, + * `multibyte` counts nodes holding valid multi-byte UTF-8 — the two together + * separate "no damage" from "no non-ASCII text to damage", which a single + * number cannot do. Telemetry is excluded: ISE payloads are machine-written + * ASCII JSON and would dilute the ratio that matters. */ +el_val_t engram_text_health_json(void) { + EngramStore* g = engram_get(); + int64_t scanned = 0, damaged = 0, multibyte = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (n->node_type && + (strcmp(n->node_type, "InternalStateEvent") == 0 || + strcmp(n->node_type, "Tag") == 0)) continue; + scanned++; + if (eg_text_loss_signature(n->content)) damaged++; + for (const char* p = n->content; p && *p; p++) { + if ((unsigned char)*p >= 0x80) { multibyte++; break; } + } + } + char buf[256]; + snprintf(buf, sizeof(buf), + "{\"scanned\":%lld,\"damaged\":%lld,\"multibyte\":%lld," + "\"damaged_pct\":%.2f,\"write_damaged\":%lld}", + (long long)scanned, (long long)damaged, (long long)multibyte, + scanned > 0 ? (100.0 * (double)damaged / (double)scanned) : 0.0, + (long long)_eg_txt_write_damaged); + return el_wrap_str(el_strdup(buf)); +} + el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label, el_val_t salience, el_val_t importance, el_val_t confidence, el_val_t tier, el_val_t tags) { @@ -7229,6 +7399,11 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label, const char* tg = EL_CSTR(tags); n->content = el_strdup_persist(c ? c : ""); n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory"); + /* Flow half of the text-integrity gauge — see eg_text_loss_signature. + * Telemetry is machine-written ASCII and is excluded so the counter stays + * a clean signal about content-bearing write paths. */ + if (strcmp(n->node_type, "InternalStateEvent") != 0 && + eg_text_loss_signature(n->content)) _eg_txt_write_damaged++; n->label = el_strdup_persist(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); n->tier = el_strdup_persist(ti && *ti ? ti : "Working"); n->tags = el_strdup_persist(tg ? tg : ""); @@ -10540,7 +10715,7 @@ el_val_t engram_act_stats_json(void) { /* 768, not 512: the write-back gauges added 2026-08-07 push the worst-case * rendering past the old bound, and snprintf would truncate the JSON into * an unparseable tail rather than fail loudly. */ - char buf[768]; + char buf[896]; /* ctx_cos (2026-07-29): cos(query, context centroid) at the LAST * activate call, measured before the query was folded in. ~1.0 = * context aligned with current query; low = divergence (expected at @@ -10555,7 +10730,13 @@ el_val_t engram_act_stats_json(void) { "\"hebb_warm\":%d," "\"hebb_wb_pending\":%d,\"hebb_wb_drained\":%lld," "\"hebb_wb_dropped\":%lld," - "\"dup_seeds\":%lld,\"dup_wm\":%lld,\"dup_wm_global\":%lld}", + "\"dup_seeds\":%lld,\"dup_wm\":%lld,\"dup_wm_global\":%lld," + /* txt_damaged: nodes created THIS process whose content carries + * the character-loss signature. Steady 0 is the healthy state; + * any climb means a write path is mangling text again. Cheap + * (counted at creation) — the full census lives in + * engram_text_health_json. (2026-08-08 self-review) */ + "\"txt_damaged\":%lld}", (long long)_eg_act_wm_evicted, (long long)_eg_act_breakthroughs, breaker_open, _eg_embed_consec_fail, @@ -10566,7 +10747,8 @@ el_val_t engram_act_stats_json(void) { _eg_hebb_wb_len, (long long)_eg_hebb_wb_drained, (long long)_eg_hebb_wb_dropped, (long long)_eg_act_dup_seeds, (long long)_eg_act_dup_wm, - (long long)_eg_act_dup_wm_global); + (long long)_eg_act_dup_wm_global, + (long long)_eg_txt_write_damaged); return el_wrap_str(el_strdup(buf)); } diff --git a/lang/releases/v1.0.0-20260501/el_runtime.h b/lang/releases/v1.0.0-20260501/el_runtime.h index 97ec897..569ce0c 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.h +++ b/lang/releases/v1.0.0-20260501/el_runtime.h @@ -618,6 +618,7 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t d el_val_t engram_activate_json(el_val_t query, el_val_t depth); el_val_t engram_stats_json(void); el_val_t engram_act_stats_json(void); +el_val_t engram_text_health_json(void); el_val_t engram_cosine_sim(el_val_t id_a, el_val_t id_b); /* Destructively pop up to `max` newly-formed Hebbian associations as a JSON * array of {from_id,to_id,weight,hebb}. The learning process (soul daemon) is