prevent engram corruption: add UTF-8 validation in engram_node_full

Reject content containing invalid UTF-8 bytes before persisting — silently
writing invalid UTF-8 garbles JSON snapshots and corrupts node reads.
This commit is contained in:
2026-06-29 11:08:52 -05:00
parent f7bd99ae45
commit e7c2dc7734
+45
View File
@@ -190,6 +190,7 @@ el_val_t println(el_val_t s) {
const char* str = EL_CSTR(s);
if (str) puts(str);
else puts("");
fflush(stdout); /* prevent startup logs from silently buffering when stdout→file */
return 0;
}
@@ -6111,6 +6112,10 @@ static double engram_type_threshold(const char* node_type, const char* tier) {
if (strcmp(tier, "Lesson") == 0) return 0.25;
}
if (node_type) {
/* Knowledge nodes: Canonical/Lesson handled by tier checks above.
* Procedural-tier Knowledge (activation_count>=50 migration): 0.20.
* (2026-06-29 self-review mirrors release runtime fix) */
if (strcmp(node_type, "Knowledge") == 0) return 0.20;
if (strcmp(node_type, "Belief") == 0) return 0.30;
if (strcmp(node_type, "Entity") == 0) return 0.30;
}
@@ -6481,6 +6486,39 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
return el_wrap_str(el_strdup(n->id));
}
/* engram_is_valid_utf8 — return 1 if s is valid UTF-8, 0 if it contains invalid bytes.
* Rejects overlong encodings, surrogate halves, and byte sequences > 4 bytes. */
static int engram_is_valid_utf8(const char* s) {
if (!s) return 1;
const unsigned char* p = (const unsigned char*)s;
while (*p) {
if (*p < 0x80) {
/* ASCII */
p++;
} else if ((*p & 0xE0) == 0xC0) {
/* 2-byte sequence */
if ((p[1] & 0xC0) != 0x80) return 0;
if ((*p & 0xFE) == 0xC0) return 0; /* overlong */
p += 2;
} else if ((*p & 0xF0) == 0xE0) {
/* 3-byte sequence */
if ((p[1] & 0xC0) != 0x80 || (p[2] & 0xC0) != 0x80) return 0;
if (*p == 0xE0 && (p[1] & 0xE0) == 0x80) return 0; /* overlong */
if (*p == 0xED && (p[1] & 0xE0) == 0xA0) return 0; /* surrogate */
p += 3;
} else if ((*p & 0xF8) == 0xF0) {
/* 4-byte sequence */
if ((p[1] & 0xC0) != 0x80 || (p[2] & 0xC0) != 0x80 || (p[3] & 0xC0) != 0x80) return 0;
if (*p == 0xF0 && (p[1] & 0xF0) == 0x80) return 0; /* overlong */
if (*p > 0xF4) return 0; /* above U+10FFFF */
p += 4;
} else {
return 0;
}
}
return 1;
}
el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
el_val_t salience, el_val_t importance, el_val_t confidence,
el_val_t tier, el_val_t tags) {
@@ -6494,6 +6532,13 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
const char* lb = EL_CSTR(label);
const char* ti = EL_CSTR(tier);
const char* tg = EL_CSTR(tags);
/* UTF-8 guard: reject content with invalid UTF-8 bytes. Persisting invalid
* UTF-8 garbles JSON snapshots and corrupts every subsequent node read. */
if (c && !engram_is_valid_utf8(c)) {
fprintf(stderr, "[engram] REJECTED node write — content contains invalid UTF-8 (label=%s)\n",
lb ? lb : "(null)");
return EL_STR("");
}
/* Persistent (el_strdup_persist, NOT el_strdup): these strings are owned by the
* persistent global node store. el_strdup tracks into the per-request arena, which
* el_request_end() frees when the creating HTTP request completes leaving the