prevent engram corruption: add UTF-8 validation in engram_node_full
Reject content containing invalid UTF-8 bytes before persisting — silently writing invalid UTF-8 garbles JSON snapshots and corrupts node reads.
This commit is contained in:
@@ -190,6 +190,7 @@ el_val_t println(el_val_t s) {
|
||||
const char* str = EL_CSTR(s);
|
||||
if (str) puts(str);
|
||||
else puts("");
|
||||
fflush(stdout); /* prevent startup logs from silently buffering when stdout→file */
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -6111,6 +6112,10 @@ static double engram_type_threshold(const char* node_type, const char* tier) {
|
||||
if (strcmp(tier, "Lesson") == 0) return 0.25;
|
||||
}
|
||||
if (node_type) {
|
||||
/* Knowledge nodes: Canonical/Lesson handled by tier checks above.
|
||||
* Procedural-tier Knowledge (activation_count>=50 migration): 0.20.
|
||||
* (2026-06-29 self-review — mirrors release runtime fix) */
|
||||
if (strcmp(node_type, "Knowledge") == 0) return 0.20;
|
||||
if (strcmp(node_type, "Belief") == 0) return 0.30;
|
||||
if (strcmp(node_type, "Entity") == 0) return 0.30;
|
||||
}
|
||||
@@ -6481,6 +6486,39 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
|
||||
return el_wrap_str(el_strdup(n->id));
|
||||
}
|
||||
|
||||
/* engram_is_valid_utf8 — return 1 if s is valid UTF-8, 0 if it contains invalid bytes.
|
||||
* Rejects overlong encodings, surrogate halves, and byte sequences > 4 bytes. */
|
||||
static int engram_is_valid_utf8(const char* s) {
|
||||
if (!s) return 1;
|
||||
const unsigned char* p = (const unsigned char*)s;
|
||||
while (*p) {
|
||||
if (*p < 0x80) {
|
||||
/* ASCII */
|
||||
p++;
|
||||
} else if ((*p & 0xE0) == 0xC0) {
|
||||
/* 2-byte sequence */
|
||||
if ((p[1] & 0xC0) != 0x80) return 0;
|
||||
if ((*p & 0xFE) == 0xC0) return 0; /* overlong */
|
||||
p += 2;
|
||||
} else if ((*p & 0xF0) == 0xE0) {
|
||||
/* 3-byte sequence */
|
||||
if ((p[1] & 0xC0) != 0x80 || (p[2] & 0xC0) != 0x80) return 0;
|
||||
if (*p == 0xE0 && (p[1] & 0xE0) == 0x80) return 0; /* overlong */
|
||||
if (*p == 0xED && (p[1] & 0xE0) == 0xA0) return 0; /* surrogate */
|
||||
p += 3;
|
||||
} else if ((*p & 0xF8) == 0xF0) {
|
||||
/* 4-byte sequence */
|
||||
if ((p[1] & 0xC0) != 0x80 || (p[2] & 0xC0) != 0x80 || (p[3] & 0xC0) != 0x80) return 0;
|
||||
if (*p == 0xF0 && (p[1] & 0xF0) == 0x80) return 0; /* overlong */
|
||||
if (*p > 0xF4) return 0; /* above U+10FFFF */
|
||||
p += 4;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
|
||||
el_val_t salience, el_val_t importance, el_val_t confidence,
|
||||
el_val_t tier, el_val_t tags) {
|
||||
@@ -6494,6 +6532,13 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
|
||||
const char* lb = EL_CSTR(label);
|
||||
const char* ti = EL_CSTR(tier);
|
||||
const char* tg = EL_CSTR(tags);
|
||||
/* UTF-8 guard: reject content with invalid UTF-8 bytes. Persisting invalid
|
||||
* UTF-8 garbles JSON snapshots and corrupts every subsequent node read. */
|
||||
if (c && !engram_is_valid_utf8(c)) {
|
||||
fprintf(stderr, "[engram] REJECTED node write — content contains invalid UTF-8 (label=%s)\n",
|
||||
lb ? lb : "(null)");
|
||||
return EL_STR("");
|
||||
}
|
||||
/* Persistent (el_strdup_persist, NOT el_strdup): these strings are owned by the
|
||||
* persistent global node store. el_strdup tracks into the per-request arena, which
|
||||
* el_request_end() frees when the creating HTTP request completes — leaving the
|
||||
|
||||
Reference in New Issue
Block a user