// tests/test_utf8_slice.el // // Guards utf8_safe_slice(), the fix for a live defect found 2026-08-06: // // The session preload cuts recalled memory content at a fixed length // (chat.el: `if str_len(acc) > 350 { str_slice(acc, 0, 350) }` and // session_preload_bullets' identical per-bullet cut). str_slice and str_len count // BYTES, so any cut landing inside a multi-byte UTF-8 character leaves a dangling // lead byte in the system prompt — and the whole request body is then invalid UTF-8. // Providers reject it outright, so the user sees "AI unavailable" with no clue why, // on both wire formats. Caught by an OpenAI-lane gate whose stub decodes strictly; // reproduced from a real memory whose content contained box-drawing rules (E2 94 80). // // Trigger is ordinary content: an em dash, a curly quote, an accented name, a table // border, an emoji — anything non-ASCII sitting on the cut boundary. It gets MORE // likely as a user's memory grows, which is the opposite of what should happen. // // §1 also pins the semantics this fix depends on: that str_char_code returns the // BYTE value at a byte index (not a decoded code point). If a future runtime changes // that, these assertions fail loudly instead of the truncation silently rotting. import "../chat.el" let pass_count: Int = 0 let fail_count: Int = 0 fn assert_eq(label: String, got: String, expected: String) -> Void { if str_eq(got, expected) { let pass_count = pass_count + 1 println(" PASS: " + label) } else { let fail_count = fail_count + 1 println(" FAIL: " + label) println(" got: " + got) println(" expected: " + expected) } } fn assert_eq_int(label: String, got: Int, expected: Int) -> Void { assert_eq(label, int_to_str(got), int_to_str(expected)) } println("") println("1. runtime semantics this fix relies on") // "─" is U+2500 = E2 94 80 (three bytes). If str_len counts bytes, len("─") is 3. let dash: String = "─" assert_eq_int("str_len counts BYTES (one box-drawing char = 3)", str_len(dash), 3) assert_eq_int("str_char_code returns the BYTE value (lead byte of U+2500 = 0xE2 = 226)", str_char_code(dash, 0), 226) assert_eq_int("str_char_code second byte = 0x94 = 148", str_char_code(dash, 1), 148) assert_eq_int("str_char_code third byte = 0x80 = 128", str_char_code(dash, 2), 128) println("") println("2. utf8_safe_slice — never leaves a partial character") // Pure ASCII: behaves exactly like str_slice. assert_eq("ascii under the limit is untouched", utf8_safe_slice("hello", 10), "hello") assert_eq("ascii over the limit cuts exactly", utf8_safe_slice("hello world", 5), "hello") // A cut landing INSIDE a 3-byte character must drop that character entirely. // "ab─cd": bytes a b E2 94 80 c d. Cutting at 3 or 4 lands mid-dash. let mixed: String = "ab─cd" assert_eq_int("fixture is 7 bytes (2 ascii + 3 + 2 ascii)", str_len(mixed), 7) assert_eq("cut inside the char (n=3) drops the partial char", utf8_safe_slice(mixed, 3), "ab") assert_eq("cut inside the char (n=4) drops the partial char", utf8_safe_slice(mixed, 4), "ab") // A cut landing exactly AFTER a complete character keeps it. assert_eq("cut on the char boundary (n=5) keeps the whole char", utf8_safe_slice(mixed, 5), "ab─") // 2-byte character (é = C3 A9) and 4-byte character (😀 = F0 9F 98 80). let acc: String = "xé" assert_eq("cut inside a 2-byte char drops it", utf8_safe_slice(acc, 2), "x") assert_eq("cut after a 2-byte char keeps it", utf8_safe_slice(acc, 3), "xé") let emo: String = "x😀" assert_eq("cut inside a 4-byte char drops it (n=3)", utf8_safe_slice(emo, 3), "x") assert_eq("cut inside a 4-byte char drops it (n=4)", utf8_safe_slice(emo, 4), "x") assert_eq("cut after a 4-byte char keeps it", utf8_safe_slice(emo, 5), "x😀") println("") println("3. the real-world shape that produced the bug") // A run of box-drawing rules, cut mid-character — the exact captured failure. let rules: String = "──────" assert_eq_int("six box rules = 18 bytes", str_len(rules), 18) // n=16 lands one byte into the sixth character. let cut16: String = utf8_safe_slice(rules, 16) assert_eq_int("cut at 16 backs off to a clean 15-byte boundary", str_len(cut16), 15) // Every byte of the result must belong to a complete character: the last byte of a // well-formed run of these is always 0x80, and 15 is divisible by 3. assert_eq_int("result ends on a complete char (last byte 0x80)", str_char_code(cut16, 14), 128) println("") println("test_utf8_slice.el: " + int_to_str(pass_count) + " passed, " + int_to_str(fail_count) + " failed")