merge round-4-delta: flat stride-2 token list + str_char_code dispatch + batch c_escape

- Flat token list: lexer emits [kind0, val0, kind1, val1, ...] instead of [{kind,val}, ...]
  Eliminates per-token ElMap allocation (~112B × N tokens)
- str_char_code hot loop: char classification via Int codes, no strdup per char
- Batch c_escape: str_slice clean runs instead of char-at per byte
- Parser updated to use tok_at/tok_kind/tok_value stride-2 accessors
This commit is contained in:
Will Anderson
2026-05-05 20:29:35 -05:00
4 changed files with 403 additions and 277 deletions
+48 -12
View File
@@ -38,10 +38,13 @@ fn is_hex_digit_byte(b: Int) -> Bool {
}
fn c_escape(s: String) -> String {
// Use index-based byte scanning via str_char_code(s, i) and str_char_at(s, i).
// This avoids native_string_chars + str_join, which corrupts high-byte (>= 0x80)
// characters because list_join's looks_like_string heuristic rejects strings
// whose first byte is >= 0x7F and emits them as decimal pointer values instead.
// Batch ASCII chars using str_slice instead of str_char_at per byte.
// Track clean_start: the beginning of the current run of bytes that need
// no escaping. On each special byte, flush the accumulated clean run via
// str_slice, then append the escape. This reduces parts-list appends from
// O(N) to O(K) where K = number of special bytes << N for normal strings.
//
// Special bytes: '"'=34, '\\'=92, '\n'=10, '\r'=13, '\t'=9, any byte>=128.
//
// IMPORTANT: after a \xNN hex escape, if the next byte is a hex digit
// (0-9, a-f, A-F), we emit `""` to split the C string literal so the C
@@ -51,46 +54,75 @@ fn c_escape(s: String) -> String {
let total: Int = str_len(s)
let parts: [String] = native_list_empty()
let i: Int = 0
let clean_start: Int = 0
let prev_was_hex_escape: Bool = false
while i < total {
let bval: Int = str_char_code(s, i)
// If the previous token was a \xNN escape and the current byte is a
// hex digit, insert an empty string literal ("") to break the escape.
// Handle the hex-escape split case first: if prev was \xNN and this
// byte is a hex digit, we must flush the clean run and insert "".
// (At this point clean_start == i since the previous special byte
// already reset it, so flush is a no-op unless something is pending.)
if prev_was_hex_escape {
if is_hex_digit_byte(bval) {
// Flush any accumulated clean bytes before the split marker.
if clean_start < i {
let parts = native_list_append(parts, str_slice(s, clean_start, i))
}
let parts = native_list_append(parts, "\"\"")
let clean_start = i
}
}
let prev_was_hex_escape = false
if bval == 34 {
// 34 = '"'
// 34 = '"' flush clean run, then escape
if clean_start < i {
let parts = native_list_append(parts, str_slice(s, clean_start, i))
}
let parts = native_list_append(parts, "\\\"")
let clean_start = i + 1
} else {
if bval == 92 {
// 92 = '\\'
if clean_start < i {
let parts = native_list_append(parts, str_slice(s, clean_start, i))
}
let parts = native_list_append(parts, "\\\\")
let clean_start = i + 1
} else {
if bval == 10 {
// 10 = '\n'
if clean_start < i {
let parts = native_list_append(parts, str_slice(s, clean_start, i))
}
let parts = native_list_append(parts, "\\n")
let clean_start = i + 1
} else {
if bval == 13 {
// 13 = '\r'
if clean_start < i {
let parts = native_list_append(parts, str_slice(s, clean_start, i))
}
let parts = native_list_append(parts, "\\r")
let clean_start = i + 1
} else {
if bval == 9 {
// 9 = '\t'
if clean_start < i {
let parts = native_list_append(parts, str_slice(s, clean_start, i))
}
let parts = native_list_append(parts, "\\t")
let clean_start = i + 1
} else {
if bval >= 128 {
// Escape non-ASCII bytes (>= 0x80) as \xNN so
// Clang does not misinterpret multi-byte UTF-8
// sequences in C string literals.
// Non-ASCII: flush, then \xNN
if clean_start < i {
let parts = native_list_append(parts, str_slice(s, clean_start, i))
}
let parts = native_list_append(parts, "\\x" + byte_to_hex2(bval))
let prev_was_hex_escape = true
} else {
let parts = native_list_append(parts, str_char_at(s, i))
let clean_start = i + 1
}
// else: plain ASCII extends the current clean run (no append)
}
}
}
@@ -98,6 +130,10 @@ fn c_escape(s: String) -> String {
}
let i = i + 1
}
// Flush the final clean run if any
if clean_start < total {
let parts = native_list_append(parts, str_slice(s, clean_start, total))
}
str_join(parts, "")
}