merge round-4-delta: flat stride-2 token list + str_char_code dispatch + batch c_escape
- Flat token list: lexer emits [kind0, val0, kind1, val1, ...] instead of [{kind,val}, ...]
Eliminates per-token ElMap allocation (~112B × N tokens)
- str_char_code hot loop: char classification via Int codes, no strdup per char
- Batch c_escape: str_slice clean runs instead of char-at per byte
- Parser updated to use tok_at/tok_kind/tok_value stride-2 accessors
This commit is contained in:
@@ -38,10 +38,13 @@ fn is_hex_digit_byte(b: Int) -> Bool {
|
||||
}
|
||||
|
||||
fn c_escape(s: String) -> String {
|
||||
// Use index-based byte scanning via str_char_code(s, i) and str_char_at(s, i).
|
||||
// This avoids native_string_chars + str_join, which corrupts high-byte (>= 0x80)
|
||||
// characters because list_join's looks_like_string heuristic rejects strings
|
||||
// whose first byte is >= 0x7F and emits them as decimal pointer values instead.
|
||||
// Batch ASCII chars using str_slice instead of str_char_at per byte.
|
||||
// Track clean_start: the beginning of the current run of bytes that need
|
||||
// no escaping. On each special byte, flush the accumulated clean run via
|
||||
// str_slice, then append the escape. This reduces parts-list appends from
|
||||
// O(N) to O(K) where K = number of special bytes << N for normal strings.
|
||||
//
|
||||
// Special bytes: '"'=34, '\\'=92, '\n'=10, '\r'=13, '\t'=9, any byte>=128.
|
||||
//
|
||||
// IMPORTANT: after a \xNN hex escape, if the next byte is a hex digit
|
||||
// (0-9, a-f, A-F), we emit `""` to split the C string literal so the C
|
||||
@@ -51,46 +54,75 @@ fn c_escape(s: String) -> String {
|
||||
let total: Int = str_len(s)
|
||||
let parts: [String] = native_list_empty()
|
||||
let i: Int = 0
|
||||
let clean_start: Int = 0
|
||||
let prev_was_hex_escape: Bool = false
|
||||
while i < total {
|
||||
let bval: Int = str_char_code(s, i)
|
||||
// If the previous token was a \xNN escape and the current byte is a
|
||||
// hex digit, insert an empty string literal ("") to break the escape.
|
||||
// Handle the hex-escape split case first: if prev was \xNN and this
|
||||
// byte is a hex digit, we must flush the clean run and insert "".
|
||||
// (At this point clean_start == i since the previous special byte
|
||||
// already reset it, so flush is a no-op unless something is pending.)
|
||||
if prev_was_hex_escape {
|
||||
if is_hex_digit_byte(bval) {
|
||||
// Flush any accumulated clean bytes before the split marker.
|
||||
if clean_start < i {
|
||||
let parts = native_list_append(parts, str_slice(s, clean_start, i))
|
||||
}
|
||||
let parts = native_list_append(parts, "\"\"")
|
||||
let clean_start = i
|
||||
}
|
||||
}
|
||||
let prev_was_hex_escape = false
|
||||
if bval == 34 {
|
||||
// 34 = '"'
|
||||
// 34 = '"' — flush clean run, then escape
|
||||
if clean_start < i {
|
||||
let parts = native_list_append(parts, str_slice(s, clean_start, i))
|
||||
}
|
||||
let parts = native_list_append(parts, "\\\"")
|
||||
let clean_start = i + 1
|
||||
} else {
|
||||
if bval == 92 {
|
||||
// 92 = '\\'
|
||||
if clean_start < i {
|
||||
let parts = native_list_append(parts, str_slice(s, clean_start, i))
|
||||
}
|
||||
let parts = native_list_append(parts, "\\\\")
|
||||
let clean_start = i + 1
|
||||
} else {
|
||||
if bval == 10 {
|
||||
// 10 = '\n'
|
||||
if clean_start < i {
|
||||
let parts = native_list_append(parts, str_slice(s, clean_start, i))
|
||||
}
|
||||
let parts = native_list_append(parts, "\\n")
|
||||
let clean_start = i + 1
|
||||
} else {
|
||||
if bval == 13 {
|
||||
// 13 = '\r'
|
||||
if clean_start < i {
|
||||
let parts = native_list_append(parts, str_slice(s, clean_start, i))
|
||||
}
|
||||
let parts = native_list_append(parts, "\\r")
|
||||
let clean_start = i + 1
|
||||
} else {
|
||||
if bval == 9 {
|
||||
// 9 = '\t'
|
||||
if clean_start < i {
|
||||
let parts = native_list_append(parts, str_slice(s, clean_start, i))
|
||||
}
|
||||
let parts = native_list_append(parts, "\\t")
|
||||
let clean_start = i + 1
|
||||
} else {
|
||||
if bval >= 128 {
|
||||
// Escape non-ASCII bytes (>= 0x80) as \xNN so
|
||||
// Clang does not misinterpret multi-byte UTF-8
|
||||
// sequences in C string literals.
|
||||
// Non-ASCII: flush, then \xNN
|
||||
if clean_start < i {
|
||||
let parts = native_list_append(parts, str_slice(s, clean_start, i))
|
||||
}
|
||||
let parts = native_list_append(parts, "\\x" + byte_to_hex2(bval))
|
||||
let prev_was_hex_escape = true
|
||||
} else {
|
||||
let parts = native_list_append(parts, str_char_at(s, i))
|
||||
let clean_start = i + 1
|
||||
}
|
||||
// else: plain ASCII — extends the current clean run (no append)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -98,6 +130,10 @@ fn c_escape(s: String) -> String {
|
||||
}
|
||||
let i = i + 1
|
||||
}
|
||||
// Flush the final clean run if any
|
||||
if clean_start < total {
|
||||
let parts = native_list_append(parts, str_slice(s, clean_start, total))
|
||||
}
|
||||
str_join(parts, "")
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user