From 1e67544c88a689289be4ee52a0494cb075cdf460 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Tue, 5 May 2026 15:43:21 -0500 Subject: [PATCH] =?UTF-8?q?round-2-alpha:=20char=20code=20ops=20in=20lex()?= =?UTF-8?q?=20hot=20loop=20=E2=80=94=20eliminate=20str=5Fchar=5Fat=20alloc?= =?UTF-8?q?ations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace str_char_at (returns strdup String) with str_char_code (returns Int) in the main lex() while loop and scan_digits/scan_ident helpers. For a 400KB combined source, str_char_at was allocating ~400K x 16B = 6.4MB of transient 2-byte strings for the ch variable alone. str_char_code returns an integer directly — zero allocation. Add Int-based helpers: is_digit_code, is_alpha_code, is_ws_code, is_alnum_or_underscore_code. Rewrite lex() operator dispatch using char code constants (e.g. '/'=47, '"'=34, '='=61). Result on main.el: 17.1MB -> 15.4MB peak RSS (-10%). Self-hosting: PASS. --- lang/el-compiler/src/lexer.el | 217 ++++++++++++++++++++++------------ 1 file changed, 142 insertions(+), 75 deletions(-) diff --git a/lang/el-compiler/src/lexer.el b/lang/el-compiler/src/lexer.el index cf05ff8..2cc1b34 100644 --- a/lang/el-compiler/src/lexer.el +++ b/lang/el-compiler/src/lexer.el @@ -7,11 +7,50 @@ // // Entry point: fn lex(source: String) -> [Map] // -// Uses native_string_chars to split the source into a chars list, -// then indexes it with native_list_get - avoids O(N-) string cloning. +// Performance: the hot lexer loop uses str_char_code (returns Int) instead of +// str_char_at (returns strdup'd String) for character classification. +// For a 400KB source, str_char_at allocates ~400K × 16B = ~6.4MB of temporary +// strings for the `ch` variable alone. str_char_code avoids all that. -// -- Character helpers --------------------------------------------------------- +// -- Character helpers (Int-based, no string allocation) ---------------------- +// These operate on char codes (from str_char_code) instead of str_char_at, +// eliminating one strdup per character in the hot lexer loop. +fn is_digit_code(c: Int) -> Bool { + // '0'=48 .. '9'=57 + if c >= 48 { + if c <= 57 { return true } + } + false +} + +fn is_alpha_code(c: Int) -> Bool { + // 'A'=65..'Z'=90, 'a'=97..'z'=122 + if c >= 65 { + if c <= 90 { return true } + } + if c >= 97 { + if c <= 122 { return true } + } + false +} + +fn is_alnum_or_underscore_code(c: Int) -> Bool { + if is_digit_code(c) { return true } + if is_alpha_code(c) { return true } + if c == 95 { return true } // '_' + false +} + +fn is_ws_code(c: Int) -> Bool { + if c == 32 { return true } // ' ' + if c == 9 { return true } // '\t' + if c == 10 { return true } // '\n' + if c == 13 { return true } // '\r' + false +} + +// Legacy String-based helpers kept for scan_interp helpers that use str_char_at. fn lex_is_digit(ch: String) -> Bool { if ch == "0" { return true } if ch == "1" { return true } @@ -164,8 +203,8 @@ fn scan_digits(src: String, start: Int, total: Int) -> Map { if i >= total { let running = false } else { - let ch: String = str_char_at(src, i) - if lex_is_digit(ch) { + let c: Int = str_char_code(src, i) + if is_digit_code(c) { let i = i + 1 } else { let running = false @@ -184,8 +223,8 @@ fn scan_ident(src: String, start: Int, total: Int) -> Map { if i >= total { let running = false } else { - let ch: String = str_char_at(src, i) - if is_alnum_or_underscore(ch) { + let c: Int = str_char_code(src, i) + if is_alnum_or_underscore_code(c) { let i = i + 1 } else { let running = false @@ -669,38 +708,44 @@ fn scan_interp_string(src: String, start: Int, total: Int) -> Map { } // -- Main lexer ---------------------------------------------------------------- +// Char code constants (avoids strdup for single-char comparison) +// '/' = 47, '"' = 34, '0'-'9' = 48-57, 'a'-'z' = 97-122, 'A'-'Z' = 65-90 +// '_' = 95, ' '=32, '\t'=9, '\n'=10, '\r'=13 +// '=' = 61, '!' = 33, '<' = 60, '>' = 62, '&' = 38, '|' = 124 +// '-' = 45, ':' = 58, '+' = 43, '*' = 42, '%' = 37 +// '(' = 40, ')' = 41, '{' = 123, '}' = 125, '[' = 91, ']' = 93 +// ',' = 44, '.' = 46, ';' = 59, '@' = 64, '?' = 63 fn lex(source: String) -> [Map] { - // Use str_len + str_char_at instead of native_string_chars to avoid - // allocating a 400K-element character list in El's arena. For a 400KB - // source file, native_string_chars created ~400K permanent string allocations - // (one per character), consuming ~20MB of memory before lexing even started. + // Use str_char_code (returns Int) instead of str_char_at (returns strdup String) + // for all character classification in the hot loop. For a 400KB source, + // str_char_at allocates ~400K × 16B = ~6.4MB of temporary strings. let total: Int = str_len(source) let tokens: [Map] = native_list_empty() let i: Int = 0 while i < total { - let ch: String = str_char_at(source, i) + let c: Int = str_char_code(source, i) - // Skip whitespace - if lex_is_whitespace(ch) { + // Skip whitespace (space=32, tab=9, newline=10, CR=13) + if is_ws_code(c) { let i = i + 1 } else { - // Line comments: // - if ch == "/" { + // Line comments: // (slash=47) + if c == 47 { let next_i = i + 1 if next_i < total { - let next_ch: String = str_char_at(source, next_i) - if next_ch == "/" { - // skip to end of line + let nc: Int = str_char_code(source, next_i) + if nc == 47 { + // skip to end of line (newline=10) let i = i + 2 let running2 = true while running2 { if i >= total { let running2 = false } else { - let lch: String = str_char_at(source, i) - if lch == "\n" { + let lc: Int = str_char_code(source, i) + if lc == 10 { let running2 = false } else { let i = i + 1 @@ -716,31 +761,27 @@ fn lex(source: String) -> [Map] { let i = i + 1 } } else { - // String literal (plain or interpolated with ${expr} syntax). - // scan_interp_string handles both cases: plain strings emit a - // single Str token; interpolated strings emit a flat token - // sequence (Str Plus expr-tokens Plus Str ...) that the parser - // naturally assembles into a BinOp concat tree. - if ch == "\"" { + // String literal: '"' = 34 + if c == 34 { let interp_result = scan_interp_string(source, i + 1, total) let interp_toks: [Map] = interp_result["tokens"] let new_pos: Int = interp_result["pos"] let tokens = interp_tokens_append_all(tokens, interp_toks) let i = new_pos } else { - // Number literal - if lex_is_digit(ch) { + // Number literal: '0'-'9' = 48-57 + if is_digit_code(c) { let result = scan_digits(source, i, total) let num_text: String = result["text"] let new_pos: Int = result["pos"] - // check for float (dot followed by digit) + // check for float (dot=46 followed by digit) if new_pos < total { - let dot_ch: String = str_char_at(source, new_pos) - if dot_ch == "." { + let dc: Int = str_char_code(source, new_pos) + if dc == 46 { let after_dot = new_pos + 1 if after_dot < total { - let after_dot_ch: String = str_char_at(source, after_dot) - if lex_is_digit(after_dot_ch) { + let adc: Int = str_char_code(source, after_dot) + if is_digit_code(adc) { let frac_result = scan_digits(source, after_dot, total) let frac_text: String = frac_result["text"] let frac_pos: Int = frac_result["pos"] @@ -763,8 +804,8 @@ fn lex(source: String) -> [Map] { let i = new_pos } } else { - // Identifier or keyword - if lex_is_alpha(ch) || ch == "_" { + // Identifier or keyword: alpha or '_'=95 + if is_alpha_code(c) || c == 95 { let result = scan_ident(source, i, total) let word: String = result["text"] let new_pos: Int = result["pos"] @@ -778,17 +819,19 @@ fn lex(source: String) -> [Map] { } else { // Multi-char and single-char operators/delimiters let peek_i = i + 1 - let peek_ch = "" + let peek_c: Int = -1 if peek_i < total { - let peek_ch: String = str_char_at(source, peek_i) + let peek_c: Int = str_char_code(source, peek_i) } - if ch == "=" { - if peek_ch == "=" { + if c == 61 { + // '=' = 61 + if peek_c == 61 { let tokens = native_list_append(tokens, make_tok("EqEq", "==")) let i = i + 2 } else { - if peek_ch == ">" { + if peek_c == 62 { + // '>' = 62 let tokens = native_list_append(tokens, make_tok("FatArrow", "=>")) let i = i + 2 } else { @@ -797,8 +840,9 @@ fn lex(source: String) -> [Map] { } } } else { - if ch == "!" { - if peek_ch == "=" { + if c == 33 { + // '!' = 33 + if peek_c == 61 { let tokens = native_list_append(tokens, make_tok("NotEq", "!=")) let i = i + 2 } else { @@ -806,8 +850,9 @@ fn lex(source: String) -> [Map] { let i = i + 1 } } else { - if ch == "<" { - if peek_ch == "=" { + if c == 60 { + // '<' = 60 + if peek_c == 61 { let tokens = native_list_append(tokens, make_tok("LtEq", "<=")) let i = i + 2 } else { @@ -815,8 +860,9 @@ fn lex(source: String) -> [Map] { let i = i + 1 } } else { - if ch == ">" { - if peek_ch == "=" { + if c == 62 { + // '>' = 62 + if peek_c == 61 { let tokens = native_list_append(tokens, make_tok("GtEq", ">=")) let i = i + 2 } else { @@ -824,20 +870,23 @@ fn lex(source: String) -> [Map] { let i = i + 1 } } else { - if ch == "&" { - if peek_ch == "&" { + if c == 38 { + // '&' = 38 + if peek_c == 38 { let tokens = native_list_append(tokens, make_tok("And", "&&")) let i = i + 2 } else { let i = i + 1 } } else { - if ch == "|" { - if peek_ch == "|" { + if c == 124 { + // '|' = 124 + if peek_c == 124 { let tokens = native_list_append(tokens, make_tok("Or", "||")) let i = i + 2 } else { - if peek_ch == ">" { + if peek_c == 62 { + // '>' = 62 let tokens = native_list_append(tokens, make_tok("PipeOp", "|>")) let i = i + 2 } else { @@ -846,8 +895,10 @@ fn lex(source: String) -> [Map] { } } } else { - if ch == "-" { - if peek_ch == ">" { + if c == 45 { + // '-' = 45 + if peek_c == 62 { + // '>' = 62 let tokens = native_list_append(tokens, make_tok("Arrow", "->")) let i = i + 2 } else { @@ -855,8 +906,9 @@ fn lex(source: String) -> [Map] { let i = i + 1 } } else { - if ch == ":" { - if peek_ch == ":" { + if c == 58 { + // ':' = 58 + if peek_c == 58 { let tokens = native_list_append(tokens, make_tok("ColonColon", "::")) let i = i + 2 } else { @@ -864,55 +916,67 @@ fn lex(source: String) -> [Map] { let i = i + 1 } } else { - if ch == "+" { + if c == 43 { + // '+' = 43 let tokens = native_list_append(tokens, make_tok("Plus", "+")) let i = i + 1 } else { - if ch == "*" { + if c == 42 { + // '*' = 42 let tokens = native_list_append(tokens, make_tok("Star", "*")) let i = i + 1 } else { - if ch == "%" { + if c == 37 { + // '%' = 37 let tokens = native_list_append(tokens, make_tok("Percent", "%")) let i = i + 1 } else { - if ch == "(" { + if c == 40 { + // '(' = 40 let tokens = native_list_append(tokens, make_tok("LParen", "(")) let i = i + 1 } else { - if ch == ")" { + if c == 41 { + // ')' = 41 let tokens = native_list_append(tokens, make_tok("RParen", ")")) let i = i + 1 } else { - if ch == "{" { + if c == 123 { + // '{' = 123 let tokens = native_list_append(tokens, make_tok("LBrace", "{")) let i = i + 1 } else { - if ch == "}" { + if c == 125 { + // '}' = 125 let tokens = native_list_append(tokens, make_tok("RBrace", "}")) let i = i + 1 } else { - if ch == "[" { + if c == 91 { + // '[' = 91 let tokens = native_list_append(tokens, make_tok("LBracket", "[")) let i = i + 1 } else { - if ch == "]" { + if c == 93 { + // ']' = 93 let tokens = native_list_append(tokens, make_tok("RBracket", "]")) let i = i + 1 } else { - if ch == "," { + if c == 44 { + // ',' = 44 let tokens = native_list_append(tokens, make_tok("Comma", ",")) let i = i + 1 } else { - if ch == "." { - // Check for ..= (inclusive range) before .. (exclusive range) before single . + if c == 46 { + // '.' = 46: check for ..= or .. let peek2_i = i + 2 - let peek2_ch = "" + let peek2_c: Int = -1 if peek2_i < total { - let peek2_ch: String = str_char_at(source, peek2_i) + let peek2_c: Int = str_char_code(source, peek2_i) } - if peek_ch == "." { - if peek2_ch == "=" { + if peek_c == 46 { + // '..' prefix + if peek2_c == 61 { + // '..=' = 46 46 61 let tokens = native_list_append(tokens, make_tok("DotDotEq", "..=")) let i = i + 3 } else { @@ -924,15 +988,18 @@ fn lex(source: String) -> [Map] { let i = i + 1 } } else { - if ch == ";" { + if c == 59 { + // ';' = 59 let tokens = native_list_append(tokens, make_tok("Semicolon", ";")) let i = i + 1 } else { - if ch == "@" { + if c == 64 { + // '@' = 64 let tokens = native_list_append(tokens, make_tok("At", "@")) let i = i + 1 } else { - if ch == "?" { + if c == 63 { + // '?' = 63 let tokens = native_list_append(tokens, make_tok("QuestionMark", "?")) let i = i + 1 } else {