neuron-technologies/el
3
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases 5 Wiki Activity

round-2-alpha: char code ops in lex() hot loop — eliminate str_char_at allocations

Browse Source 
Replace str_char_at (returns strdup String) with str_char_code (returns Int)
in the main lex() while loop and scan_digits/scan_ident helpers.

For a 400KB combined source, str_char_at was allocating ~400K x 16B = 6.4MB
of transient 2-byte strings for the ch variable alone. str_char_code returns
an integer directly — zero allocation.

Add Int-based helpers: is_digit_code, is_alpha_code, is_ws_code,
is_alnum_or_underscore_code. Rewrite lex() operator dispatch using char
code constants (e.g. '/'=47, '"'=34, '='=61).

Result on main.el: 17.1MB -> 15.4MB peak RSS (-10%).
Self-hosting: PASS.
This commit is contained in:
Will Anderson
2026-05-05 15:43:21 -05:00
parent 2ac11a67b1
commit 1e67544c88
1 changed files with 142 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lang/el-compiler/src/lexer.el
+142 -75
View File
@@ -7,11 +7,50 @@
// //
// Entry point: fn lex(source: String) -> [Map<String, Any>] // Entry point: fn lex(source: String) -> [Map<String, Any>]
// //
// Uses native_string_chars to split the source into a chars list, // Performance: the hot lexer loop uses str_char_code (returns Int) instead of
// then indexes it with native_list_get - avoids O(N-) string cloning. // str_char_at (returns strdup'd String) for character classification.
// For a 400KB source, str_char_at allocates ~400K × 16B = ~6.4MB of temporary
// strings for the `ch` variable alone. str_char_code avoids all that.
// -- Character helpers --------------------------------------------------------- // -- Character helpers (Int-based, no string allocation) ----------------------
// These operate on char codes (from str_char_code) instead of str_char_at,
// eliminating one strdup per character in the hot lexer loop.
fn is_digit_code(c: Int) -> Bool {
// '0'=48 .. '9'=57
if c >= 48 {
if c <= 57 { return true }
}
false
}
fn is_alpha_code(c: Int) -> Bool {
// 'A'=65..'Z'=90, 'a'=97..'z'=122
if c >= 65 {
if c <= 90 { return true }
}
if c >= 97 {
if c <= 122 { return true }
}
false
}
fn is_alnum_or_underscore_code(c: Int) -> Bool {
if is_digit_code(c) { return true }
if is_alpha_code(c) { return true }
if c == 95 { return true } // '_'
false
}
fn is_ws_code(c: Int) -> Bool {
if c == 32 { return true } // ' '
if c == 9 { return true } // '\t'
if c == 10 { return true } // '\n'
if c == 13 { return true } // '\r'
false
}
// Legacy String-based helpers kept for scan_interp helpers that use str_char_at.
fn lex_is_digit(ch: String) -> Bool { fn lex_is_digit(ch: String) -> Bool {
if ch == "0" { return true } if ch == "0" { return true }
if ch == "1" { return true } if ch == "1" { return true }
@@ -164,8 +203,8 @@ fn scan_digits(src: String, start: Int, total: Int) -> Map<String, Any> {
if i >= total { if i >= total {
let running = false let running = false
} else { } else {
let ch: String = str_char_at(src, i) let c: Int = str_char_code(src, i)
if lex_is_digit(ch) { if is_digit_code(c) {
let i = i + 1 let i = i + 1
} else { } else {
let running = false let running = false
@@ -184,8 +223,8 @@ fn scan_ident(src: String, start: Int, total: Int) -> Map<String, Any> {
if i >= total { if i >= total {
let running = false let running = false
} else { } else {
let ch: String = str_char_at(src, i) let c: Int = str_char_code(src, i)
if is_alnum_or_underscore(ch) { if is_alnum_or_underscore_code(c) {
let i = i + 1 let i = i + 1
} else { } else {
let running = false let running = false
@@ -669,38 +708,44 @@ fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
} }
// -- Main lexer ---------------------------------------------------------------- // -- Main lexer ----------------------------------------------------------------
// Char code constants (avoids strdup for single-char comparison)
// '/' = 47, '"' = 34, '0'-'9' = 48-57, 'a'-'z' = 97-122, 'A'-'Z' = 65-90
// '_' = 95, ' '=32, '\t'=9, '\n'=10, '\r'=13
// '=' = 61, '!' = 33, '<' = 60, '>' = 62, '&' = 38, '|' = 124
// '-' = 45, ':' = 58, '+' = 43, '*' = 42, '%' = 37
// '(' = 40, ')' = 41, '{' = 123, '}' = 125, '[' = 91, ']' = 93
// ',' = 44, '.' = 46, ';' = 59, '@' = 64, '?' = 63
fn lex(source: String) -> [Map<String, Any>] { fn lex(source: String) -> [Map<String, Any>] {
// Use str_len + str_char_at instead of native_string_chars to avoid // Use str_char_code (returns Int) instead of str_char_at (returns strdup String)
// allocating a 400K-element character list in El's arena. For a 400KB // for all character classification in the hot loop. For a 400KB source,
// source file, native_string_chars created ~400K permanent string allocations // str_char_at allocates ~400K × 16B = ~6.4MB of temporary strings.
// (one per character), consuming ~20MB of memory before lexing even started.
let total: Int = str_len(source) let total: Int = str_len(source)
let tokens: [Map<String, Any>] = native_list_empty() let tokens: [Map<String, Any>] = native_list_empty()
let i: Int = 0 let i: Int = 0
while i < total { while i < total {
let ch: String = str_char_at(source, i) let c: Int = str_char_code(source, i)
// Skip whitespace // Skip whitespace (space=32, tab=9, newline=10, CR=13)
if lex_is_whitespace(ch) { if is_ws_code(c) {
let i = i + 1 let i = i + 1
} else { } else {
// Line comments: // // Line comments: // (slash=47)
if ch == "/" { if c == 47 {
let next_i = i + 1 let next_i = i + 1
if next_i < total { if next_i < total {
let next_ch: String = str_char_at(source, next_i) let nc: Int = str_char_code(source, next_i)
if next_ch == "/" { if nc == 47 {
// skip to end of line // skip to end of line (newline=10)
let i = i + 2 let i = i + 2
let running2 = true let running2 = true
while running2 { while running2 {
if i >= total { if i >= total {
let running2 = false let running2 = false
} else { } else {
let lch: String = str_char_at(source, i) let lc: Int = str_char_code(source, i)
if lch == "\n" { if lc == 10 {
let running2 = false let running2 = false
} else { } else {
let i = i + 1 let i = i + 1
@@ -716,31 +761,27 @@ fn lex(source: String) -> [Map<String, Any>] {
let i = i + 1 let i = i + 1
} }
} else { } else {
// String literal (plain or interpolated with ${expr} syntax). // String literal: '"' = 34
// scan_interp_string handles both cases: plain strings emit a if c == 34 {
// single Str token; interpolated strings emit a flat token
// sequence (Str Plus expr-tokens Plus Str ...) that the parser
// naturally assembles into a BinOp concat tree.
if ch == "\"" {
let interp_result = scan_interp_string(source, i + 1, total) let interp_result = scan_interp_string(source, i + 1, total)
let interp_toks: [Map<String, Any>] = interp_result["tokens"] let interp_toks: [Map<String, Any>] = interp_result["tokens"]
let new_pos: Int = interp_result["pos"] let new_pos: Int = interp_result["pos"]
let tokens = interp_tokens_append_all(tokens, interp_toks) let tokens = interp_tokens_append_all(tokens, interp_toks)
let i = new_pos let i = new_pos
} else { } else {
// Number literal // Number literal: '0'-'9' = 48-57
if lex_is_digit(ch) { if is_digit_code(c) {
let result = scan_digits(source, i, total) let result = scan_digits(source, i, total)
let num_text: String = result["text"] let num_text: String = result["text"]
let new_pos: Int = result["pos"] let new_pos: Int = result["pos"]
// check for float (dot followed by digit) // check for float (dot=46 followed by digit)
if new_pos < total { if new_pos < total {
let dot_ch: String = str_char_at(source, new_pos) let dc: Int = str_char_code(source, new_pos)
if dot_ch == "." { if dc == 46 {
let after_dot = new_pos + 1 let after_dot = new_pos + 1
if after_dot < total { if after_dot < total {
let after_dot_ch: String = str_char_at(source, after_dot) let adc: Int = str_char_code(source, after_dot)
if lex_is_digit(after_dot_ch) { if is_digit_code(adc) {
let frac_result = scan_digits(source, after_dot, total) let frac_result = scan_digits(source, after_dot, total)
let frac_text: String = frac_result["text"] let frac_text: String = frac_result["text"]
let frac_pos: Int = frac_result["pos"] let frac_pos: Int = frac_result["pos"]
@@ -763,8 +804,8 @@ fn lex(source: String) -> [Map<String, Any>] {
let i = new_pos let i = new_pos
} }
} else { } else {
// Identifier or keyword // Identifier or keyword: alpha or '_'=95
if lex_is_alpha(ch) || ch == "_" { if is_alpha_code(c) || c == 95 {
let result = scan_ident(source, i, total) let result = scan_ident(source, i, total)
let word: String = result["text"] let word: String = result["text"]
let new_pos: Int = result["pos"] let new_pos: Int = result["pos"]
@@ -778,17 +819,19 @@ fn lex(source: String) -> [Map<String, Any>] {
} else { } else {
// Multi-char and single-char operators/delimiters // Multi-char and single-char operators/delimiters
let peek_i = i + 1 let peek_i = i + 1
let peek_ch = "" let peek_c: Int = -1
if peek_i < total { if peek_i < total {
let peek_ch: String = str_char_at(source, peek_i) let peek_c: Int = str_char_code(source, peek_i)
} }
if ch == "=" { if c == 61 {
if peek_ch == "=" { // '=' = 61
if peek_c == 61 {
let tokens = native_list_append(tokens, make_tok("EqEq", "==")) let tokens = native_list_append(tokens, make_tok("EqEq", "=="))
let i = i + 2 let i = i + 2
} else { } else {
if peek_ch == ">" { if peek_c == 62 {
// '>' = 62
let tokens = native_list_append(tokens, make_tok("FatArrow", "=>")) let tokens = native_list_append(tokens, make_tok("FatArrow", "=>"))
let i = i + 2 let i = i + 2
} else { } else {
@@ -797,8 +840,9 @@ fn lex(source: String) -> [Map<String, Any>] {
} }
} }
} else { } else {
if ch == "!" { if c == 33 {
if peek_ch == "=" { // '!' = 33
if peek_c == 61 {
let tokens = native_list_append(tokens, make_tok("NotEq", "!=")) let tokens = native_list_append(tokens, make_tok("NotEq", "!="))
let i = i + 2 let i = i + 2
} else { } else {
@@ -806,8 +850,9 @@ fn lex(source: String) -> [Map<String, Any>] {
let i = i + 1 let i = i + 1
} }
} else { } else {
if ch == "<" { if c == 60 {
if peek_ch == "=" { // '<' = 60
if peek_c == 61 {
let tokens = native_list_append(tokens, make_tok("LtEq", "<=")) let tokens = native_list_append(tokens, make_tok("LtEq", "<="))
let i = i + 2 let i = i + 2
} else { } else {
@@ -815,8 +860,9 @@ fn lex(source: String) -> [Map<String, Any>] {
let i = i + 1 let i = i + 1
} }
} else { } else {
if ch == ">" { if c == 62 {
if peek_ch == "=" { // '>' = 62
if peek_c == 61 {
let tokens = native_list_append(tokens, make_tok("GtEq", ">=")) let tokens = native_list_append(tokens, make_tok("GtEq", ">="))
let i = i + 2 let i = i + 2
} else { } else {
@@ -824,20 +870,23 @@ fn lex(source: String) -> [Map<String, Any>] {
let i = i + 1 let i = i + 1
} }
} else { } else {
if ch == "&" { if c == 38 {
if peek_ch == "&" { // '&' = 38
if peek_c == 38 {
let tokens = native_list_append(tokens, make_tok("And", "&&")) let tokens = native_list_append(tokens, make_tok("And", "&&"))
let i = i + 2 let i = i + 2
} else { } else {
let i = i + 1 let i = i + 1
} }
} else { } else {
if ch == "|" { if c == 124 {
if peek_ch == "|" { // '|' = 124
if peek_c == 124 {
let tokens = native_list_append(tokens, make_tok("Or", "||")) let tokens = native_list_append(tokens, make_tok("Or", "||"))
let i = i + 2 let i = i + 2
} else { } else {
if peek_ch == ">" { if peek_c == 62 {
// '>' = 62
let tokens = native_list_append(tokens, make_tok("PipeOp", "|>")) let tokens = native_list_append(tokens, make_tok("PipeOp", "|>"))
let i = i + 2 let i = i + 2
} else { } else {
@@ -846,8 +895,10 @@ fn lex(source: String) -> [Map<String, Any>] {
} }
} }
} else { } else {
if ch == "-" { if c == 45 {
if peek_ch == ">" { // '-' = 45
if peek_c == 62 {
// '>' = 62
let tokens = native_list_append(tokens, make_tok("Arrow", "->")) let tokens = native_list_append(tokens, make_tok("Arrow", "->"))
let i = i + 2 let i = i + 2
} else { } else {
@@ -855,8 +906,9 @@ fn lex(source: String) -> [Map<String, Any>] {
let i = i + 1 let i = i + 1
} }
} else { } else {
if ch == ":" { if c == 58 {
if peek_ch == ":" { // ':' = 58
if peek_c == 58 {
let tokens = native_list_append(tokens, make_tok("ColonColon", "::")) let tokens = native_list_append(tokens, make_tok("ColonColon", "::"))
let i = i + 2 let i = i + 2
} else { } else {
@@ -864,55 +916,67 @@ fn lex(source: String) -> [Map<String, Any>] {
let i = i + 1 let i = i + 1
} }
} else { } else {
if ch == "+" { if c == 43 {
// '+' = 43
let tokens = native_list_append(tokens, make_tok("Plus", "+")) let tokens = native_list_append(tokens, make_tok("Plus", "+"))
let i = i + 1 let i = i + 1
} else { } else {
if ch == "*" { if c == 42 {
// '*' = 42
let tokens = native_list_append(tokens, make_tok("Star", "*")) let tokens = native_list_append(tokens, make_tok("Star", "*"))
let i = i + 1 let i = i + 1
} else { } else {
if ch == "%" { if c == 37 {
// '%' = 37
let tokens = native_list_append(tokens, make_tok("Percent", "%")) let tokens = native_list_append(tokens, make_tok("Percent", "%"))
let i = i + 1 let i = i + 1
} else { } else {
if ch == "(" { if c == 40 {
// '(' = 40
let tokens = native_list_append(tokens, make_tok("LParen", "(")) let tokens = native_list_append(tokens, make_tok("LParen", "("))
let i = i + 1 let i = i + 1
} else { } else {
if ch == ")" { if c == 41 {
// ')' = 41
let tokens = native_list_append(tokens, make_tok("RParen", ")")) let tokens = native_list_append(tokens, make_tok("RParen", ")"))
let i = i + 1 let i = i + 1
} else { } else {
if ch == "{" { if c == 123 {
// '{' = 123
let tokens = native_list_append(tokens, make_tok("LBrace", "{")) let tokens = native_list_append(tokens, make_tok("LBrace", "{"))
let i = i + 1 let i = i + 1
} else { } else {
if ch == "}" { if c == 125 {
// '}' = 125
let tokens = native_list_append(tokens, make_tok("RBrace", "}")) let tokens = native_list_append(tokens, make_tok("RBrace", "}"))
let i = i + 1 let i = i + 1
} else { } else {
if ch == "[" { if c == 91 {
// '[' = 91
let tokens = native_list_append(tokens, make_tok("LBracket", "[")) let tokens = native_list_append(tokens, make_tok("LBracket", "["))
let i = i + 1 let i = i + 1
} else { } else {
if ch == "]" { if c == 93 {
// ']' = 93
let tokens = native_list_append(tokens, make_tok("RBracket", "]")) let tokens = native_list_append(tokens, make_tok("RBracket", "]"))
let i = i + 1 let i = i + 1
} else { } else {
if ch == "," { if c == 44 {
// ',' = 44
let tokens = native_list_append(tokens, make_tok("Comma", ",")) let tokens = native_list_append(tokens, make_tok("Comma", ","))
let i = i + 1 let i = i + 1
} else { } else {
if ch == "." { if c == 46 {
// Check for ..= (inclusive range) before .. (exclusive range) before single . // '.' = 46: check for ..= or ..
let peek2_i = i + 2 let peek2_i = i + 2
let peek2_ch = "" let peek2_c: Int = -1
if peek2_i < total { if peek2_i < total {
let peek2_ch: String = str_char_at(source, peek2_i) let peek2_c: Int = str_char_code(source, peek2_i)
} }
if peek_ch == "." { if peek_c == 46 {
if peek2_ch == "=" { // '..' prefix
if peek2_c == 61 {
// '..=' = 46 46 61
let tokens = native_list_append(tokens, make_tok("DotDotEq", "..=")) let tokens = native_list_append(tokens, make_tok("DotDotEq", "..="))
let i = i + 3 let i = i + 3
} else { } else {
@@ -924,15 +988,18 @@ fn lex(source: String) -> [Map<String, Any>] {
let i = i + 1 let i = i + 1
} }
} else { } else {
if ch == ";" { if c == 59 {
// ';' = 59
let tokens = native_list_append(tokens, make_tok("Semicolon", ";")) let tokens = native_list_append(tokens, make_tok("Semicolon", ";"))
let i = i + 1 let i = i + 1
} else { } else {
if ch == "@" { if c == 64 {
// '@' = 64
let tokens = native_list_append(tokens, make_tok("At", "@")) let tokens = native_list_append(tokens, make_tok("At", "@"))
let i = i + 1 let i = i + 1
} else { } else {
if ch == "?" { if c == 63 {
// '?' = 63
let tokens = native_list_append(tokens, make_tok("QuestionMark", "?")) let tokens = native_list_append(tokens, make_tok("QuestionMark", "?"))
let i = i + 1 let i = i + 1
} else { } else {