round-2-alpha: char code ops in lex() hot loop — eliminate str_char_at allocations
Replace str_char_at (returns strdup String) with str_char_code (returns Int) in the main lex() while loop and scan_digits/scan_ident helpers. For a 400KB combined source, str_char_at was allocating ~400K x 16B = 6.4MB of transient 2-byte strings for the ch variable alone. str_char_code returns an integer directly — zero allocation. Add Int-based helpers: is_digit_code, is_alpha_code, is_ws_code, is_alnum_or_underscore_code. Rewrite lex() operator dispatch using char code constants (e.g. '/'=47, '"'=34, '='=61). Result on main.el: 17.1MB -> 15.4MB peak RSS (-10%). Self-hosting: PASS.
This commit is contained in:
Will Anderson
+142
-75
@@ -7,11 +7,50 @@
|
||||
//
|
||||
// Entry point: fn lex(source: String) -> [Map<String, Any>]
|
||||
//
|
||||
// Uses native_string_chars to split the source into a chars list,
|
||||
// then indexes it with native_list_get - avoids O(N-) string cloning.
|
||||
// Performance: the hot lexer loop uses str_char_code (returns Int) instead of
|
||||
// str_char_at (returns strdup'd String) for character classification.
|
||||
// For a 400KB source, str_char_at allocates ~400K × 16B = ~6.4MB of temporary
|
||||
// strings for the `ch` variable alone. str_char_code avoids all that.
|
||||
|
||||
// -- Character helpers ---------------------------------------------------------
|
||||
// -- Character helpers (Int-based, no string allocation) ----------------------
|
||||
// These operate on char codes (from str_char_code) instead of str_char_at,
|
||||
// eliminating one strdup per character in the hot lexer loop.
|
||||
|
||||
fn is_digit_code(c: Int) -> Bool {
|
||||
// '0'=48 .. '9'=57
|
||||
if c >= 48 {
|
||||
if c <= 57 { return true }
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn is_alpha_code(c: Int) -> Bool {
|
||||
// 'A'=65..'Z'=90, 'a'=97..'z'=122
|
||||
if c >= 65 {
|
||||
if c <= 90 { return true }
|
||||
}
|
||||
if c >= 97 {
|
||||
if c <= 122 { return true }
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn is_alnum_or_underscore_code(c: Int) -> Bool {
|
||||
if is_digit_code(c) { return true }
|
||||
if is_alpha_code(c) { return true }
|
||||
if c == 95 { return true } // '_'
|
||||
false
|
||||
}
|
||||
|
||||
fn is_ws_code(c: Int) -> Bool {
|
||||
if c == 32 { return true } // ' '
|
||||
if c == 9 { return true } // '\t'
|
||||
if c == 10 { return true } // '\n'
|
||||
if c == 13 { return true } // '\r'
|
||||
false
|
||||
}
|
||||
|
||||
// Legacy String-based helpers kept for scan_interp helpers that use str_char_at.
|
||||
fn lex_is_digit(ch: String) -> Bool {
|
||||
if ch == "0" { return true }
|
||||
if ch == "1" { return true }
|
||||
@@ -164,8 +203,8 @@ fn scan_digits(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||||
if i >= total {
|
||||
let running = false
|
||||
} else {
|
||||
let ch: String = str_char_at(src, i)
|
||||
if lex_is_digit(ch) {
|
||||
let c: Int = str_char_code(src, i)
|
||||
if is_digit_code(c) {
|
||||
let i = i + 1
|
||||
} else {
|
||||
let running = false
|
||||
@@ -184,8 +223,8 @@ fn scan_ident(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||||
if i >= total {
|
||||
let running = false
|
||||
} else {
|
||||
let ch: String = str_char_at(src, i)
|
||||
if is_alnum_or_underscore(ch) {
|
||||
let c: Int = str_char_code(src, i)
|
||||
if is_alnum_or_underscore_code(c) {
|
||||
let i = i + 1
|
||||
} else {
|
||||
let running = false
|
||||
@@ -669,38 +708,44 @@ fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||||
}
|
||||
|
||||
// -- Main lexer ----------------------------------------------------------------
|
||||
// Char code constants (avoids strdup for single-char comparison)
|
||||
// '/' = 47, '"' = 34, '0'-'9' = 48-57, 'a'-'z' = 97-122, 'A'-'Z' = 65-90
|
||||
// '_' = 95, ' '=32, '\t'=9, '\n'=10, '\r'=13
|
||||
// '=' = 61, '!' = 33, '<' = 60, '>' = 62, '&' = 38, '|' = 124
|
||||
// '-' = 45, ':' = 58, '+' = 43, '*' = 42, '%' = 37
|
||||
// '(' = 40, ')' = 41, '{' = 123, '}' = 125, '[' = 91, ']' = 93
|
||||
// ',' = 44, '.' = 46, ';' = 59, '@' = 64, '?' = 63
|
||||
|
||||
fn lex(source: String) -> [Map<String, Any>] {
|
||||
// Use str_len + str_char_at instead of native_string_chars to avoid
|
||||
// allocating a 400K-element character list in El's arena. For a 400KB
|
||||
// source file, native_string_chars created ~400K permanent string allocations
|
||||
// (one per character), consuming ~20MB of memory before lexing even started.
|
||||
// Use str_char_code (returns Int) instead of str_char_at (returns strdup String)
|
||||
// for all character classification in the hot loop. For a 400KB source,
|
||||
// str_char_at allocates ~400K × 16B = ~6.4MB of temporary strings.
|
||||
let total: Int = str_len(source)
|
||||
let tokens: [Map<String, Any>] = native_list_empty()
|
||||
let i: Int = 0
|
||||
|
||||
while i < total {
|
||||
let ch: String = str_char_at(source, i)
|
||||
let c: Int = str_char_code(source, i)
|
||||
|
||||
// Skip whitespace
|
||||
if lex_is_whitespace(ch) {
|
||||
// Skip whitespace (space=32, tab=9, newline=10, CR=13)
|
||||
if is_ws_code(c) {
|
||||
let i = i + 1
|
||||
} else {
|
||||
// Line comments: //
|
||||
if ch == "/" {
|
||||
// Line comments: // (slash=47)
|
||||
if c == 47 {
|
||||
let next_i = i + 1
|
||||
if next_i < total {
|
||||
let next_ch: String = str_char_at(source, next_i)
|
||||
if next_ch == "/" {
|
||||
// skip to end of line
|
||||
let nc: Int = str_char_code(source, next_i)
|
||||
if nc == 47 {
|
||||
// skip to end of line (newline=10)
|
||||
let i = i + 2
|
||||
let running2 = true
|
||||
while running2 {
|
||||
if i >= total {
|
||||
let running2 = false
|
||||
} else {
|
||||
let lch: String = str_char_at(source, i)
|
||||
if lch == "\n" {
|
||||
let lc: Int = str_char_code(source, i)
|
||||
if lc == 10 {
|
||||
let running2 = false
|
||||
} else {
|
||||
let i = i + 1
|
||||
@@ -716,31 +761,27 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let i = i + 1
|
||||
}
|
||||
} else {
|
||||
// String literal (plain or interpolated with ${expr} syntax).
|
||||
// scan_interp_string handles both cases: plain strings emit a
|
||||
// single Str token; interpolated strings emit a flat token
|
||||
// sequence (Str Plus expr-tokens Plus Str ...) that the parser
|
||||
// naturally assembles into a BinOp concat tree.
|
||||
if ch == "\"" {
|
||||
// String literal: '"' = 34
|
||||
if c == 34 {
|
||||
let interp_result = scan_interp_string(source, i + 1, total)
|
||||
let interp_toks: [Map<String, Any>] = interp_result["tokens"]
|
||||
let new_pos: Int = interp_result["pos"]
|
||||
let tokens = interp_tokens_append_all(tokens, interp_toks)
|
||||
let i = new_pos
|
||||
} else {
|
||||
// Number literal
|
||||
if lex_is_digit(ch) {
|
||||
// Number literal: '0'-'9' = 48-57
|
||||
if is_digit_code(c) {
|
||||
let result = scan_digits(source, i, total)
|
||||
let num_text: String = result["text"]
|
||||
let new_pos: Int = result["pos"]
|
||||
// check for float (dot followed by digit)
|
||||
// check for float (dot=46 followed by digit)
|
||||
if new_pos < total {
|
||||
let dot_ch: String = str_char_at(source, new_pos)
|
||||
if dot_ch == "." {
|
||||
let dc: Int = str_char_code(source, new_pos)
|
||||
if dc == 46 {
|
||||
let after_dot = new_pos + 1
|
||||
if after_dot < total {
|
||||
let after_dot_ch: String = str_char_at(source, after_dot)
|
||||
if lex_is_digit(after_dot_ch) {
|
||||
let adc: Int = str_char_code(source, after_dot)
|
||||
if is_digit_code(adc) {
|
||||
let frac_result = scan_digits(source, after_dot, total)
|
||||
let frac_text: String = frac_result["text"]
|
||||
let frac_pos: Int = frac_result["pos"]
|
||||
@@ -763,8 +804,8 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let i = new_pos
|
||||
}
|
||||
} else {
|
||||
// Identifier or keyword
|
||||
if lex_is_alpha(ch) || ch == "_" {
|
||||
// Identifier or keyword: alpha or '_'=95
|
||||
if is_alpha_code(c) || c == 95 {
|
||||
let result = scan_ident(source, i, total)
|
||||
let word: String = result["text"]
|
||||
let new_pos: Int = result["pos"]
|
||||
@@ -778,17 +819,19 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
} else {
|
||||
// Multi-char and single-char operators/delimiters
|
||||
let peek_i = i + 1
|
||||
let peek_ch = ""
|
||||
let peek_c: Int = -1
|
||||
if peek_i < total {
|
||||
let peek_ch: String = str_char_at(source, peek_i)
|
||||
let peek_c: Int = str_char_code(source, peek_i)
|
||||
}
|
||||
|
||||
if ch == "=" {
|
||||
if peek_ch == "=" {
|
||||
if c == 61 {
|
||||
// '=' = 61
|
||||
if peek_c == 61 {
|
||||
let tokens = native_list_append(tokens, make_tok("EqEq", "=="))
|
||||
let i = i + 2
|
||||
} else {
|
||||
if peek_ch == ">" {
|
||||
if peek_c == 62 {
|
||||
// '>' = 62
|
||||
let tokens = native_list_append(tokens, make_tok("FatArrow", "=>"))
|
||||
let i = i + 2
|
||||
} else {
|
||||
@@ -797,8 +840,9 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if ch == "!" {
|
||||
if peek_ch == "=" {
|
||||
if c == 33 {
|
||||
// '!' = 33
|
||||
if peek_c == 61 {
|
||||
let tokens = native_list_append(tokens, make_tok("NotEq", "!="))
|
||||
let i = i + 2
|
||||
} else {
|
||||
@@ -806,8 +850,9 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let i = i + 1
|
||||
}
|
||||
} else {
|
||||
if ch == "<" {
|
||||
if peek_ch == "=" {
|
||||
if c == 60 {
|
||||
// '<' = 60
|
||||
if peek_c == 61 {
|
||||
let tokens = native_list_append(tokens, make_tok("LtEq", "<="))
|
||||
let i = i + 2
|
||||
} else {
|
||||
@@ -815,8 +860,9 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let i = i + 1
|
||||
}
|
||||
} else {
|
||||
if ch == ">" {
|
||||
if peek_ch == "=" {
|
||||
if c == 62 {
|
||||
// '>' = 62
|
||||
if peek_c == 61 {
|
||||
let tokens = native_list_append(tokens, make_tok("GtEq", ">="))
|
||||
let i = i + 2
|
||||
} else {
|
||||
@@ -824,20 +870,23 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let i = i + 1
|
||||
}
|
||||
} else {
|
||||
if ch == "&" {
|
||||
if peek_ch == "&" {
|
||||
if c == 38 {
|
||||
// '&' = 38
|
||||
if peek_c == 38 {
|
||||
let tokens = native_list_append(tokens, make_tok("And", "&&"))
|
||||
let i = i + 2
|
||||
} else {
|
||||
let i = i + 1
|
||||
}
|
||||
} else {
|
||||
if ch == "|" {
|
||||
if peek_ch == "|" {
|
||||
if c == 124 {
|
||||
// '|' = 124
|
||||
if peek_c == 124 {
|
||||
let tokens = native_list_append(tokens, make_tok("Or", "||"))
|
||||
let i = i + 2
|
||||
} else {
|
||||
if peek_ch == ">" {
|
||||
if peek_c == 62 {
|
||||
// '>' = 62
|
||||
let tokens = native_list_append(tokens, make_tok("PipeOp", "|>"))
|
||||
let i = i + 2
|
||||
} else {
|
||||
@@ -846,8 +895,10 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if ch == "-" {
|
||||
if peek_ch == ">" {
|
||||
if c == 45 {
|
||||
// '-' = 45
|
||||
if peek_c == 62 {
|
||||
// '>' = 62
|
||||
let tokens = native_list_append(tokens, make_tok("Arrow", "->"))
|
||||
let i = i + 2
|
||||
} else {
|
||||
@@ -855,8 +906,9 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let i = i + 1
|
||||
}
|
||||
} else {
|
||||
if ch == ":" {
|
||||
if peek_ch == ":" {
|
||||
if c == 58 {
|
||||
// ':' = 58
|
||||
if peek_c == 58 {
|
||||
let tokens = native_list_append(tokens, make_tok("ColonColon", "::"))
|
||||
let i = i + 2
|
||||
} else {
|
||||
@@ -864,55 +916,67 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let i = i + 1
|
||||
}
|
||||
} else {
|
||||
if ch == "+" {
|
||||
if c == 43 {
|
||||
// '+' = 43
|
||||
let tokens = native_list_append(tokens, make_tok("Plus", "+"))
|
||||
let i = i + 1
|
||||
} else {
|
||||
if ch == "*" {
|
||||
if c == 42 {
|
||||
// '*' = 42
|
||||
let tokens = native_list_append(tokens, make_tok("Star", "*"))
|
||||
let i = i + 1
|
||||
} else {
|
||||
if ch == "%" {
|
||||
if c == 37 {
|
||||
// '%' = 37
|
||||
let tokens = native_list_append(tokens, make_tok("Percent", "%"))
|
||||
let i = i + 1
|
||||
} else {
|
||||
if ch == "(" {
|
||||
if c == 40 {
|
||||
// '(' = 40
|
||||
let tokens = native_list_append(tokens, make_tok("LParen", "("))
|
||||
let i = i + 1
|
||||
} else {
|
||||
if ch == ")" {
|
||||
if c == 41 {
|
||||
// ')' = 41
|
||||
let tokens = native_list_append(tokens, make_tok("RParen", ")"))
|
||||
let i = i + 1
|
||||
} else {
|
||||
if ch == "{" {
|
||||
if c == 123 {
|
||||
// '{' = 123
|
||||
let tokens = native_list_append(tokens, make_tok("LBrace", "{"))
|
||||
let i = i + 1
|
||||
} else {
|
||||
if ch == "}" {
|
||||
if c == 125 {
|
||||
// '}' = 125
|
||||
let tokens = native_list_append(tokens, make_tok("RBrace", "}"))
|
||||
let i = i + 1
|
||||
} else {
|
||||
if ch == "[" {
|
||||
if c == 91 {
|
||||
// '[' = 91
|
||||
let tokens = native_list_append(tokens, make_tok("LBracket", "["))
|
||||
let i = i + 1
|
||||
} else {
|
||||
if ch == "]" {
|
||||
if c == 93 {
|
||||
// ']' = 93
|
||||
let tokens = native_list_append(tokens, make_tok("RBracket", "]"))
|
||||
let i = i + 1
|
||||
} else {
|
||||
if ch == "," {
|
||||
if c == 44 {
|
||||
// ',' = 44
|
||||
let tokens = native_list_append(tokens, make_tok("Comma", ","))
|
||||
let i = i + 1
|
||||
} else {
|
||||
if ch == "." {
|
||||
// Check for ..= (inclusive range) before .. (exclusive range) before single .
|
||||
if c == 46 {
|
||||
// '.' = 46: check for ..= or ..
|
||||
let peek2_i = i + 2
|
||||
let peek2_ch = ""
|
||||
let peek2_c: Int = -1
|
||||
if peek2_i < total {
|
||||
let peek2_ch: String = str_char_at(source, peek2_i)
|
||||
let peek2_c: Int = str_char_code(source, peek2_i)
|
||||
}
|
||||
if peek_ch == "." {
|
||||
if peek2_ch == "=" {
|
||||
if peek_c == 46 {
|
||||
// '..' prefix
|
||||
if peek2_c == 61 {
|
||||
// '..=' = 46 46 61
|
||||
let tokens = native_list_append(tokens, make_tok("DotDotEq", "..="))
|
||||
let i = i + 3
|
||||
} else {
|
||||
@@ -924,15 +988,18 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let i = i + 1
|
||||
}
|
||||
} else {
|
||||
if ch == ";" {
|
||||
if c == 59 {
|
||||
// ';' = 59
|
||||
let tokens = native_list_append(tokens, make_tok("Semicolon", ";"))
|
||||
let i = i + 1
|
||||
} else {
|
||||
if ch == "@" {
|
||||
if c == 64 {
|
||||
// '@' = 64
|
||||
let tokens = native_list_append(tokens, make_tok("At", "@"))
|
||||
let i = i + 1
|
||||
} else {
|
||||
if ch == "?" {
|
||||
if c == 63 {
|
||||
// '?' = 63
|
||||
let tokens = native_list_append(tokens, make_tok("QuestionMark", "?"))
|
||||
let i = i + 1
|
||||
} else {
|
||||
|
||||
Reference in New Issue
Block a user