diff --git a/lang/el-compiler/src/lexer.el b/lang/el-compiler/src/lexer.el index 504c9c6..cf05ff8 100644 --- a/lang/el-compiler/src/lexer.el +++ b/lang/el-compiler/src/lexer.el @@ -157,45 +157,43 @@ fn keyword_kind(word: String) -> String { // scan_digits - advance i while chars[i] is a digit // Returns { "text": ..., "pos": i } -fn scan_digits(chars: [String], start: Int, total: Int) -> Map { +fn scan_digits(src: String, start: Int, total: Int) -> Map { let i = start - let parts: [String] = native_list_empty() let running = true while running { if i >= total { let running = false } else { - let ch: String = native_list_get(chars, i) + let ch: String = str_char_at(src, i) if lex_is_digit(ch) { - let parts = native_list_append(parts, ch) let i = i + 1 } else { let running = false } } } - { "text": str_join(parts, ""), "pos": i } + // Use str_slice instead of building a parts list — O(1) allocation, O(n) copy. + { "text": str_slice(src, start, i), "pos": i } } // scan_ident - advance i while chars[i] is alphanumeric or underscore -fn scan_ident(chars: [String], start: Int, total: Int) -> Map { +fn scan_ident(src: String, start: Int, total: Int) -> Map { let i = start - let parts: [String] = native_list_empty() let running = true while running { if i >= total { let running = false } else { - let ch: String = native_list_get(chars, i) + let ch: String = str_char_at(src, i) if is_alnum_or_underscore(ch) { - let parts = native_list_append(parts, ch) let i = i + 1 } else { let running = false } } } - { "text": str_join(parts, ""), "pos": i } + // Use str_slice instead of building a parts list — O(1) allocation, O(n) copy. + { "text": str_slice(src, start, i), "pos": i } } // -- Code-bearing string detection + comment strip ---------------------------- @@ -208,34 +206,16 @@ fn scan_ident(chars: [String], start: Int, total: Int) -> Map { // looks_like_code - heuristic gate so we only strip strings that actually // embed JS or CSS. Plain prose, hex blobs, JSON, etc. pass through verbatim. -fn substr_at(chars: [String], start: Int, total: Int, needle: String) -> Bool { - let nchars: [String] = native_string_chars(needle) - let nlen: Int = native_list_len(nchars) +fn substr_at(src: String, start: Int, total: Int, needle: String) -> Bool { + let nlen: Int = str_len(needle) if start + nlen > total { return false } - let i = 0 - let matched = true - while i < nlen { - let a: String = native_list_get(chars, start + i) - let b: String = native_list_get(nchars, i) - if a == b { let i = i + 1 } else { let matched = false; let i = nlen } - } - matched + // Use str_slice comparison instead of char-by-char loop. + str_eq(str_slice(src, start, start + nlen), needle) } fn str_has(s: String, needle: String) -> Bool { - let chars: [String] = native_string_chars(s) - let total: Int = native_list_len(chars) - let i = 0 - let found = false - while i < total { - if substr_at(chars, i, total, needle) { - let found = true - let i = total - } else { - let i = i + 1 - } - } - found + // Use the built-in str_contains which is implemented in native C — O(n) single pass. + str_contains(s, needle) } fn looks_like_code(s: String) -> Bool { @@ -254,8 +234,7 @@ fn looks_like_code(s: String) -> Bool { // comment opener: if the char immediately before '/' is ':', emit the '/' // literally and advance one position. fn strip_code_comments(s: String) -> String { - let chars: [String] = native_string_chars(s) - let total: Int = native_list_len(chars) + let total: Int = str_len(s) let out_parts: [String] = native_list_empty() let i = 0 let in_squote = false @@ -263,7 +242,7 @@ fn strip_code_comments(s: String) -> String { let in_btick = false let prev = "" while i < total { - let ch: String = native_list_get(chars, i) + let ch: String = str_char_at(s, i) let in_js_string = false if in_squote { let in_js_string = true } if in_dquote { let in_js_string = true } @@ -275,7 +254,7 @@ fn strip_code_comments(s: String) -> String { let out_parts = native_list_append(out_parts, ch) let next_i = i + 1 if next_i < total { - let nc: String = native_list_get(chars, next_i) + let nc: String = str_char_at(s, next_i) let out_parts = native_list_append(out_parts, nc) let prev = nc let i = next_i + 1 @@ -304,7 +283,7 @@ fn strip_code_comments(s: String) -> String { let next_i = i + 1 let next_ch = "" if next_i < total { - let next_ch: String = native_list_get(chars, next_i) + let next_ch: String = str_char_at(s, next_i) } if ch == "/" { @@ -323,7 +302,7 @@ fn strip_code_comments(s: String) -> String { if i >= total { let scanning = false } else { - let lc: String = native_list_get(chars, i) + let lc: String = str_char_at(s, i) if lc == "\n" { let scanning = false } else { @@ -342,11 +321,11 @@ fn strip_code_comments(s: String) -> String { if i >= total { let scanning2 = false } else { - let bc: String = native_list_get(chars, i) + let bc: String = str_char_at(s, i) if bc == "*" { let after = i + 1 if after < total { - let nc2: String = native_list_get(chars, after) + let nc2: String = str_char_at(s, after) if nc2 == "/" { let i = after + 1 let scanning2 = false @@ -402,7 +381,7 @@ fn strip_code_comments(s: String) -> String { // scan_string - scan a quoted string literal, handling \" escapes. // Starts AFTER the opening quote. Returns { "text": content, "pos": i_after_close } -fn scan_string(chars: [String], start: Int, total: Int) -> Map { +fn scan_string(src: String, start: Int, total: Int) -> Map { let i = start let parts: [String] = native_list_empty() let running = true @@ -410,12 +389,12 @@ fn scan_string(chars: [String], start: Int, total: Int) -> Map { if i >= total { let running = false } else { - let ch: String = native_list_get(chars, i) + let ch: String = str_char_at(src, i) if ch == "\\" { // escape: peek next char let next_i = i + 1 if next_i < total { - let next_ch: String = native_list_get(chars, next_i) + let next_ch: String = str_char_at(src, next_i) if next_ch == "\"" { let parts = native_list_append(parts, "\"") let i = next_i + 1 @@ -465,19 +444,17 @@ fn scan_string(chars: [String], start: Int, total: Int) -> Map { // scan_interp_brace - scan from `start` (the char after `${`) to the matching // `}`, tracking brace depth so inner braces (e.g. fn calls, map literals) are // handled correctly. Returns { "text": inner_source, "pos": i_after_close }. -fn scan_interp_brace(chars: [String], start: Int, total: Int) -> Map { +fn scan_interp_brace(src: String, start: Int, total: Int) -> Map { let i = start - let parts: [String] = native_list_empty() let depth = 1 let running = true while running { if i >= total { let running = false } else { - let ch: String = native_list_get(chars, i) + let ch: String = str_char_at(src, i) if ch == "{" { let depth = depth + 1 - let parts = native_list_append(parts, ch) let i = i + 1 } else { if ch == "}" { @@ -487,17 +464,16 @@ fn scan_interp_brace(chars: [String], start: Int, total: Int) -> Map], src: [Map]) -> // // Supported escape sequences: \" \n \t \r \\ \$ (literal dollar sign). // Nested quotes inside ${} are not supported; use a variable instead. -fn scan_interp_string(chars: [String], start: Int, total: Int) -> Map { +fn scan_interp_string(src: String, start: Int, total: Int) -> Map { let i = start let out_tokens: [Map] = native_list_empty() let cur_part: [String] = native_list_empty() @@ -548,13 +524,13 @@ fn scan_interp_string(chars: [String], start: Int, total: Int) -> Map= total { let running = false } else { - let ch: String = native_list_get(chars, i) + let ch: String = str_char_at(src, i) if ch == "\\" { // Escape sequence let next_i = i + 1 if next_i < total { - let next_ch: String = native_list_get(chars, next_i) + let next_ch: String = str_char_at(src, next_i) if next_ch == "$" { // \$ => literal '$' (escape for interpolation syntax) let cur_part = native_list_append(cur_part, "$") @@ -602,7 +578,7 @@ fn scan_interp_string(chars: [String], start: Int, total: Int) -> Map Map Map [Map] { - let chars: [String] = native_string_chars(source) - let total: Int = native_list_len(chars) + // Use str_len + str_char_at instead of native_string_chars to avoid + // allocating a 400K-element character list in El's arena. For a 400KB + // source file, native_string_chars created ~400K permanent string allocations + // (one per character), consuming ~20MB of memory before lexing even started. + let total: Int = str_len(source) let tokens: [Map] = native_list_empty() let i: Int = 0 while i < total { - let ch: String = native_list_get(chars, i) + let ch: String = str_char_at(source, i) // Skip whitespace if lex_is_whitespace(ch) { @@ -711,7 +690,7 @@ fn lex(source: String) -> [Map] { if ch == "/" { let next_i = i + 1 if next_i < total { - let next_ch: String = native_list_get(chars, next_i) + let next_ch: String = str_char_at(source, next_i) if next_ch == "/" { // skip to end of line let i = i + 2 @@ -720,7 +699,7 @@ fn lex(source: String) -> [Map] { if i >= total { let running2 = false } else { - let lch: String = native_list_get(chars, i) + let lch: String = str_char_at(source, i) if lch == "\n" { let running2 = false } else { @@ -743,7 +722,7 @@ fn lex(source: String) -> [Map] { // sequence (Str Plus expr-tokens Plus Str ...) that the parser // naturally assembles into a BinOp concat tree. if ch == "\"" { - let interp_result = scan_interp_string(chars, i + 1, total) + let interp_result = scan_interp_string(source, i + 1, total) let interp_toks: [Map] = interp_result["tokens"] let new_pos: Int = interp_result["pos"] let tokens = interp_tokens_append_all(tokens, interp_toks) @@ -751,18 +730,18 @@ fn lex(source: String) -> [Map] { } else { // Number literal if lex_is_digit(ch) { - let result = scan_digits(chars, i, total) + let result = scan_digits(source, i, total) let num_text: String = result["text"] let new_pos: Int = result["pos"] // check for float (dot followed by digit) if new_pos < total { - let dot_ch: String = native_list_get(chars, new_pos) + let dot_ch: String = str_char_at(source, new_pos) if dot_ch == "." { let after_dot = new_pos + 1 if after_dot < total { - let after_dot_ch: String = native_list_get(chars, after_dot) + let after_dot_ch: String = str_char_at(source, after_dot) if lex_is_digit(after_dot_ch) { - let frac_result = scan_digits(chars, after_dot, total) + let frac_result = scan_digits(source, after_dot, total) let frac_text: String = frac_result["text"] let frac_pos: Int = frac_result["pos"] let tokens = native_list_append(tokens, make_tok("Float", num_text + "." + frac_text)) @@ -786,7 +765,7 @@ fn lex(source: String) -> [Map] { } else { // Identifier or keyword if lex_is_alpha(ch) || ch == "_" { - let result = scan_ident(chars, i, total) + let result = scan_ident(source, i, total) let word: String = result["text"] let new_pos: Int = result["pos"] let kw = keyword_kind(word) @@ -801,7 +780,7 @@ fn lex(source: String) -> [Map] { let peek_i = i + 1 let peek_ch = "" if peek_i < total { - let peek_ch: String = native_list_get(chars, peek_i) + let peek_ch: String = str_char_at(source, peek_i) } if ch == "=" { @@ -930,7 +909,7 @@ fn lex(source: String) -> [Map] { let peek2_i = i + 2 let peek2_ch = "" if peek2_i < total { - let peek2_ch: String = native_list_get(chars, peek2_i) + let peek2_ch: String = str_char_at(source, peek2_i) } if peek_ch == "." { if peek2_ch == "=" {