neuron-technologies/el
2
0
Fork 0
Code Issues 1 Pull Requests 5 Actions Packages Projects Releases 5 Wiki Activity

beta: replace native_string_chars with str_char_at/str_slice in lexer — 49% memory reduction on large files

Browse Source
This commit is contained in:
Will Anderson
2026-05-05 15:19:59 -05:00
parent b580a63540
commit 2ac11a67b1
1 changed files with 49 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lang/el-compiler/src/lexer.el
+49 -70
View File
@@ -157,45 +157,43 @@ fn keyword_kind(word: String) -> String {
// scan_digits - advance i while chars[i] is a digit
// Returns { "text": ..., "pos": i }
fn scan_digits(chars: [String], start: Int, total: Int) -> Map<String, Any> {
fn scan_digits(src: String, start: Int, total: Int) -> Map<String, Any> {
let i = start
let parts: [String] = native_list_empty()
let running = true
while running {
if i >= total {
let running = false
} else {
let ch: String = native_list_get(chars, i)
let ch: String = str_char_at(src, i)
if lex_is_digit(ch) {
let parts = native_list_append(parts, ch)
let i = i + 1
} else {
let running = false
}
}
}
{ "text": str_join(parts, ""), "pos": i }
// Use str_slice instead of building a parts list — O(1) allocation, O(n) copy.
{ "text": str_slice(src, start, i), "pos": i }
}
// scan_ident - advance i while chars[i] is alphanumeric or underscore
fn scan_ident(chars: [String], start: Int, total: Int) -> Map<String, Any> {
fn scan_ident(src: String, start: Int, total: Int) -> Map<String, Any> {
let i = start
let parts: [String] = native_list_empty()
let running = true
while running {
if i >= total {
let running = false
} else {
let ch: String = native_list_get(chars, i)
let ch: String = str_char_at(src, i)
if is_alnum_or_underscore(ch) {
let parts = native_list_append(parts, ch)
let i = i + 1
} else {
let running = false
}
}
}
{ "text": str_join(parts, ""), "pos": i }
// Use str_slice instead of building a parts list — O(1) allocation, O(n) copy.
{ "text": str_slice(src, start, i), "pos": i }
}
// -- Code-bearing string detection + comment strip ----------------------------
@@ -208,34 +206,16 @@ fn scan_ident(chars: [String], start: Int, total: Int) -> Map<String, Any> {
// looks_like_code - heuristic gate so we only strip strings that actually
// embed JS or CSS. Plain prose, hex blobs, JSON, etc. pass through verbatim.
fn substr_at(chars: [String], start: Int, total: Int, needle: String) -> Bool {
let nchars: [String] = native_string_chars(needle)
let nlen: Int = native_list_len(nchars)
fn substr_at(src: String, start: Int, total: Int, needle: String) -> Bool {
let nlen: Int = str_len(needle)
if start + nlen > total { return false }
let i = 0
let matched = true
while i < nlen {
let a: String = native_list_get(chars, start + i)
let b: String = native_list_get(nchars, i)
if a == b { let i = i + 1 } else { let matched = false; let i = nlen }
}
matched
// Use str_slice comparison instead of char-by-char loop.
str_eq(str_slice(src, start, start + nlen), needle)
}
fn str_has(s: String, needle: String) -> Bool {
let chars: [String] = native_string_chars(s)
let total: Int = native_list_len(chars)
let i = 0
let found = false
while i < total {
if substr_at(chars, i, total, needle) {
let found = true
let i = total
} else {
let i = i + 1
}
}
found
// Use the built-in str_contains which is implemented in native C — O(n) single pass.
str_contains(s, needle)
}
fn looks_like_code(s: String) -> Bool {
@@ -254,8 +234,7 @@ fn looks_like_code(s: String) -> Bool {
// comment opener: if the char immediately before '/' is ':', emit the '/'
// literally and advance one position.
fn strip_code_comments(s: String) -> String {
let chars: [String] = native_string_chars(s)
let total: Int = native_list_len(chars)
let total: Int = str_len(s)
let out_parts: [String] = native_list_empty()
let i = 0
let in_squote = false
@@ -263,7 +242,7 @@ fn strip_code_comments(s: String) -> String {
let in_btick = false
let prev = ""
while i < total {
let ch: String = native_list_get(chars, i)
let ch: String = str_char_at(s, i)
let in_js_string = false
if in_squote { let in_js_string = true }
if in_dquote { let in_js_string = true }
@@ -275,7 +254,7 @@ fn strip_code_comments(s: String) -> String {
let out_parts = native_list_append(out_parts, ch)
let next_i = i + 1
if next_i < total {
let nc: String = native_list_get(chars, next_i)
let nc: String = str_char_at(s, next_i)
let out_parts = native_list_append(out_parts, nc)
let prev = nc
let i = next_i + 1
@@ -304,7 +283,7 @@ fn strip_code_comments(s: String) -> String {
let next_i = i + 1
let next_ch = ""
if next_i < total {
let next_ch: String = native_list_get(chars, next_i)
let next_ch: String = str_char_at(s, next_i)
}
if ch == "/" {
@@ -323,7 +302,7 @@ fn strip_code_comments(s: String) -> String {
if i >= total {
let scanning = false
} else {
let lc: String = native_list_get(chars, i)
let lc: String = str_char_at(s, i)
if lc == "\n" {
let scanning = false
} else {
@@ -342,11 +321,11 @@ fn strip_code_comments(s: String) -> String {
if i >= total {
let scanning2 = false
} else {
let bc: String = native_list_get(chars, i)
let bc: String = str_char_at(s, i)
if bc == "*" {
let after = i + 1
if after < total {
let nc2: String = native_list_get(chars, after)
let nc2: String = str_char_at(s, after)
if nc2 == "/" {
let i = after + 1
let scanning2 = false
@@ -402,7 +381,7 @@ fn strip_code_comments(s: String) -> String {
// scan_string - scan a quoted string literal, handling \" escapes.
// Starts AFTER the opening quote. Returns { "text": content, "pos": i_after_close }
fn scan_string(chars: [String], start: Int, total: Int) -> Map<String, Any> {
fn scan_string(src: String, start: Int, total: Int) -> Map<String, Any> {
let i = start
let parts: [String] = native_list_empty()
let running = true
@@ -410,12 +389,12 @@ fn scan_string(chars: [String], start: Int, total: Int) -> Map<String, Any> {
if i >= total {
let running = false
} else {
let ch: String = native_list_get(chars, i)
let ch: String = str_char_at(src, i)
if ch == "\\" {
// escape: peek next char
let next_i = i + 1
if next_i < total {
let next_ch: String = native_list_get(chars, next_i)
let next_ch: String = str_char_at(src, next_i)
if next_ch == "\"" {
let parts = native_list_append(parts, "\"")
let i = next_i + 1
@@ -465,19 +444,17 @@ fn scan_string(chars: [String], start: Int, total: Int) -> Map<String, Any> {
// scan_interp_brace - scan from `start` (the char after `${`) to the matching
// `}`, tracking brace depth so inner braces (e.g. fn calls, map literals) are
// handled correctly. Returns { "text": inner_source, "pos": i_after_close }.
fn scan_interp_brace(chars: [String], start: Int, total: Int) -> Map<String, Any> {
fn scan_interp_brace(src: String, start: Int, total: Int) -> Map<String, Any> {
let i = start
let parts: [String] = native_list_empty()
let depth = 1
let running = true
while running {
if i >= total {
let running = false
} else {
let ch: String = native_list_get(chars, i)
let ch: String = str_char_at(src, i)
if ch == "{" {
let depth = depth + 1
let parts = native_list_append(parts, ch)
let i = i + 1
} else {
if ch == "}" {
@@ -487,17 +464,16 @@ fn scan_interp_brace(chars: [String], start: Int, total: Int) -> Map<String, Any
let i = i + 1
let running = false
} else {
let parts = native_list_append(parts, ch)
let i = i + 1
}
} else {
let parts = native_list_append(parts, ch)
let i = i + 1
}
}
}
}
{ "text": str_join(parts, ""), "pos": i }
// Use str_slice instead of parts list — the inner source is a contiguous substring.
{ "text": str_slice(src, start, i - 1), "pos": i }
}
// interp_tokens_append_all - copy every token from src into dst, skipping the
@@ -536,7 +512,7 @@ fn interp_tokens_append_all(dst: [Map<String, Any>], src: [Map<String, Any>]) ->
//
// Supported escape sequences: \" \n \t \r \\ \$ (literal dollar sign).
// Nested quotes inside ${} are not supported; use a variable instead.
fn scan_interp_string(chars: [String], start: Int, total: Int) -> Map<String, Any> {
fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
let i = start
let out_tokens: [Map<String, Any>] = native_list_empty()
let cur_part: [String] = native_list_empty()
@@ -548,13 +524,13 @@ fn scan_interp_string(chars: [String], start: Int, total: Int) -> Map<String, An
if i >= total {
let running = false
} else {
let ch: String = native_list_get(chars, i)
let ch: String = str_char_at(src, i)
if ch == "\\" {
// Escape sequence
let next_i = i + 1
if next_i < total {
let next_ch: String = native_list_get(chars, next_i)
let next_ch: String = str_char_at(src, next_i)
if next_ch == "$" {
// \$ => literal '$' (escape for interpolation syntax)
let cur_part = native_list_append(cur_part, "$")
@@ -602,7 +578,7 @@ fn scan_interp_string(chars: [String], start: Int, total: Int) -> Map<String, An
let next_i = i + 1
let is_interp = false
if next_i < total {
let next_ch: String = native_list_get(chars, next_i)
let next_ch: String = str_char_at(src, next_i)
if next_ch == "{" {
let is_interp = true
}
@@ -626,7 +602,7 @@ fn scan_interp_string(chars: [String], start: Int, total: Int) -> Map<String, An
let has_interp = true
// Scan brace-balanced expression source
let brace_result = scan_interp_brace(chars, next_i + 1, total)
let brace_result = scan_interp_brace(src, next_i + 1, total)
let expr_src: String = brace_result["text"]
let new_i: Int = brace_result["pos"]
let i = new_i
@@ -695,13 +671,16 @@ fn scan_interp_string(chars: [String], start: Int, total: Int) -> Map<String, An
// -- Main lexer ----------------------------------------------------------------
fn lex(source: String) -> [Map<String, Any>] {
let chars: [String] = native_string_chars(source)
let total: Int = native_list_len(chars)
// Use str_len + str_char_at instead of native_string_chars to avoid
// allocating a 400K-element character list in El's arena. For a 400KB
// source file, native_string_chars created ~400K permanent string allocations
// (one per character), consuming ~20MB of memory before lexing even started.
let total: Int = str_len(source)
let tokens: [Map<String, Any>] = native_list_empty()
let i: Int = 0
while i < total {
let ch: String = native_list_get(chars, i)
let ch: String = str_char_at(source, i)
// Skip whitespace
if lex_is_whitespace(ch) {
@@ -711,7 +690,7 @@ fn lex(source: String) -> [Map<String, Any>] {
if ch == "/" {
let next_i = i + 1
if next_i < total {
let next_ch: String = native_list_get(chars, next_i)
let next_ch: String = str_char_at(source, next_i)
if next_ch == "/" {
// skip to end of line
let i = i + 2
@@ -720,7 +699,7 @@ fn lex(source: String) -> [Map<String, Any>] {
if i >= total {
let running2 = false
} else {
let lch: String = native_list_get(chars, i)
let lch: String = str_char_at(source, i)
if lch == "\n" {
let running2 = false
} else {
@@ -743,7 +722,7 @@ fn lex(source: String) -> [Map<String, Any>] {
// sequence (Str Plus expr-tokens Plus Str ...) that the parser
// naturally assembles into a BinOp concat tree.
if ch == "\"" {
let interp_result = scan_interp_string(chars, i + 1, total)
let interp_result = scan_interp_string(source, i + 1, total)
let interp_toks: [Map<String, Any>] = interp_result["tokens"]
let new_pos: Int = interp_result["pos"]
let tokens = interp_tokens_append_all(tokens, interp_toks)
@@ -751,18 +730,18 @@ fn lex(source: String) -> [Map<String, Any>] {
} else {
// Number literal
if lex_is_digit(ch) {
let result = scan_digits(chars, i, total)
let result = scan_digits(source, i, total)
let num_text: String = result["text"]
let new_pos: Int = result["pos"]
// check for float (dot followed by digit)
if new_pos < total {
let dot_ch: String = native_list_get(chars, new_pos)
let dot_ch: String = str_char_at(source, new_pos)
if dot_ch == "." {
let after_dot = new_pos + 1
if after_dot < total {
let after_dot_ch: String = native_list_get(chars, after_dot)
let after_dot_ch: String = str_char_at(source, after_dot)
if lex_is_digit(after_dot_ch) {
let frac_result = scan_digits(chars, after_dot, total)
let frac_result = scan_digits(source, after_dot, total)
let frac_text: String = frac_result["text"]
let frac_pos: Int = frac_result["pos"]
let tokens = native_list_append(tokens, make_tok("Float", num_text + "." + frac_text))
@@ -786,7 +765,7 @@ fn lex(source: String) -> [Map<String, Any>] {
} else {
// Identifier or keyword
if lex_is_alpha(ch) || ch == "_" {
let result = scan_ident(chars, i, total)
let result = scan_ident(source, i, total)
let word: String = result["text"]
let new_pos: Int = result["pos"]
let kw = keyword_kind(word)
@@ -801,7 +780,7 @@ fn lex(source: String) -> [Map<String, Any>] {
let peek_i = i + 1
let peek_ch = ""
if peek_i < total {
let peek_ch: String = native_list_get(chars, peek_i)
let peek_ch: String = str_char_at(source, peek_i)
}
if ch == "=" {
@@ -930,7 +909,7 @@ fn lex(source: String) -> [Map<String, Any>] {
let peek2_i = i + 2
let peek2_ch = ""
if peek2_i < total {
let peek2_ch: String = native_list_get(chars, peek2_i)
let peek2_ch: String = str_char_at(source, peek2_i)
}
if peek_ch == "." {
if peek2_ch == "=" {