neuron-technologies/el
3
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

round-2-gamma: combine flat token list + char code dispatch — max round-2 savings

Browse Source 
Combines two orthogonal optimizations:
1. Flat token list (from beta): lex() returns [Any] with alternating kind/value
   pairs instead of [Map], eliminating one ElMap per token (~3 mallocs each).
   Parser updated: tok_kind(t,i) = t[2*i], tok_value(t,i) = t[2*i+1].

2. Char code dispatch (from alpha): lex() hot loop uses str_char_code -> Int
   instead of str_char_at -> strdup String for all character classification.
   Eliminates ~400K x 16B = 6.4MB of temporary string allocations.

scan_digits and scan_ident also updated to use str_char_code.

Result on main.el: 17.1MB -> 14.4MB peak RSS (-16%).
Self-hosting: PASS.
This commit is contained in:
Will Anderson
2026-05-05 15:46:20 -05:00
parent 1e67544c88
commit 1eef9928f4
3 changed files with 111 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lang/el-compiler/src/lexer.el
+73 -68
View File
@@ -136,8 +136,11 @@ fn lex_is_whitespace(ch: String) -> Bool {
false
}
fn make_tok(kind: String, value: String) -> Map<String, Any> {
{ "kind": kind, "value": value }
// tok_append — append a (kind, value) pair to a flat token list.
// Returns the updated list. Gamma combines flat-list + char-code for max savings.
fn tok_append(tokens: [Any], kind: String, value: String) -> [Any] {
let tokens = native_list_append(tokens, kind)
native_list_append(tokens, value)
}
// -- Keyword lookup ------------------------------------------------------------
@@ -515,20 +518,21 @@ fn scan_interp_brace(src: String, start: Int, total: Int) -> Map<String, Any> {
{ "text": str_slice(src, start, i - 1), "pos": i }
}
// interp_tokens_append_all - copy every token from src into dst, skipping the
// trailing Eof sentinel that lex() always appends. Returns the updated dst list.
fn interp_tokens_append_all(dst: [Map<String, Any>], src: [Map<String, Any>]) -> [Map<String, Any>] {
// interp_tokens_append_all - copy every (kind, value) pair from flat src list
// into flat dst list, skipping the trailing Eof pair that lex() always appends.
fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] {
let src_len: Int = native_list_len(src)
let j = 0
let result = dst
while j < src_len {
let tok: Map<String, Any> = native_list_get(src, j)
let tk: String = tok["kind"]
if tk == "Eof" {
let kind: String = native_list_get(src, j)
if kind == "Eof" {
let j = src_len
} else {
let result = native_list_append(result, tok)
let j = j + 1
let val: String = native_list_get(src, j + 1)
let result = native_list_append(result, kind)
let result = native_list_append(result, val)
let j = j + 2
}
}
result
@@ -553,7 +557,7 @@ fn interp_tokens_append_all(dst: [Map<String, Any>], src: [Map<String, Any>]) ->
// Nested quotes inside ${} are not supported; use a variable instead.
fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
let i = start
let out_tokens: [Map<String, Any>] = native_list_empty()
let out_tokens: [Any] = native_list_empty()
let cur_part: [String] = native_list_empty()
let has_interp = false
let need_plus = false
@@ -628,13 +632,13 @@ fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
if part_len > 0 {
let part_text = str_join(cur_part, "")
if need_plus {
let out_tokens = native_list_append(out_tokens, make_tok("Plus", "+"))
let out_tokens = tok_append(out_tokens, "Plus", "+")
}
let clean_part = part_text
if looks_like_code(part_text) {
let clean_part = strip_code_comments(part_text)
}
let out_tokens = native_list_append(out_tokens, make_tok("Str", clean_part))
let out_tokens = tok_append(out_tokens, "Str", clean_part)
let need_plus = true
}
let cur_part = native_list_empty()
@@ -651,19 +655,20 @@ fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
// age + 1) are parsed as a grouped sub-expression
// rather than merging with the surrounding concat
// Plus tokens at the wrong precedence level.
let inner_toks: [Map<String, Any>] = lex(expr_src)
let inner_toks: [Any] = lex(expr_src)
let inner_len: Int = native_list_len(inner_toks)
if need_plus {
let out_tokens = native_list_append(out_tokens, make_tok("Plus", "+"))
let out_tokens = tok_append(out_tokens, "Plus", "+")
}
// Empty interpolation ${} => empty string segment
if inner_len <= 1 {
let out_tokens = native_list_append(out_tokens, make_tok("Str", ""))
// inner_len <= 2 = only the Eof pair (kind="Eof", value="")
if inner_len <= 2 {
let out_tokens = tok_append(out_tokens, "Str", "")
} else {
let out_tokens = native_list_append(out_tokens, make_tok("LParen", "("))
let out_tokens = tok_append(out_tokens, "LParen", "(")
let out_tokens = interp_tokens_append_all(out_tokens, inner_toks)
let out_tokens = native_list_append(out_tokens, make_tok("RParen", ")"))
let out_tokens = tok_append(out_tokens, "RParen", ")")
}
let need_plus = true
} else {
@@ -691,9 +696,9 @@ fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
let clean_part = strip_code_comments(part_text)
}
if need_plus {
let out_tokens = native_list_append(out_tokens, make_tok("Plus", "+"))
let out_tokens = tok_append(out_tokens, "Plus", "+")
}
let out_tokens = native_list_append(out_tokens, make_tok("Str", clean_part))
let out_tokens = tok_append(out_tokens, "Str", clean_part)
}
} else {
// Plain string with no interpolation - same behaviour as old scan_string
@@ -701,7 +706,7 @@ fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
if looks_like_code(part_text) {
let clean_text = strip_code_comments(part_text)
}
let out_tokens = native_list_append(out_tokens, make_tok("Str", clean_text))
let out_tokens = tok_append(out_tokens, "Str", clean_text)
}
{ "tokens": out_tokens, "pos": i }
@@ -716,12 +721,12 @@ fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
// '(' = 40, ')' = 41, '{' = 123, '}' = 125, '[' = 91, ']' = 93
// ',' = 44, '.' = 46, ';' = 59, '@' = 64, '?' = 63
fn lex(source: String) -> [Map<String, Any>] {
fn lex(source: String) -> [Any] {
// Use str_char_code (returns Int) instead of str_char_at (returns strdup String)
// for all character classification in the hot loop. For a 400KB source,
// str_char_at allocates ~400K × 16B = ~6.4MB of temporary strings.
let total: Int = str_len(source)
let tokens: [Map<String, Any>] = native_list_empty()
let tokens: [Any] = native_list_empty()
let i: Int = 0
while i < total {
@@ -753,18 +758,18 @@ fn lex(source: String) -> [Map<String, Any>] {
}
}
} else {
let tokens = native_list_append(tokens, make_tok("Slash", "/"))
let tokens = tok_append(tokens, "Slash", "/")
let i = i + 1
}
} else {
let tokens = native_list_append(tokens, make_tok("Slash", "/"))
let tokens = tok_append(tokens, "Slash", "/")
let i = i + 1
}
} else {
// String literal: '"' = 34
if c == 34 {
let interp_result = scan_interp_string(source, i + 1, total)
let interp_toks: [Map<String, Any>] = interp_result["tokens"]
let interp_toks: [Any] = interp_result["tokens"]
let new_pos: Int = interp_result["pos"]
let tokens = interp_tokens_append_all(tokens, interp_toks)
let i = new_pos
@@ -785,22 +790,22 @@ fn lex(source: String) -> [Map<String, Any>] {
let frac_result = scan_digits(source, after_dot, total)
let frac_text: String = frac_result["text"]
let frac_pos: Int = frac_result["pos"]
let tokens = native_list_append(tokens, make_tok("Float", num_text + "." + frac_text))
let tokens = tok_append(tokens, "Float", num_text + "." + frac_text)
let i = frac_pos
} else {
let tokens = native_list_append(tokens, make_tok("Int", num_text))
let tokens = tok_append(tokens, "Int", num_text)
let i = new_pos
}
} else {
let tokens = native_list_append(tokens, make_tok("Int", num_text))
let tokens = tok_append(tokens, "Int", num_text)
let i = new_pos
}
} else {
let tokens = native_list_append(tokens, make_tok("Int", num_text))
let tokens = tok_append(tokens, "Int", num_text)
let i = new_pos
}
} else {
let tokens = native_list_append(tokens, make_tok("Int", num_text))
let tokens = tok_append(tokens, "Int", num_text)
let i = new_pos
}
} else {
@@ -811,9 +816,9 @@ fn lex(source: String) -> [Map<String, Any>] {
let new_pos: Int = result["pos"]
let kw = keyword_kind(word)
if kw == "" {
let tokens = native_list_append(tokens, make_tok("Ident", word))
let tokens = tok_append(tokens, "Ident", word)
} else {
let tokens = native_list_append(tokens, make_tok(kw, word))
let tokens = tok_append(tokens, kw, word)
}
let i = new_pos
} else {
@@ -827,15 +832,15 @@ fn lex(source: String) -> [Map<String, Any>] {
if c == 61 {
// '=' = 61
if peek_c == 61 {
let tokens = native_list_append(tokens, make_tok("EqEq", "=="))
let tokens = tok_append(tokens, "EqEq", "==")
let i = i + 2
} else {
if peek_c == 62 {
// '>' = 62
let tokens = native_list_append(tokens, make_tok("FatArrow", "=>"))
let tokens = tok_append(tokens, "FatArrow", "=>")
let i = i + 2
} else {
let tokens = native_list_append(tokens, make_tok("Eq", "="))
let tokens = tok_append(tokens, "Eq", "=")
let i = i + 1
}
}
@@ -843,37 +848,37 @@ fn lex(source: String) -> [Map<String, Any>] {
if c == 33 {
// '!' = 33
if peek_c == 61 {
let tokens = native_list_append(tokens, make_tok("NotEq", "!="))
let tokens = tok_append(tokens, "NotEq", "!=")
let i = i + 2
} else {
let tokens = native_list_append(tokens, make_tok("Not", "!"))
let tokens = tok_append(tokens, "Not", "!")
let i = i + 1
}
} else {
if c == 60 {
// '<' = 60
if peek_c == 61 {
let tokens = native_list_append(tokens, make_tok("LtEq", "<="))
let tokens = tok_append(tokens, "LtEq", "<=")
let i = i + 2
} else {
let tokens = native_list_append(tokens, make_tok("Lt", "<"))
let tokens = tok_append(tokens, "Lt", "<")
let i = i + 1
}
} else {
if c == 62 {
// '>' = 62
if peek_c == 61 {
let tokens = native_list_append(tokens, make_tok("GtEq", ">="))
let tokens = tok_append(tokens, "GtEq", ">=")
let i = i + 2
} else {
let tokens = native_list_append(tokens, make_tok("Gt", ">"))
let tokens = tok_append(tokens, "Gt", ">")
let i = i + 1
}
} else {
if c == 38 {
// '&' = 38
if peek_c == 38 {
let tokens = native_list_append(tokens, make_tok("And", "&&"))
let tokens = tok_append(tokens, "And", "&&")
let i = i + 2
} else {
let i = i + 1
@@ -882,15 +887,15 @@ fn lex(source: String) -> [Map<String, Any>] {
if c == 124 {
// '|' = 124
if peek_c == 124 {
let tokens = native_list_append(tokens, make_tok("Or", "||"))
let tokens = tok_append(tokens, "Or", "||")
let i = i + 2
} else {
if peek_c == 62 {
// '>' = 62
let tokens = native_list_append(tokens, make_tok("PipeOp", "|>"))
let tokens = tok_append(tokens, "PipeOp", "|>")
let i = i + 2
} else {
let tokens = native_list_append(tokens, make_tok("Pipe", "|"))
let tokens = tok_append(tokens, "Pipe", "|")
let i = i + 1
}
}
@@ -899,71 +904,71 @@ fn lex(source: String) -> [Map<String, Any>] {
// '-' = 45
if peek_c == 62 {
// '>' = 62
let tokens = native_list_append(tokens, make_tok("Arrow", "->"))
let tokens = tok_append(tokens, "Arrow", "->")
let i = i + 2
} else {
let tokens = native_list_append(tokens, make_tok("Minus", "-"))
let tokens = tok_append(tokens, "Minus", "-")
let i = i + 1
}
} else {
if c == 58 {
// ':' = 58
if peek_c == 58 {
let tokens = native_list_append(tokens, make_tok("ColonColon", "::"))
let tokens = tok_append(tokens, "ColonColon", "::")
let i = i + 2
} else {
let tokens = native_list_append(tokens, make_tok("Colon", ":"))
let tokens = tok_append(tokens, "Colon", ":")
let i = i + 1
}
} else {
if c == 43 {
// '+' = 43
let tokens = native_list_append(tokens, make_tok("Plus", "+"))
let tokens = tok_append(tokens, "Plus", "+")
let i = i + 1
} else {
if c == 42 {
// '*' = 42
let tokens = native_list_append(tokens, make_tok("Star", "*"))
let tokens = tok_append(tokens, "Star", "*")
let i = i + 1
} else {
if c == 37 {
// '%' = 37
let tokens = native_list_append(tokens, make_tok("Percent", "%"))
let tokens = tok_append(tokens, "Percent", "%")
let i = i + 1
} else {
if c == 40 {
// '(' = 40
let tokens = native_list_append(tokens, make_tok("LParen", "("))
let tokens = tok_append(tokens, "LParen", "(")
let i = i + 1
} else {
if c == 41 {
// ')' = 41
let tokens = native_list_append(tokens, make_tok("RParen", ")"))
let tokens = tok_append(tokens, "RParen", ")")
let i = i + 1
} else {
if c == 123 {
// '{' = 123
let tokens = native_list_append(tokens, make_tok("LBrace", "{"))
let tokens = tok_append(tokens, "LBrace", "{")
let i = i + 1
} else {
if c == 125 {
// '}' = 125
let tokens = native_list_append(tokens, make_tok("RBrace", "}"))
let tokens = tok_append(tokens, "RBrace", "}")
let i = i + 1
} else {
if c == 91 {
// '[' = 91
let tokens = native_list_append(tokens, make_tok("LBracket", "["))
let tokens = tok_append(tokens, "LBracket", "[")
let i = i + 1
} else {
if c == 93 {
// ']' = 93
let tokens = native_list_append(tokens, make_tok("RBracket", "]"))
let tokens = tok_append(tokens, "RBracket", "]")
let i = i + 1
} else {
if c == 44 {
// ',' = 44
let tokens = native_list_append(tokens, make_tok("Comma", ","))
let tokens = tok_append(tokens, "Comma", ",")
let i = i + 1
} else {
if c == 46 {
@@ -977,30 +982,30 @@ fn lex(source: String) -> [Map<String, Any>] {
// '..' prefix
if peek2_c == 61 {
// '..=' = 46 46 61
let tokens = native_list_append(tokens, make_tok("DotDotEq", "..="))
let tokens = tok_append(tokens, "DotDotEq", "..=")
let i = i + 3
} else {
let tokens = native_list_append(tokens, make_tok("DotDot", ".."))
let tokens = tok_append(tokens, "DotDot", "..")
let i = i + 2
}
} else {
let tokens = native_list_append(tokens, make_tok("Dot", "."))
let tokens = tok_append(tokens, "Dot", ".")
let i = i + 1
}
} else {
if c == 59 {
// ';' = 59
let tokens = native_list_append(tokens, make_tok("Semicolon", ";"))
let tokens = tok_append(tokens, "Semicolon", ";")
let i = i + 1
} else {
if c == 64 {
// '@' = 64
let tokens = native_list_append(tokens, make_tok("At", "@"))
let tokens = tok_append(tokens, "At", "@")
let i = i + 1
} else {
if c == 63 {
// '?' = 63
let tokens = native_list_append(tokens, make_tok("QuestionMark", "?"))
let tokens = tok_append(tokens, "QuestionMark", "?")
let i = i + 1
} else {
// unknown char - skip
@@ -1034,6 +1039,6 @@ fn lex(source: String) -> [Map<String, Any>] {
}
}
let tokens = native_list_append(tokens, make_tok("Eof", ""))
let tokens = tok_append(tokens, "Eof", "")
tokens
}