neuron-technologies/el
3
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

round-2-gamma: combine flat token list + char code dispatch — max round-2 savings

Browse Source 
Combines two orthogonal optimizations:
1. Flat token list (from beta): lex() returns [Any] with alternating kind/value
   pairs instead of [Map], eliminating one ElMap per token (~3 mallocs each).
   Parser updated: tok_kind(t,i) = t[2*i], tok_value(t,i) = t[2*i+1].

2. Char code dispatch (from alpha): lex() hot loop uses str_char_code -> Int
   instead of str_char_at -> strdup String for all character classification.
   Eliminates ~400K x 16B = 6.4MB of temporary string allocations.

scan_digits and scan_ident also updated to use str_char_code.

Result on main.el: 17.1MB -> 14.4MB peak RSS (-16%).
Self-hosting: PASS.
This commit is contained in:
Will Anderson
2026-05-05 15:46:20 -05:00
parent 1e67544c88
commit 1eef9928f4
3 changed files with 111 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lang/el-compiler/src/parser.el
+34 -30
View File
@@ -9,25 +9,28 @@
// The token list is passed as a parameter to all parse functions.
// native_list_get is used to index into it without cloning.
//
// Entry point: fn parse(tokens: [Map<String, Any>]) -> [Map<String, Any>]
// Entry point: fn parse(tokens: [Any]) -> [Map<String, Any>]
// -- Token access helpers ------------------------------------------------------
// Tokens is a flat [Any] list: tokens[2*i] = kind, tokens[2*i+1] = value.
// This avoids one ElMap allocation per token (~112B each), saving ~4MB on large
// programs. All callers use these helpers -- only these three need updating.
fn tok_at(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
native_list_get(tokens, pos)
fn tok_at(tokens: [Any], pos: Int) -> Map<String, Any> {
let kind: String = native_list_get(tokens, pos * 2)
let value: String = native_list_get(tokens, pos * 2 + 1)
{ "kind": kind, "value": value }
}
fn tok_kind(tokens: [Map<String, Any>], pos: Int) -> String {
let t = native_list_get(tokens, pos)
t["kind"]
fn tok_kind(tokens: [Any], pos: Int) -> String {
native_list_get(tokens, pos * 2)
}
fn tok_value(tokens: [Map<String, Any>], pos: Int) -> String {
let t = native_list_get(tokens, pos)
t["value"]
fn tok_value(tokens: [Any], pos: Int) -> String {
native_list_get(tokens, pos * 2 + 1)
}
fn expect(tokens: [Map<String, Any>], pos: Int, kind: String) -> Int {
fn expect(tokens: [Any], pos: Int, kind: String) -> Int {
let k = tok_kind(tokens, pos)
if k == kind {
return pos + 1
@@ -46,7 +49,7 @@ fn make_result(node: Map<String, Any>, pos: Int) -> Map<String, Any> {
// Skips over a type annotation, returning the new position.
// Types can be: Ident, [Type], Map<K,V>, Type?, Type<Type,...>
fn skip_type(tokens: [Map<String, Any>], pos: Int) -> Int {
fn skip_type(tokens: [Any], pos: Int) -> Int {
let k = tok_kind(tokens, pos)
// Array type: [Type]
if k == "LBracket" {
@@ -103,7 +106,7 @@ fn skip_type(tokens: [Map<String, Any>], pos: Int) -> Int {
// -- Parameter list ------------------------------------------------------------
// Parses (name: Type, name: Type, ...) - returns { "params": [...], "pos": ... }
fn parse_params(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_params(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "LParen")
let params: [Map<String, Any>] = native_list_empty()
let running = true
@@ -292,7 +295,7 @@ fn is_void_element(name: String) -> Bool {
// Collect tokens as text content until we hit Lt, LBrace, Eof, or a
// closing-tag marker (Lt Slash). Returns { "text": "...", "pos": p }
fn parse_html_text_tokens(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_html_text_tokens(tokens: [Any], pos: Int) -> Map<String, Any> {
let parts: [String] = native_list_empty()
let p = pos
let running = true
@@ -322,7 +325,7 @@ fn parse_html_text_tokens(tokens: [Map<String, Any>], pos: Int) -> Map<String, A
// Parse an attribute list: (attrname | attrname="val" | attrname={expr})*
// Stops at Gt or Slash (for self-closing />).
fn parse_html_attrs(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_html_attrs(tokens: [Any], pos: Int) -> Map<String, Any> {
let attrs: [Map<String, Any>] = native_list_empty()
let p = pos
let running = true
@@ -374,7 +377,7 @@ fn parse_html_attrs(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
// Parse the children of an HTML element until we see the closing tag </tag>
// or EOF. Returns { "children": [...], "pos": p_after_closing_tag }
fn parse_html_children(tokens: [Map<String, Any>], pos: Int, parent_tag: String) -> Map<String, Any> {
fn parse_html_children(tokens: [Any], pos: Int, parent_tag: String) -> Map<String, Any> {
let children: [Map<String, Any>] = native_list_empty()
let p = pos
let running = true
@@ -514,14 +517,14 @@ fn parse_html_children(tokens: [Map<String, Any>], pos: Int, parent_tag: String)
// Parse body of {#each} until {/each}. Mirrors parse_html_children but
// stops at the {/each} sentinel rather than a closing element tag.
fn parse_html_each_body(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_html_each_body(tokens: [Any], pos: Int) -> Map<String, Any> {
parse_html_children(tokens, pos, "__each__")
}
// Parse a single HTML element: <tag attrs> children </tag>
// or self-closing: <tag attrs/>
// Pos points to the Lt token.
fn parse_html_element(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_html_element(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = pos
// consume <
let p = expect(tokens, p, "Lt")
@@ -558,7 +561,7 @@ fn parse_html_element(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any>
// Entry point for HTML template parsing.
// Pos points to Lt (or Lt Not for <!doctype>).
// May parse an optional <!doctype html> prefix followed by the root element.
fn parse_html_template(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_html_template(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = pos
// Check for <!doctype html>
let doctype = false
@@ -596,7 +599,7 @@ fn parse_html_template(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any>
make_result({ "expr": "HtmlTemplate", "root": root_with_doctype }, p)
}
fn parse_primary(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_primary(tokens: [Any], pos: Int) -> Map<String, Any> {
let k = tok_kind(tokens, pos)
let v = tok_value(tokens, pos)
@@ -819,7 +822,7 @@ fn parse_primary(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "expr": "Nil" }, pos + 1)
}
fn parse_if(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_if(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "If")
// Suppress Map-literal parsing in the cond so a stray `{` (the start
// of the then-block) isn't gobbled as a Map.
@@ -855,7 +858,7 @@ fn parse_if(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "expr": "If", "cond": cond, "then": then_stmts, "else": else_stmts, "has_else": has_else }, p)
}
fn parse_match(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_match(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "Match")
let prev_no_block: String = state_get("__no_block_expr")
state_set("__no_block_expr", "1")
@@ -895,7 +898,7 @@ fn parse_match(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "expr": "Match", "subject": subject, "arms": arms }, p)
}
fn parse_pattern(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_pattern(tokens: [Any], pos: Int) -> Map<String, Any> {
let k = tok_kind(tokens, pos)
if k == "Ident" {
let v = tok_value(tokens, pos)
@@ -924,7 +927,7 @@ fn parse_pattern(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "pattern": "Wildcard" }, pos + 1)
}
fn parse_for_expr(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_for_expr(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "For")
let item_name = tok_value(tokens, p)
let p = p + 1
@@ -941,7 +944,7 @@ fn parse_for_expr(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "expr": "For", "item": item_name, "list": list_expr, "body": body }, p)
}
fn parse_block(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_block(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "LBrace")
let stmts: [Map<String, Any>] = native_list_empty()
let running = true
@@ -998,7 +1001,7 @@ fn is_duration_unit(name: String) -> Bool {
false
}
fn parse_postfix(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_postfix(tokens: [Any], pos: Int) -> Map<String, Any> {
let r = parse_primary(tokens, pos)
let node = r["node"]
let p = r["pos"]
@@ -1115,7 +1118,7 @@ fn is_binop(kind: String) -> Bool {
false
}
fn parse_binop(tokens: [Map<String, Any>], pos: Int, min_prec: Int) -> Map<String, Any> {
fn parse_binop(tokens: [Any], pos: Int, min_prec: Int) -> Map<String, Any> {
let r = parse_postfix(tokens, pos)
let left = r["node"]
let p = r["pos"]
@@ -1140,13 +1143,13 @@ fn parse_binop(tokens: [Map<String, Any>], pos: Int, min_prec: Int) -> Map<Strin
make_result(left, p)
}
fn parse_expr(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_expr(tokens: [Any], pos: Int) -> Map<String, Any> {
parse_binop(tokens, pos, 1)
}
// -- Statement parsing ---------------------------------------------------------
fn parse_stmt(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
fn parse_stmt(tokens: [Any], pos: Int) -> Map<String, Any> {
let k = tok_kind(tokens, pos)
// let binding
@@ -1619,8 +1622,9 @@ fn parse_stmt(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
// -- Top-level parse ------------------------------------------------------------
fn parse(tokens: [Map<String, Any>]) -> [Map<String, Any>] {
let total: Int = native_list_len(tokens)
fn parse(tokens: [Any]) -> [Map<String, Any>] {
// Flat list: 2 entries per token, so divide by 2 for token count.
let total: Int = native_list_len(tokens) / 2
let stmts: [Map<String, Any>] = native_list_empty()
let pos: Int = 0
let running = true