neuron-technologies/el
3
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

round-2-gamma: combine flat token list + char code dispatch — max round-2 savings

Browse Source 
Combines two orthogonal optimizations:
1. Flat token list (from beta): lex() returns [Any] with alternating kind/value
   pairs instead of [Map], eliminating one ElMap per token (~3 mallocs each).
   Parser updated: tok_kind(t,i) = t[2*i], tok_value(t,i) = t[2*i+1].

2. Char code dispatch (from alpha): lex() hot loop uses str_char_code -> Int
   instead of str_char_at -> strdup String for all character classification.
   Eliminates ~400K x 16B = 6.4MB of temporary string allocations.

scan_digits and scan_ident also updated to use str_char_code.

Result on main.el: 17.1MB -> 14.4MB peak RSS (-16%).
Self-hosting: PASS.
This commit is contained in:
Will Anderson
2026-05-05 15:46:20 -05:00
parent 1e67544c88
commit 1eef9928f4
3 changed files with 111 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lang/el-compiler/src/compiler.el
+4 -4
View File
@@ -21,7 +21,7 @@ import "codegen-js.el"
// compile — full pipeline (C target): source string -> C source string // compile — full pipeline (C target): source string -> C source string
fn compile(source: String) -> String { fn compile(source: String) -> String {
let tokens: [Map<String, Any>] = lex(source) let tokens: [Any] = lex(source)
let stmts: [Map<String, Any>] = parse(tokens) let stmts: [Map<String, Any>] = parse(tokens)
// Token list is no longer needed after parsing — release it to free memory // Token list is no longer needed after parsing — release it to free memory
// before codegen allocates its own working data on large source files. // before codegen allocates its own working data on large source files.
@@ -31,7 +31,7 @@ fn compile(source: String) -> String {
// compile_js — full pipeline (JS target, module mode): source string -> JS source string // compile_js — full pipeline (JS target, module mode): source string -> JS source string
fn compile_js(source: String) -> String { fn compile_js(source: String) -> String {
let tokens: [Map<String, Any>] = lex(source) let tokens: [Any] = lex(source)
let stmts: [Map<String, Any>] = parse(tokens) let stmts: [Map<String, Any>] = parse(tokens)
// Token list is no longer needed after parsing — release it to free memory. // Token list is no longer needed after parsing — release it to free memory.
el_release(tokens) el_release(tokens)
@@ -41,7 +41,7 @@ fn compile_js(source: String) -> String {
// compile_js_with_bundle — JS target in bundle mode. // compile_js_with_bundle — JS target in bundle mode.
// Reads el_runtime.js from runtime_path and inlines it inside an IIFE. // Reads el_runtime.js from runtime_path and inlines it inside an IIFE.
fn compile_js_with_bundle(source: String, runtime_path: String) -> String { fn compile_js_with_bundle(source: String, runtime_path: String) -> String {
let tokens: [Map<String, Any>] = lex(source) let tokens: [Any] = lex(source)
let stmts: [Map<String, Any>] = parse(tokens) let stmts: [Map<String, Any>] = parse(tokens)
el_release(tokens) el_release(tokens)
let runtime_content: String = fs_read(runtime_path) let runtime_content: String = fs_read(runtime_path)
@@ -501,7 +501,7 @@ fn main() -> Void {
// (without inlining imports) and write out a .elh file alongside the .c. // (without inlining imports) and write out a .elh file alongside the .c.
if do_emit_header { if do_emit_header {
let raw_source: String = fs_read(src_path) let raw_source: String = fs_read(src_path)
let hdr_tokens: [Map<String, Any>] = lex(raw_source) let hdr_tokens: [Any] = lex(raw_source)
let hdr_stmts: [Map<String, Any>] = parse(hdr_tokens) let hdr_stmts: [Map<String, Any>] = parse(hdr_tokens)
el_release(hdr_tokens) el_release(hdr_tokens)
let hdr_path: String = str_slice(src_path, 0, str_len(src_path) - 3) + ".elh" let hdr_path: String = str_slice(src_path, 0, str_len(src_path) - 3) + ".elh"
lang/el-compiler/src/lexer.el
+73 -68
View File
@@ -136,8 +136,11 @@ fn lex_is_whitespace(ch: String) -> Bool {
false false
} }
fn make_tok(kind: String, value: String) -> Map<String, Any> { // tok_append — append a (kind, value) pair to a flat token list.
{ "kind": kind, "value": value } // Returns the updated list. Gamma combines flat-list + char-code for max savings.
fn tok_append(tokens: [Any], kind: String, value: String) -> [Any] {
let tokens = native_list_append(tokens, kind)
native_list_append(tokens, value)
} }
// -- Keyword lookup ------------------------------------------------------------ // -- Keyword lookup ------------------------------------------------------------
@@ -515,20 +518,21 @@ fn scan_interp_brace(src: String, start: Int, total: Int) -> Map<String, Any> {
{ "text": str_slice(src, start, i - 1), "pos": i } { "text": str_slice(src, start, i - 1), "pos": i }
} }
// interp_tokens_append_all - copy every token from src into dst, skipping the // interp_tokens_append_all - copy every (kind, value) pair from flat src list
// trailing Eof sentinel that lex() always appends. Returns the updated dst list. // into flat dst list, skipping the trailing Eof pair that lex() always appends.
fn interp_tokens_append_all(dst: [Map<String, Any>], src: [Map<String, Any>]) -> [Map<String, Any>] { fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] {
let src_len: Int = native_list_len(src) let src_len: Int = native_list_len(src)
let j = 0 let j = 0
let result = dst let result = dst
while j < src_len { while j < src_len {
let tok: Map<String, Any> = native_list_get(src, j) let kind: String = native_list_get(src, j)
let tk: String = tok["kind"] if kind == "Eof" {
if tk == "Eof" {
let j = src_len let j = src_len
} else { } else {
let result = native_list_append(result, tok) let val: String = native_list_get(src, j + 1)
let j = j + 1 let result = native_list_append(result, kind)
let result = native_list_append(result, val)
let j = j + 2
} }
} }
result result
@@ -553,7 +557,7 @@ fn interp_tokens_append_all(dst: [Map<String, Any>], src: [Map<String, Any>]) ->
// Nested quotes inside ${} are not supported; use a variable instead. // Nested quotes inside ${} are not supported; use a variable instead.
fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> { fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
let i = start let i = start
let out_tokens: [Map<String, Any>] = native_list_empty() let out_tokens: [Any] = native_list_empty()
let cur_part: [String] = native_list_empty() let cur_part: [String] = native_list_empty()
let has_interp = false let has_interp = false
let need_plus = false let need_plus = false
@@ -628,13 +632,13 @@ fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
if part_len > 0 { if part_len > 0 {
let part_text = str_join(cur_part, "") let part_text = str_join(cur_part, "")
if need_plus { if need_plus {
let out_tokens = native_list_append(out_tokens, make_tok("Plus", "+")) let out_tokens = tok_append(out_tokens, "Plus", "+")
} }
let clean_part = part_text let clean_part = part_text
if looks_like_code(part_text) { if looks_like_code(part_text) {
let clean_part = strip_code_comments(part_text) let clean_part = strip_code_comments(part_text)
} }
let out_tokens = native_list_append(out_tokens, make_tok("Str", clean_part)) let out_tokens = tok_append(out_tokens, "Str", clean_part)
let need_plus = true let need_plus = true
} }
let cur_part = native_list_empty() let cur_part = native_list_empty()
@@ -651,19 +655,20 @@ fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
// age + 1) are parsed as a grouped sub-expression // age + 1) are parsed as a grouped sub-expression
// rather than merging with the surrounding concat // rather than merging with the surrounding concat
// Plus tokens at the wrong precedence level. // Plus tokens at the wrong precedence level.
let inner_toks: [Map<String, Any>] = lex(expr_src) let inner_toks: [Any] = lex(expr_src)
let inner_len: Int = native_list_len(inner_toks) let inner_len: Int = native_list_len(inner_toks)
if need_plus { if need_plus {
let out_tokens = native_list_append(out_tokens, make_tok("Plus", "+")) let out_tokens = tok_append(out_tokens, "Plus", "+")
} }
// Empty interpolation ${} => empty string segment // Empty interpolation ${} => empty string segment
if inner_len <= 1 { // inner_len <= 2 = only the Eof pair (kind="Eof", value="")
let out_tokens = native_list_append(out_tokens, make_tok("Str", "")) if inner_len <= 2 {
let out_tokens = tok_append(out_tokens, "Str", "")
} else { } else {
let out_tokens = native_list_append(out_tokens, make_tok("LParen", "(")) let out_tokens = tok_append(out_tokens, "LParen", "(")
let out_tokens = interp_tokens_append_all(out_tokens, inner_toks) let out_tokens = interp_tokens_append_all(out_tokens, inner_toks)
let out_tokens = native_list_append(out_tokens, make_tok("RParen", ")")) let out_tokens = tok_append(out_tokens, "RParen", ")")
} }
let need_plus = true let need_plus = true
} else { } else {
@@ -691,9 +696,9 @@ fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
let clean_part = strip_code_comments(part_text) let clean_part = strip_code_comments(part_text)
} }
if need_plus { if need_plus {
let out_tokens = native_list_append(out_tokens, make_tok("Plus", "+")) let out_tokens = tok_append(out_tokens, "Plus", "+")
} }
let out_tokens = native_list_append(out_tokens, make_tok("Str", clean_part)) let out_tokens = tok_append(out_tokens, "Str", clean_part)
} }
} else { } else {
// Plain string with no interpolation - same behaviour as old scan_string // Plain string with no interpolation - same behaviour as old scan_string
@@ -701,7 +706,7 @@ fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
if looks_like_code(part_text) { if looks_like_code(part_text) {
let clean_text = strip_code_comments(part_text) let clean_text = strip_code_comments(part_text)
} }
let out_tokens = native_list_append(out_tokens, make_tok("Str", clean_text)) let out_tokens = tok_append(out_tokens, "Str", clean_text)
} }
{ "tokens": out_tokens, "pos": i } { "tokens": out_tokens, "pos": i }
@@ -716,12 +721,12 @@ fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
// '(' = 40, ')' = 41, '{' = 123, '}' = 125, '[' = 91, ']' = 93 // '(' = 40, ')' = 41, '{' = 123, '}' = 125, '[' = 91, ']' = 93
// ',' = 44, '.' = 46, ';' = 59, '@' = 64, '?' = 63 // ',' = 44, '.' = 46, ';' = 59, '@' = 64, '?' = 63
fn lex(source: String) -> [Map<String, Any>] { fn lex(source: String) -> [Any] {
// Use str_char_code (returns Int) instead of str_char_at (returns strdup String) // Use str_char_code (returns Int) instead of str_char_at (returns strdup String)
// for all character classification in the hot loop. For a 400KB source, // for all character classification in the hot loop. For a 400KB source,
// str_char_at allocates ~400K × 16B = ~6.4MB of temporary strings. // str_char_at allocates ~400K × 16B = ~6.4MB of temporary strings.
let total: Int = str_len(source) let total: Int = str_len(source)
let tokens: [Map<String, Any>] = native_list_empty() let tokens: [Any] = native_list_empty()
let i: Int = 0 let i: Int = 0
while i < total { while i < total {
@@ -753,18 +758,18 @@ fn lex(source: String) -> [Map<String, Any>] {
} }
} }
} else { } else {
let tokens = native_list_append(tokens, make_tok("Slash", "/")) let tokens = tok_append(tokens, "Slash", "/")
let i = i + 1 let i = i + 1
} }
} else { } else {
let tokens = native_list_append(tokens, make_tok("Slash", "/")) let tokens = tok_append(tokens, "Slash", "/")
let i = i + 1 let i = i + 1
} }
} else { } else {
// String literal: '"' = 34 // String literal: '"' = 34
if c == 34 { if c == 34 {
let interp_result = scan_interp_string(source, i + 1, total) let interp_result = scan_interp_string(source, i + 1, total)
let interp_toks: [Map<String, Any>] = interp_result["tokens"] let interp_toks: [Any] = interp_result["tokens"]
let new_pos: Int = interp_result["pos"] let new_pos: Int = interp_result["pos"]
let tokens = interp_tokens_append_all(tokens, interp_toks) let tokens = interp_tokens_append_all(tokens, interp_toks)
let i = new_pos let i = new_pos
@@ -785,22 +790,22 @@ fn lex(source: String) -> [Map<String, Any>] {
let frac_result = scan_digits(source, after_dot, total) let frac_result = scan_digits(source, after_dot, total)
let frac_text: String = frac_result["text"] let frac_text: String = frac_result["text"]
let frac_pos: Int = frac_result["pos"] let frac_pos: Int = frac_result["pos"]
let tokens = native_list_append(tokens, make_tok("Float", num_text + "." + frac_text)) let tokens = tok_append(tokens, "Float", num_text + "." + frac_text)
let i = frac_pos let i = frac_pos
} else { } else {
let tokens = native_list_append(tokens, make_tok("Int", num_text)) let tokens = tok_append(tokens, "Int", num_text)
let i = new_pos let i = new_pos
} }
} else { } else {
let tokens = native_list_append(tokens, make_tok("Int", num_text)) let tokens = tok_append(tokens, "Int", num_text)
let i = new_pos let i = new_pos
} }
} else { } else {
let tokens = native_list_append(tokens, make_tok("Int", num_text)) let tokens = tok_append(tokens, "Int", num_text)
let i = new_pos let i = new_pos
} }
} else { } else {
let tokens = native_list_append(tokens, make_tok("Int", num_text)) let tokens = tok_append(tokens, "Int", num_text)
let i = new_pos let i = new_pos
} }
} else { } else {
@@ -811,9 +816,9 @@ fn lex(source: String) -> [Map<String, Any>] {
let new_pos: Int = result["pos"] let new_pos: Int = result["pos"]
let kw = keyword_kind(word) let kw = keyword_kind(word)
if kw == "" { if kw == "" {
let tokens = native_list_append(tokens, make_tok("Ident", word)) let tokens = tok_append(tokens, "Ident", word)
} else { } else {
let tokens = native_list_append(tokens, make_tok(kw, word)) let tokens = tok_append(tokens, kw, word)
} }
let i = new_pos let i = new_pos
} else { } else {
@@ -827,15 +832,15 @@ fn lex(source: String) -> [Map<String, Any>] {
if c == 61 { if c == 61 {
// '=' = 61 // '=' = 61
if peek_c == 61 { if peek_c == 61 {
let tokens = native_list_append(tokens, make_tok("EqEq", "==")) let tokens = tok_append(tokens, "EqEq", "==")
let i = i + 2 let i = i + 2
} else { } else {
if peek_c == 62 { if peek_c == 62 {
// '>' = 62 // '>' = 62
let tokens = native_list_append(tokens, make_tok("FatArrow", "=>")) let tokens = tok_append(tokens, "FatArrow", "=>")
let i = i + 2 let i = i + 2
} else { } else {
let tokens = native_list_append(tokens, make_tok("Eq", "=")) let tokens = tok_append(tokens, "Eq", "=")
let i = i + 1 let i = i + 1
} }
} }
@@ -843,37 +848,37 @@ fn lex(source: String) -> [Map<String, Any>] {
if c == 33 { if c == 33 {
// '!' = 33 // '!' = 33
if peek_c == 61 { if peek_c == 61 {
let tokens = native_list_append(tokens, make_tok("NotEq", "!=")) let tokens = tok_append(tokens, "NotEq", "!=")
let i = i + 2 let i = i + 2
} else { } else {
let tokens = native_list_append(tokens, make_tok("Not", "!")) let tokens = tok_append(tokens, "Not", "!")
let i = i + 1 let i = i + 1
} }
} else { } else {
if c == 60 { if c == 60 {
// '<' = 60 // '<' = 60
if peek_c == 61 { if peek_c == 61 {
let tokens = native_list_append(tokens, make_tok("LtEq", "<=")) let tokens = tok_append(tokens, "LtEq", "<=")
let i = i + 2 let i = i + 2
} else { } else {
let tokens = native_list_append(tokens, make_tok("Lt", "<")) let tokens = tok_append(tokens, "Lt", "<")
let i = i + 1 let i = i + 1
} }
} else { } else {
if c == 62 { if c == 62 {
// '>' = 62 // '>' = 62
if peek_c == 61 { if peek_c == 61 {
let tokens = native_list_append(tokens, make_tok("GtEq", ">=")) let tokens = tok_append(tokens, "GtEq", ">=")
let i = i + 2 let i = i + 2
} else { } else {
let tokens = native_list_append(tokens, make_tok("Gt", ">")) let tokens = tok_append(tokens, "Gt", ">")
let i = i + 1 let i = i + 1
} }
} else { } else {
if c == 38 { if c == 38 {
// '&' = 38 // '&' = 38
if peek_c == 38 { if peek_c == 38 {
let tokens = native_list_append(tokens, make_tok("And", "&&")) let tokens = tok_append(tokens, "And", "&&")
let i = i + 2 let i = i + 2
} else { } else {
let i = i + 1 let i = i + 1
@@ -882,15 +887,15 @@ fn lex(source: String) -> [Map<String, Any>] {
if c == 124 { if c == 124 {
// '|' = 124 // '|' = 124
if peek_c == 124 { if peek_c == 124 {
let tokens = native_list_append(tokens, make_tok("Or", "||")) let tokens = tok_append(tokens, "Or", "||")
let i = i + 2 let i = i + 2
} else { } else {
if peek_c == 62 { if peek_c == 62 {
// '>' = 62 // '>' = 62
let tokens = native_list_append(tokens, make_tok("PipeOp", "|>")) let tokens = tok_append(tokens, "PipeOp", "|>")
let i = i + 2 let i = i + 2
} else { } else {
let tokens = native_list_append(tokens, make_tok("Pipe", "|")) let tokens = tok_append(tokens, "Pipe", "|")
let i = i + 1 let i = i + 1
} }
} }
@@ -899,71 +904,71 @@ fn lex(source: String) -> [Map<String, Any>] {
// '-' = 45 // '-' = 45
if peek_c == 62 { if peek_c == 62 {
// '>' = 62 // '>' = 62
let tokens = native_list_append(tokens, make_tok("Arrow", "->")) let tokens = tok_append(tokens, "Arrow", "->")
let i = i + 2 let i = i + 2
} else { } else {
let tokens = native_list_append(tokens, make_tok("Minus", "-")) let tokens = tok_append(tokens, "Minus", "-")
let i = i + 1 let i = i + 1
} }
} else { } else {
if c == 58 { if c == 58 {
// ':' = 58 // ':' = 58
if peek_c == 58 { if peek_c == 58 {
let tokens = native_list_append(tokens, make_tok("ColonColon", "::")) let tokens = tok_append(tokens, "ColonColon", "::")
let i = i + 2 let i = i + 2
} else { } else {
let tokens = native_list_append(tokens, make_tok("Colon", ":")) let tokens = tok_append(tokens, "Colon", ":")
let i = i + 1 let i = i + 1
} }
} else { } else {
if c == 43 { if c == 43 {
// '+' = 43 // '+' = 43
let tokens = native_list_append(tokens, make_tok("Plus", "+")) let tokens = tok_append(tokens, "Plus", "+")
let i = i + 1 let i = i + 1
} else { } else {
if c == 42 { if c == 42 {
// '*' = 42 // '*' = 42
let tokens = native_list_append(tokens, make_tok("Star", "*")) let tokens = tok_append(tokens, "Star", "*")
let i = i + 1 let i = i + 1
} else { } else {
if c == 37 { if c == 37 {
// '%' = 37 // '%' = 37
let tokens = native_list_append(tokens, make_tok("Percent", "%")) let tokens = tok_append(tokens, "Percent", "%")
let i = i + 1 let i = i + 1
} else { } else {
if c == 40 { if c == 40 {
// '(' = 40 // '(' = 40
let tokens = native_list_append(tokens, make_tok("LParen", "(")) let tokens = tok_append(tokens, "LParen", "(")
let i = i + 1 let i = i + 1
} else { } else {
if c == 41 { if c == 41 {
// ')' = 41 // ')' = 41
let tokens = native_list_append(tokens, make_tok("RParen", ")")) let tokens = tok_append(tokens, "RParen", ")")
let i = i + 1 let i = i + 1
} else { } else {
if c == 123 { if c == 123 {
// '{' = 123 // '{' = 123
let tokens = native_list_append(tokens, make_tok("LBrace", "{")) let tokens = tok_append(tokens, "LBrace", "{")
let i = i + 1 let i = i + 1
} else { } else {
if c == 125 { if c == 125 {
// '}' = 125 // '}' = 125
let tokens = native_list_append(tokens, make_tok("RBrace", "}")) let tokens = tok_append(tokens, "RBrace", "}")
let i = i + 1 let i = i + 1
} else { } else {
if c == 91 { if c == 91 {
// '[' = 91 // '[' = 91
let tokens = native_list_append(tokens, make_tok("LBracket", "[")) let tokens = tok_append(tokens, "LBracket", "[")
let i = i + 1 let i = i + 1
} else { } else {
if c == 93 { if c == 93 {
// ']' = 93 // ']' = 93
let tokens = native_list_append(tokens, make_tok("RBracket", "]")) let tokens = tok_append(tokens, "RBracket", "]")
let i = i + 1 let i = i + 1
} else { } else {
if c == 44 { if c == 44 {
// ',' = 44 // ',' = 44
let tokens = native_list_append(tokens, make_tok("Comma", ",")) let tokens = tok_append(tokens, "Comma", ",")
let i = i + 1 let i = i + 1
} else { } else {
if c == 46 { if c == 46 {
@@ -977,30 +982,30 @@ fn lex(source: String) -> [Map<String, Any>] {
// '..' prefix // '..' prefix
if peek2_c == 61 { if peek2_c == 61 {
// '..=' = 46 46 61 // '..=' = 46 46 61
let tokens = native_list_append(tokens, make_tok("DotDotEq", "..=")) let tokens = tok_append(tokens, "DotDotEq", "..=")
let i = i + 3 let i = i + 3
} else { } else {
let tokens = native_list_append(tokens, make_tok("DotDot", "..")) let tokens = tok_append(tokens, "DotDot", "..")
let i = i + 2 let i = i + 2
} }
} else { } else {
let tokens = native_list_append(tokens, make_tok("Dot", ".")) let tokens = tok_append(tokens, "Dot", ".")
let i = i + 1 let i = i + 1
} }
} else { } else {
if c == 59 { if c == 59 {
// ';' = 59 // ';' = 59
let tokens = native_list_append(tokens, make_tok("Semicolon", ";")) let tokens = tok_append(tokens, "Semicolon", ";")
let i = i + 1 let i = i + 1
} else { } else {
if c == 64 { if c == 64 {
// '@' = 64 // '@' = 64
let tokens = native_list_append(tokens, make_tok("At", "@")) let tokens = tok_append(tokens, "At", "@")
let i = i + 1 let i = i + 1
} else { } else {
if c == 63 { if c == 63 {
// '?' = 63 // '?' = 63
let tokens = native_list_append(tokens, make_tok("QuestionMark", "?")) let tokens = tok_append(tokens, "QuestionMark", "?")
let i = i + 1 let i = i + 1
} else { } else {
// unknown char - skip // unknown char - skip
@@ -1034,6 +1039,6 @@ fn lex(source: String) -> [Map<String, Any>] {
} }
} }
let tokens = native_list_append(tokens, make_tok("Eof", "")) let tokens = tok_append(tokens, "Eof", "")
tokens tokens
} }
lang/el-compiler/src/parser.el
+34 -30
View File
@@ -9,25 +9,28 @@
// The token list is passed as a parameter to all parse functions. // The token list is passed as a parameter to all parse functions.
// native_list_get is used to index into it without cloning. // native_list_get is used to index into it without cloning.
// //
// Entry point: fn parse(tokens: [Map<String, Any>]) -> [Map<String, Any>] // Entry point: fn parse(tokens: [Any]) -> [Map<String, Any>]
// -- Token access helpers ------------------------------------------------------ // -- Token access helpers ------------------------------------------------------
// Tokens is a flat [Any] list: tokens[2*i] = kind, tokens[2*i+1] = value.
// This avoids one ElMap allocation per token (~112B each), saving ~4MB on large
// programs. All callers use these helpers -- only these three need updating.
fn tok_at(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn tok_at(tokens: [Any], pos: Int) -> Map<String, Any> {
native_list_get(tokens, pos) let kind: String = native_list_get(tokens, pos * 2)
let value: String = native_list_get(tokens, pos * 2 + 1)
{ "kind": kind, "value": value }
} }
fn tok_kind(tokens: [Map<String, Any>], pos: Int) -> String { fn tok_kind(tokens: [Any], pos: Int) -> String {
let t = native_list_get(tokens, pos) native_list_get(tokens, pos * 2)
t["kind"]
} }
fn tok_value(tokens: [Map<String, Any>], pos: Int) -> String { fn tok_value(tokens: [Any], pos: Int) -> String {
let t = native_list_get(tokens, pos) native_list_get(tokens, pos * 2 + 1)
t["value"]
} }
fn expect(tokens: [Map<String, Any>], pos: Int, kind: String) -> Int { fn expect(tokens: [Any], pos: Int, kind: String) -> Int {
let k = tok_kind(tokens, pos) let k = tok_kind(tokens, pos)
if k == kind { if k == kind {
return pos + 1 return pos + 1
@@ -46,7 +49,7 @@ fn make_result(node: Map<String, Any>, pos: Int) -> Map<String, Any> {
// Skips over a type annotation, returning the new position. // Skips over a type annotation, returning the new position.
// Types can be: Ident, [Type], Map<K,V>, Type?, Type<Type,...> // Types can be: Ident, [Type], Map<K,V>, Type?, Type<Type,...>
fn skip_type(tokens: [Map<String, Any>], pos: Int) -> Int { fn skip_type(tokens: [Any], pos: Int) -> Int {
let k = tok_kind(tokens, pos) let k = tok_kind(tokens, pos)
// Array type: [Type] // Array type: [Type]
if k == "LBracket" { if k == "LBracket" {
@@ -103,7 +106,7 @@ fn skip_type(tokens: [Map<String, Any>], pos: Int) -> Int {
// -- Parameter list ------------------------------------------------------------ // -- Parameter list ------------------------------------------------------------
// Parses (name: Type, name: Type, ...) - returns { "params": [...], "pos": ... } // Parses (name: Type, name: Type, ...) - returns { "params": [...], "pos": ... }
fn parse_params(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_params(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "LParen") let p = expect(tokens, pos, "LParen")
let params: [Map<String, Any>] = native_list_empty() let params: [Map<String, Any>] = native_list_empty()
let running = true let running = true
@@ -292,7 +295,7 @@ fn is_void_element(name: String) -> Bool {
// Collect tokens as text content until we hit Lt, LBrace, Eof, or a // Collect tokens as text content until we hit Lt, LBrace, Eof, or a
// closing-tag marker (Lt Slash). Returns { "text": "...", "pos": p } // closing-tag marker (Lt Slash). Returns { "text": "...", "pos": p }
fn parse_html_text_tokens(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_html_text_tokens(tokens: [Any], pos: Int) -> Map<String, Any> {
let parts: [String] = native_list_empty() let parts: [String] = native_list_empty()
let p = pos let p = pos
let running = true let running = true
@@ -322,7 +325,7 @@ fn parse_html_text_tokens(tokens: [Map<String, Any>], pos: Int) -> Map<String, A
// Parse an attribute list: (attrname | attrname="val" | attrname={expr})* // Parse an attribute list: (attrname | attrname="val" | attrname={expr})*
// Stops at Gt or Slash (for self-closing />). // Stops at Gt or Slash (for self-closing />).
fn parse_html_attrs(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_html_attrs(tokens: [Any], pos: Int) -> Map<String, Any> {
let attrs: [Map<String, Any>] = native_list_empty() let attrs: [Map<String, Any>] = native_list_empty()
let p = pos let p = pos
let running = true let running = true
@@ -374,7 +377,7 @@ fn parse_html_attrs(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
// Parse the children of an HTML element until we see the closing tag </tag> // Parse the children of an HTML element until we see the closing tag </tag>
// or EOF. Returns { "children": [...], "pos": p_after_closing_tag } // or EOF. Returns { "children": [...], "pos": p_after_closing_tag }
fn parse_html_children(tokens: [Map<String, Any>], pos: Int, parent_tag: String) -> Map<String, Any> { fn parse_html_children(tokens: [Any], pos: Int, parent_tag: String) -> Map<String, Any> {
let children: [Map<String, Any>] = native_list_empty() let children: [Map<String, Any>] = native_list_empty()
let p = pos let p = pos
let running = true let running = true
@@ -514,14 +517,14 @@ fn parse_html_children(tokens: [Map<String, Any>], pos: Int, parent_tag: String)
// Parse body of {#each} until {/each}. Mirrors parse_html_children but // Parse body of {#each} until {/each}. Mirrors parse_html_children but
// stops at the {/each} sentinel rather than a closing element tag. // stops at the {/each} sentinel rather than a closing element tag.
fn parse_html_each_body(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_html_each_body(tokens: [Any], pos: Int) -> Map<String, Any> {
parse_html_children(tokens, pos, "__each__") parse_html_children(tokens, pos, "__each__")
} }
// Parse a single HTML element: <tag attrs> children </tag> // Parse a single HTML element: <tag attrs> children </tag>
// or self-closing: <tag attrs/> // or self-closing: <tag attrs/>
// Pos points to the Lt token. // Pos points to the Lt token.
fn parse_html_element(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_html_element(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = pos let p = pos
// consume < // consume <
let p = expect(tokens, p, "Lt") let p = expect(tokens, p, "Lt")
@@ -558,7 +561,7 @@ fn parse_html_element(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any>
// Entry point for HTML template parsing. // Entry point for HTML template parsing.
// Pos points to Lt (or Lt Not for <!doctype>). // Pos points to Lt (or Lt Not for <!doctype>).
// May parse an optional <!doctype html> prefix followed by the root element. // May parse an optional <!doctype html> prefix followed by the root element.
fn parse_html_template(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_html_template(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = pos let p = pos
// Check for <!doctype html> // Check for <!doctype html>
let doctype = false let doctype = false
@@ -596,7 +599,7 @@ fn parse_html_template(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any>
make_result({ "expr": "HtmlTemplate", "root": root_with_doctype }, p) make_result({ "expr": "HtmlTemplate", "root": root_with_doctype }, p)
} }
fn parse_primary(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_primary(tokens: [Any], pos: Int) -> Map<String, Any> {
let k = tok_kind(tokens, pos) let k = tok_kind(tokens, pos)
let v = tok_value(tokens, pos) let v = tok_value(tokens, pos)
@@ -819,7 +822,7 @@ fn parse_primary(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "expr": "Nil" }, pos + 1) make_result({ "expr": "Nil" }, pos + 1)
} }
fn parse_if(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_if(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "If") let p = expect(tokens, pos, "If")
// Suppress Map-literal parsing in the cond so a stray `{` (the start // Suppress Map-literal parsing in the cond so a stray `{` (the start
// of the then-block) isn't gobbled as a Map. // of the then-block) isn't gobbled as a Map.
@@ -855,7 +858,7 @@ fn parse_if(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "expr": "If", "cond": cond, "then": then_stmts, "else": else_stmts, "has_else": has_else }, p) make_result({ "expr": "If", "cond": cond, "then": then_stmts, "else": else_stmts, "has_else": has_else }, p)
} }
fn parse_match(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_match(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "Match") let p = expect(tokens, pos, "Match")
let prev_no_block: String = state_get("__no_block_expr") let prev_no_block: String = state_get("__no_block_expr")
state_set("__no_block_expr", "1") state_set("__no_block_expr", "1")
@@ -895,7 +898,7 @@ fn parse_match(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "expr": "Match", "subject": subject, "arms": arms }, p) make_result({ "expr": "Match", "subject": subject, "arms": arms }, p)
} }
fn parse_pattern(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_pattern(tokens: [Any], pos: Int) -> Map<String, Any> {
let k = tok_kind(tokens, pos) let k = tok_kind(tokens, pos)
if k == "Ident" { if k == "Ident" {
let v = tok_value(tokens, pos) let v = tok_value(tokens, pos)
@@ -924,7 +927,7 @@ fn parse_pattern(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "pattern": "Wildcard" }, pos + 1) make_result({ "pattern": "Wildcard" }, pos + 1)
} }
fn parse_for_expr(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_for_expr(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "For") let p = expect(tokens, pos, "For")
let item_name = tok_value(tokens, p) let item_name = tok_value(tokens, p)
let p = p + 1 let p = p + 1
@@ -941,7 +944,7 @@ fn parse_for_expr(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "expr": "For", "item": item_name, "list": list_expr, "body": body }, p) make_result({ "expr": "For", "item": item_name, "list": list_expr, "body": body }, p)
} }
fn parse_block(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_block(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "LBrace") let p = expect(tokens, pos, "LBrace")
let stmts: [Map<String, Any>] = native_list_empty() let stmts: [Map<String, Any>] = native_list_empty()
let running = true let running = true
@@ -998,7 +1001,7 @@ fn is_duration_unit(name: String) -> Bool {
false false
} }
fn parse_postfix(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_postfix(tokens: [Any], pos: Int) -> Map<String, Any> {
let r = parse_primary(tokens, pos) let r = parse_primary(tokens, pos)
let node = r["node"] let node = r["node"]
let p = r["pos"] let p = r["pos"]
@@ -1115,7 +1118,7 @@ fn is_binop(kind: String) -> Bool {
false false
} }
fn parse_binop(tokens: [Map<String, Any>], pos: Int, min_prec: Int) -> Map<String, Any> { fn parse_binop(tokens: [Any], pos: Int, min_prec: Int) -> Map<String, Any> {
let r = parse_postfix(tokens, pos) let r = parse_postfix(tokens, pos)
let left = r["node"] let left = r["node"]
let p = r["pos"] let p = r["pos"]
@@ -1140,13 +1143,13 @@ fn parse_binop(tokens: [Map<String, Any>], pos: Int, min_prec: Int) -> Map<Strin
make_result(left, p) make_result(left, p)
} }
fn parse_expr(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_expr(tokens: [Any], pos: Int) -> Map<String, Any> {
parse_binop(tokens, pos, 1) parse_binop(tokens, pos, 1)
} }
// -- Statement parsing --------------------------------------------------------- // -- Statement parsing ---------------------------------------------------------
fn parse_stmt(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_stmt(tokens: [Any], pos: Int) -> Map<String, Any> {
let k = tok_kind(tokens, pos) let k = tok_kind(tokens, pos)
// let binding // let binding
@@ -1619,8 +1622,9 @@ fn parse_stmt(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
// -- Top-level parse ------------------------------------------------------------ // -- Top-level parse ------------------------------------------------------------
fn parse(tokens: [Map<String, Any>]) -> [Map<String, Any>] { fn parse(tokens: [Any]) -> [Map<String, Any>] {
let total: Int = native_list_len(tokens) // Flat list: 2 entries per token, so divide by 2 for token count.
let total: Int = native_list_len(tokens) / 2
let stmts: [Map<String, Any>] = native_list_empty() let stmts: [Map<String, Any>] = native_list_empty()
let pos: Int = 0 let pos: Int = 0
let running = true let running = true