neuron-technologies/el
3
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

merge round-4-delta: flat stride-2 token list + str_char_code dispatch + batch c_escape

Browse Source 
- Flat token list: lexer emits [kind0, val0, kind1, val1, ...] instead of [{kind,val}, ...]
  Eliminates per-token ElMap allocation (~112B × N tokens)
- str_char_code hot loop: char classification via Int codes, no strdup per char
- Batch c_escape: str_slice clean runs instead of char-at per byte
- Parser updated to use tok_at/tok_kind/tok_value stride-2 accessors
This commit is contained in:
Will Anderson
2026-05-05 20:29:35 -05:00
parent eb52be4ade e587bedf30
commit ee86736eab
4 changed files with 403 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lang/el-compiler/src/codegen.el
+48 -12
View File
@@ -38,10 +38,13 @@ fn is_hex_digit_byte(b: Int) -> Bool {
} }
fn c_escape(s: String) -> String { fn c_escape(s: String) -> String {
// Use index-based byte scanning via str_char_code(s, i) and str_char_at(s, i). // Batch ASCII chars using str_slice instead of str_char_at per byte.
// This avoids native_string_chars + str_join, which corrupts high-byte (>= 0x80) // Track clean_start: the beginning of the current run of bytes that need
// characters because list_join's looks_like_string heuristic rejects strings // no escaping. On each special byte, flush the accumulated clean run via
// whose first byte is >= 0x7F and emits them as decimal pointer values instead. // str_slice, then append the escape. This reduces parts-list appends from
// O(N) to O(K) where K = number of special bytes << N for normal strings.
//
// Special bytes: '"'=34, '\\'=92, '\n'=10, '\r'=13, '\t'=9, any byte>=128.
// //
// IMPORTANT: after a \xNN hex escape, if the next byte is a hex digit // IMPORTANT: after a \xNN hex escape, if the next byte is a hex digit
// (0-9, a-f, A-F), we emit `""` to split the C string literal so the C // (0-9, a-f, A-F), we emit `""` to split the C string literal so the C
@@ -51,46 +54,75 @@ fn c_escape(s: String) -> String {
let total: Int = str_len(s) let total: Int = str_len(s)
let parts: [String] = native_list_empty() let parts: [String] = native_list_empty()
let i: Int = 0 let i: Int = 0
let clean_start: Int = 0
let prev_was_hex_escape: Bool = false let prev_was_hex_escape: Bool = false
while i < total { while i < total {
let bval: Int = str_char_code(s, i) let bval: Int = str_char_code(s, i)
// If the previous token was a \xNN escape and the current byte is a // Handle the hex-escape split case first: if prev was \xNN and this
// hex digit, insert an empty string literal ("") to break the escape. // byte is a hex digit, we must flush the clean run and insert "".
// (At this point clean_start == i since the previous special byte
// already reset it, so flush is a no-op unless something is pending.)
if prev_was_hex_escape { if prev_was_hex_escape {
if is_hex_digit_byte(bval) { if is_hex_digit_byte(bval) {
// Flush any accumulated clean bytes before the split marker.
if clean_start < i {
let parts = native_list_append(parts, str_slice(s, clean_start, i))
}
let parts = native_list_append(parts, "\"\"") let parts = native_list_append(parts, "\"\"")
let clean_start = i
} }
} }
let prev_was_hex_escape = false let prev_was_hex_escape = false
if bval == 34 { if bval == 34 {
// 34 = '"' // 34 = '"' — flush clean run, then escape
if clean_start < i {
let parts = native_list_append(parts, str_slice(s, clean_start, i))
}
let parts = native_list_append(parts, "\\\"") let parts = native_list_append(parts, "\\\"")
let clean_start = i + 1
} else { } else {
if bval == 92 { if bval == 92 {
// 92 = '\\' // 92 = '\\'
if clean_start < i {
let parts = native_list_append(parts, str_slice(s, clean_start, i))
}
let parts = native_list_append(parts, "\\\\") let parts = native_list_append(parts, "\\\\")
let clean_start = i + 1
} else { } else {
if bval == 10 { if bval == 10 {
// 10 = '\n' // 10 = '\n'
if clean_start < i {
let parts = native_list_append(parts, str_slice(s, clean_start, i))
}
let parts = native_list_append(parts, "\\n") let parts = native_list_append(parts, "\\n")
let clean_start = i + 1
} else { } else {
if bval == 13 { if bval == 13 {
// 13 = '\r' // 13 = '\r'
if clean_start < i {
let parts = native_list_append(parts, str_slice(s, clean_start, i))
}
let parts = native_list_append(parts, "\\r") let parts = native_list_append(parts, "\\r")
let clean_start = i + 1
} else { } else {
if bval == 9 { if bval == 9 {
// 9 = '\t' // 9 = '\t'
if clean_start < i {
let parts = native_list_append(parts, str_slice(s, clean_start, i))
}
let parts = native_list_append(parts, "\\t") let parts = native_list_append(parts, "\\t")
let clean_start = i + 1
} else { } else {
if bval >= 128 { if bval >= 128 {
// Escape non-ASCII bytes (>= 0x80) as \xNN so // Non-ASCII: flush, then \xNN
// Clang does not misinterpret multi-byte UTF-8 if clean_start < i {
// sequences in C string literals. let parts = native_list_append(parts, str_slice(s, clean_start, i))
}
let parts = native_list_append(parts, "\\x" + byte_to_hex2(bval)) let parts = native_list_append(parts, "\\x" + byte_to_hex2(bval))
let prev_was_hex_escape = true let prev_was_hex_escape = true
} else { let clean_start = i + 1
let parts = native_list_append(parts, str_char_at(s, i))
} }
// else: plain ASCII — extends the current clean run (no append)
} }
} }
} }
@@ -98,6 +130,10 @@ fn c_escape(s: String) -> String {
} }
let i = i + 1 let i = i + 1
} }
// Flush the final clean run if any
if clean_start < total {
let parts = native_list_append(parts, str_slice(s, clean_start, total))
}
str_join(parts, "") str_join(parts, "")
} }
lang/el-compiler/src/compiler.el
+4 -4
View File
@@ -21,7 +21,7 @@ import "codegen-js.el"
// compile — full pipeline (C target): source string -> C source string // compile — full pipeline (C target): source string -> C source string
fn compile(source: String) -> String { fn compile(source: String) -> String {
let tokens: [Map<String, Any>] = lex(source) let tokens: [Any] = lex(source)
let stmts: [Map<String, Any>] = parse(tokens) let stmts: [Map<String, Any>] = parse(tokens)
// Token list is no longer needed after parsing — release it to free memory // Token list is no longer needed after parsing — release it to free memory
// before codegen allocates its own working data on large source files. // before codegen allocates its own working data on large source files.
@@ -31,7 +31,7 @@ fn compile(source: String) -> String {
// compile_js — full pipeline (JS target, module mode): source string -> JS source string // compile_js — full pipeline (JS target, module mode): source string -> JS source string
fn compile_js(source: String) -> String { fn compile_js(source: String) -> String {
let tokens: [Map<String, Any>] = lex(source) let tokens: [Any] = lex(source)
let stmts: [Map<String, Any>] = parse(tokens) let stmts: [Map<String, Any>] = parse(tokens)
// Token list is no longer needed after parsing — release it to free memory. // Token list is no longer needed after parsing — release it to free memory.
el_release(tokens) el_release(tokens)
@@ -41,7 +41,7 @@ fn compile_js(source: String) -> String {
// compile_js_with_bundle — JS target in bundle mode. // compile_js_with_bundle — JS target in bundle mode.
// Reads el_runtime.js from runtime_path and inlines it inside an IIFE. // Reads el_runtime.js from runtime_path and inlines it inside an IIFE.
fn compile_js_with_bundle(source: String, runtime_path: String) -> String { fn compile_js_with_bundle(source: String, runtime_path: String) -> String {
let tokens: [Map<String, Any>] = lex(source) let tokens: [Any] = lex(source)
let stmts: [Map<String, Any>] = parse(tokens) let stmts: [Map<String, Any>] = parse(tokens)
el_release(tokens) el_release(tokens)
let runtime_content: String = fs_read(runtime_path) let runtime_content: String = fs_read(runtime_path)
@@ -501,7 +501,7 @@ fn main() -> Void {
// (without inlining imports) and write out a .elh file alongside the .c. // (without inlining imports) and write out a .elh file alongside the .c.
if do_emit_header { if do_emit_header {
let raw_source: String = fs_read(src_path) let raw_source: String = fs_read(src_path)
let hdr_tokens: [Map<String, Any>] = lex(raw_source) let hdr_tokens: [Any] = lex(raw_source)
let hdr_stmts: [Map<String, Any>] = parse(hdr_tokens) let hdr_stmts: [Map<String, Any>] = parse(hdr_tokens)
el_release(hdr_tokens) el_release(hdr_tokens)
let hdr_path: String = str_slice(src_path, 0, str_len(src_path) - 3) + ".elh" let hdr_path: String = str_slice(src_path, 0, str_len(src_path) - 3) + ".elh"
lang/el-compiler/src/lexer.el
+319 -233
View File
@@ -7,11 +7,50 @@
// //
// Entry point: fn lex(source: String) -> [Map<String, Any>] // Entry point: fn lex(source: String) -> [Map<String, Any>]
// //
// Uses native_string_chars to split the source into a chars list, // Performance: the hot lexer loop uses str_char_code (returns Int) instead of
// then indexes it with native_list_get - avoids O(N-) string cloning. // str_char_at (returns strdup'd String) for character classification.
// For a 400KB source, str_char_at allocates ~400K × 16B = ~6.4MB of temporary
// strings for the `ch` variable alone. str_char_code avoids all that.
// -- Character helpers --------------------------------------------------------- // -- Character helpers (Int-based, no string allocation) ----------------------
// These operate on char codes (from str_char_code) instead of str_char_at,
// eliminating one strdup per character in the hot lexer loop.
fn is_digit_code(c: Int) -> Bool {
// '0'=48 .. '9'=57
if c >= 48 {
if c <= 57 { return true }
}
false
}
fn is_alpha_code(c: Int) -> Bool {
// 'A'=65..'Z'=90, 'a'=97..'z'=122
if c >= 65 {
if c <= 90 { return true }
}
if c >= 97 {
if c <= 122 { return true }
}
false
}
fn is_alnum_or_underscore_code(c: Int) -> Bool {
if is_digit_code(c) { return true }
if is_alpha_code(c) { return true }
if c == 95 { return true } // '_'
false
}
fn is_ws_code(c: Int) -> Bool {
if c == 32 { return true } // ' '
if c == 9 { return true } // '\t'
if c == 10 { return true } // '\n'
if c == 13 { return true } // '\r'
false
}
// Legacy String-based helpers kept for scan_interp helpers that use str_char_at.
fn lex_is_digit(ch: String) -> Bool { fn lex_is_digit(ch: String) -> Bool {
if ch == "0" { return true } if ch == "0" { return true }
if ch == "1" { return true } if ch == "1" { return true }
@@ -97,8 +136,11 @@ fn lex_is_whitespace(ch: String) -> Bool {
false false
} }
fn make_tok(kind: String, value: String) -> Map<String, Any> { // tok_append — append a (kind, value) pair to a flat token list.
{ "kind": kind, "value": value } // Returns the updated list. Gamma combines flat-list + char-code for max savings.
fn tok_append(tokens: [Any], kind: String, value: String) -> [Any] {
let tokens = native_list_append(tokens, kind)
native_list_append(tokens, value)
} }
// -- Keyword lookup ------------------------------------------------------------ // -- Keyword lookup ------------------------------------------------------------
@@ -157,45 +199,43 @@ fn keyword_kind(word: String) -> String {
// scan_digits - advance i while chars[i] is a digit // scan_digits - advance i while chars[i] is a digit
// Returns { "text": ..., "pos": i } // Returns { "text": ..., "pos": i }
fn scan_digits(chars: [String], start: Int, total: Int) -> Map<String, Any> { fn scan_digits(src: String, start: Int, total: Int) -> Map<String, Any> {
let i = start let i = start
let parts: [String] = native_list_empty()
let running = true let running = true
while running { while running {
if i >= total { if i >= total {
let running = false let running = false
} else { } else {
let ch: String = native_list_get(chars, i) let c: Int = str_char_code(src, i)
if lex_is_digit(ch) { if is_digit_code(c) {
let parts = native_list_append(parts, ch)
let i = i + 1 let i = i + 1
} else { } else {
let running = false let running = false
} }
} }
} }
{ "text": str_join(parts, ""), "pos": i } // Use str_slice instead of building a parts list — O(1) allocation, O(n) copy.
{ "text": str_slice(src, start, i), "pos": i }
} }
// scan_ident - advance i while chars[i] is alphanumeric or underscore // scan_ident - advance i while chars[i] is alphanumeric or underscore
fn scan_ident(chars: [String], start: Int, total: Int) -> Map<String, Any> { fn scan_ident(src: String, start: Int, total: Int) -> Map<String, Any> {
let i = start let i = start
let parts: [String] = native_list_empty()
let running = true let running = true
while running { while running {
if i >= total { if i >= total {
let running = false let running = false
} else { } else {
let ch: String = native_list_get(chars, i) let c: Int = str_char_code(src, i)
if is_alnum_or_underscore(ch) { if is_alnum_or_underscore_code(c) {
let parts = native_list_append(parts, ch)
let i = i + 1 let i = i + 1
} else { } else {
let running = false let running = false
} }
} }
} }
{ "text": str_join(parts, ""), "pos": i } // Use str_slice instead of building a parts list — O(1) allocation, O(n) copy.
{ "text": str_slice(src, start, i), "pos": i }
} }
// -- Code-bearing string detection + comment strip ---------------------------- // -- Code-bearing string detection + comment strip ----------------------------
@@ -208,34 +248,16 @@ fn scan_ident(chars: [String], start: Int, total: Int) -> Map<String, Any> {
// looks_like_code - heuristic gate so we only strip strings that actually // looks_like_code - heuristic gate so we only strip strings that actually
// embed JS or CSS. Plain prose, hex blobs, JSON, etc. pass through verbatim. // embed JS or CSS. Plain prose, hex blobs, JSON, etc. pass through verbatim.
fn substr_at(chars: [String], start: Int, total: Int, needle: String) -> Bool { fn substr_at(src: String, start: Int, total: Int, needle: String) -> Bool {
let nchars: [String] = native_string_chars(needle) let nlen: Int = str_len(needle)
let nlen: Int = native_list_len(nchars)
if start + nlen > total { return false } if start + nlen > total { return false }
let i = 0 // Use str_slice comparison instead of char-by-char loop.
let matched = true str_eq(str_slice(src, start, start + nlen), needle)
while i < nlen {
let a: String = native_list_get(chars, start + i)
let b: String = native_list_get(nchars, i)
if a == b { let i = i + 1 } else { let matched = false; let i = nlen }
}
matched
} }
fn str_has(s: String, needle: String) -> Bool { fn str_has(s: String, needle: String) -> Bool {
let chars: [String] = native_string_chars(s) // Use the built-in str_contains which is implemented in native C — O(n) single pass.
let total: Int = native_list_len(chars) str_contains(s, needle)
let i = 0
let found = false
while i < total {
if substr_at(chars, i, total, needle) {
let found = true
let i = total
} else {
let i = i + 1
}
}
found
} }
fn looks_like_code(s: String) -> Bool { fn looks_like_code(s: String) -> Bool {
@@ -254,8 +276,7 @@ fn looks_like_code(s: String) -> Bool {
// comment opener: if the char immediately before '/' is ':', emit the '/' // comment opener: if the char immediately before '/' is ':', emit the '/'
// literally and advance one position. // literally and advance one position.
fn strip_code_comments(s: String) -> String { fn strip_code_comments(s: String) -> String {
let chars: [String] = native_string_chars(s) let total: Int = str_len(s)
let total: Int = native_list_len(chars)
let out_parts: [String] = native_list_empty() let out_parts: [String] = native_list_empty()
let i = 0 let i = 0
let in_squote = false let in_squote = false
@@ -263,7 +284,7 @@ fn strip_code_comments(s: String) -> String {
let in_btick = false let in_btick = false
let prev = "" let prev = ""
while i < total { while i < total {
let ch: String = native_list_get(chars, i) let ch: String = str_char_at(s, i)
let in_js_string = false let in_js_string = false
if in_squote { let in_js_string = true } if in_squote { let in_js_string = true }
if in_dquote { let in_js_string = true } if in_dquote { let in_js_string = true }
@@ -275,7 +296,7 @@ fn strip_code_comments(s: String) -> String {
let out_parts = native_list_append(out_parts, ch) let out_parts = native_list_append(out_parts, ch)
let next_i = i + 1 let next_i = i + 1
if next_i < total { if next_i < total {
let nc: String = native_list_get(chars, next_i) let nc: String = str_char_at(s, next_i)
let out_parts = native_list_append(out_parts, nc) let out_parts = native_list_append(out_parts, nc)
let prev = nc let prev = nc
let i = next_i + 1 let i = next_i + 1
@@ -304,7 +325,7 @@ fn strip_code_comments(s: String) -> String {
let next_i = i + 1 let next_i = i + 1
let next_ch = "" let next_ch = ""
if next_i < total { if next_i < total {
let next_ch: String = native_list_get(chars, next_i) let next_ch: String = str_char_at(s, next_i)
} }
if ch == "/" { if ch == "/" {
@@ -323,7 +344,7 @@ fn strip_code_comments(s: String) -> String {
if i >= total { if i >= total {
let scanning = false let scanning = false
} else { } else {
let lc: String = native_list_get(chars, i) let lc: String = str_char_at(s, i)
if lc == "\n" { if lc == "\n" {
let scanning = false let scanning = false
} else { } else {
@@ -342,11 +363,11 @@ fn strip_code_comments(s: String) -> String {
if i >= total { if i >= total {
let scanning2 = false let scanning2 = false
} else { } else {
let bc: String = native_list_get(chars, i) let bc: String = str_char_at(s, i)
if bc == "*" { if bc == "*" {
let after = i + 1 let after = i + 1
if after < total { if after < total {
let nc2: String = native_list_get(chars, after) let nc2: String = str_char_at(s, after)
if nc2 == "/" { if nc2 == "/" {
let i = after + 1 let i = after + 1
let scanning2 = false let scanning2 = false
@@ -402,7 +423,7 @@ fn strip_code_comments(s: String) -> String {
// scan_string - scan a quoted string literal, handling \" escapes. // scan_string - scan a quoted string literal, handling \" escapes.
// Starts AFTER the opening quote. Returns { "text": content, "pos": i_after_close } // Starts AFTER the opening quote. Returns { "text": content, "pos": i_after_close }
fn scan_string(chars: [String], start: Int, total: Int) -> Map<String, Any> { fn scan_string(src: String, start: Int, total: Int) -> Map<String, Any> {
let i = start let i = start
let parts: [String] = native_list_empty() let parts: [String] = native_list_empty()
let running = true let running = true
@@ -410,12 +431,12 @@ fn scan_string(chars: [String], start: Int, total: Int) -> Map<String, Any> {
if i >= total { if i >= total {
let running = false let running = false
} else { } else {
let ch: String = native_list_get(chars, i) let ch: String = str_char_at(src, i)
if ch == "\\" { if ch == "\\" {
// escape: peek next char // escape: peek next char
let next_i = i + 1 let next_i = i + 1
if next_i < total { if next_i < total {
let next_ch: String = native_list_get(chars, next_i) let next_ch: String = str_char_at(src, next_i)
if next_ch == "\"" { if next_ch == "\"" {
let parts = native_list_append(parts, "\"") let parts = native_list_append(parts, "\"")
let i = next_i + 1 let i = next_i + 1
@@ -465,19 +486,17 @@ fn scan_string(chars: [String], start: Int, total: Int) -> Map<String, Any> {
// scan_interp_brace - scan from `start` (the char after `${`) to the matching // scan_interp_brace - scan from `start` (the char after `${`) to the matching
// `}`, tracking brace depth so inner braces (e.g. fn calls, map literals) are // `}`, tracking brace depth so inner braces (e.g. fn calls, map literals) are
// handled correctly. Returns { "text": inner_source, "pos": i_after_close }. // handled correctly. Returns { "text": inner_source, "pos": i_after_close }.
fn scan_interp_brace(chars: [String], start: Int, total: Int) -> Map<String, Any> { fn scan_interp_brace(src: String, start: Int, total: Int) -> Map<String, Any> {
let i = start let i = start
let parts: [String] = native_list_empty()
let depth = 1 let depth = 1
let running = true let running = true
while running { while running {
if i >= total { if i >= total {
let running = false let running = false
} else { } else {
let ch: String = native_list_get(chars, i) let ch: String = str_char_at(src, i)
if ch == "{" { if ch == "{" {
let depth = depth + 1 let depth = depth + 1
let parts = native_list_append(parts, ch)
let i = i + 1 let i = i + 1
} else { } else {
if ch == "}" { if ch == "}" {
@@ -487,33 +506,33 @@ fn scan_interp_brace(chars: [String], start: Int, total: Int) -> Map<String, Any
let i = i + 1 let i = i + 1
let running = false let running = false
} else { } else {
let parts = native_list_append(parts, ch)
let i = i + 1 let i = i + 1
} }
} else { } else {
let parts = native_list_append(parts, ch)
let i = i + 1 let i = i + 1
} }
} }
} }
} }
{ "text": str_join(parts, ""), "pos": i } // Use str_slice instead of parts list — the inner source is a contiguous substring.
{ "text": str_slice(src, start, i - 1), "pos": i }
} }
// interp_tokens_append_all - copy every token from src into dst, skipping the // interp_tokens_append_all - copy every (kind, value) pair from flat src list
// trailing Eof sentinel that lex() always appends. Returns the updated dst list. // into flat dst list, skipping the trailing Eof pair that lex() always appends.
fn interp_tokens_append_all(dst: [Map<String, Any>], src: [Map<String, Any>]) -> [Map<String, Any>] { fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] {
let src_len: Int = native_list_len(src) let src_len: Int = native_list_len(src)
let j = 0 let j = 0
let result = dst let result = dst
while j < src_len { while j < src_len {
let tok: Map<String, Any> = native_list_get(src, j) let kind: String = native_list_get(src, j)
let tk: String = tok["kind"] if kind == "Eof" {
if tk == "Eof" {
let j = src_len let j = src_len
} else { } else {
let result = native_list_append(result, tok) let val: String = native_list_get(src, j + 1)
let j = j + 1 let result = native_list_append(result, kind)
let result = native_list_append(result, val)
let j = j + 2
} }
} }
result result
@@ -536,10 +555,17 @@ fn interp_tokens_append_all(dst: [Map<String, Any>], src: [Map<String, Any>]) ->
// //
// Supported escape sequences: \" \n \t \r \\ \$ (literal dollar sign). // Supported escape sequences: \" \n \t \r \\ \$ (literal dollar sign).
// Nested quotes inside ${} are not supported; use a variable instead. // Nested quotes inside ${} are not supported; use a variable instead.
fn scan_interp_string(chars: [String], start: Int, total: Int) -> Map<String, Any> { //
// Performance: uses str_char_code (Int) for all character dispatch, eliminating
// per-character strdup. Plain runs are batched into str_slice segments instead
// of accumulating single-char strings, reducing list appends from O(N) to O(K)
// where K = number of escape/special chars in the literal.
// Char codes: '\' = 92, '"' = 34, '$' = 36, '{' = 123
fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
let i = start let i = start
let out_tokens: [Map<String, Any>] = native_list_empty() let out_tokens: [Any] = native_list_empty()
let cur_part: [String] = native_list_empty() let cur_parts: [String] = native_list_empty()
let clean_start = start
let has_interp = false let has_interp = false
let need_plus = false let need_plus = false
let running = true let running = true
@@ -548,39 +574,55 @@ fn scan_interp_string(chars: [String], start: Int, total: Int) -> Map<String, An
if i >= total { if i >= total {
let running = false let running = false
} else { } else {
let ch: String = native_list_get(chars, i) let c: Int = str_char_code(src, i)
if ch == "\\" { if c == 92 {
// Escape sequence // '\\' = 92 — escape sequence: flush clean run, append resolved char
if clean_start < i {
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
}
let next_i = i + 1 let next_i = i + 1
if next_i < total { if next_i < total {
let next_ch: String = native_list_get(chars, next_i) let nc: Int = str_char_code(src, next_i)
if next_ch == "$" { if nc == 36 {
// \$ => literal '$' (escape for interpolation syntax) // '\$' => literal '$' (36 = '$')
let cur_part = native_list_append(cur_part, "$") let cur_parts = native_list_append(cur_parts, "$")
let clean_start = next_i + 1
let i = next_i + 1 let i = next_i + 1
} else { } else {
if next_ch == "\"" { if nc == 34 {
let cur_part = native_list_append(cur_part, "\"") // '\"' => literal '"' (34 = '"')
let cur_parts = native_list_append(cur_parts, "\"")
let clean_start = next_i + 1
let i = next_i + 1 let i = next_i + 1
} else { } else {
if next_ch == "n" { if nc == 110 {
let cur_part = native_list_append(cur_part, "\n") // '\n' (110 = 'n')
let cur_parts = native_list_append(cur_parts, "\n")
let clean_start = next_i + 1
let i = next_i + 1 let i = next_i + 1
} else { } else {
if next_ch == "t" { if nc == 116 {
let cur_part = native_list_append(cur_part, "\t") // '\t' (116 = 't')
let cur_parts = native_list_append(cur_parts, "\t")
let clean_start = next_i + 1
let i = next_i + 1 let i = next_i + 1
} else { } else {
if next_ch == "r" { if nc == 114 {
let cur_part = native_list_append(cur_part, "\r") // '\r' (114 = 'r')
let cur_parts = native_list_append(cur_parts, "\r")
let clean_start = next_i + 1
let i = next_i + 1 let i = next_i + 1
} else { } else {
if next_ch == "\\" { if nc == 92 {
let cur_part = native_list_append(cur_part, "\\") // '\\' (92)
let cur_parts = native_list_append(cur_parts, "\\")
let clean_start = next_i + 1
let i = next_i + 1 let i = next_i + 1
} else { } else {
let cur_part = native_list_append(cur_part, next_ch) // Unknown escape: emit the escaped char verbatim
let cur_parts = native_list_append(cur_parts, str_slice(src, next_i, next_i + 1))
let clean_start = next_i + 1
let i = next_i + 1 let i = next_i + 1
} }
} }
@@ -589,75 +631,85 @@ fn scan_interp_string(chars: [String], start: Int, total: Int) -> Map<String, An
} }
} }
} else { } else {
let clean_start = next_i
let i = next_i
}
} else {
if c == 34 {
// '"' = 34 — closing quote: flush clean run, stop
if clean_start < i {
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
}
let i = i + 1 let i = i + 1
} let clean_start = i
} else {
if ch == "\"" {
// Closing quote - stop scanning
let i = i + 1
let running = false let running = false
} else { } else {
if ch == "$" { if c == 36 {
// Check for ${ (start of interpolation) // '$' = 36 — possible interpolation start
let next_i = i + 1 let next_i = i + 1
let is_interp = false let is_interp = false
if next_i < total { if next_i < total {
let next_ch: String = native_list_get(chars, next_i) let nc2: Int = str_char_code(src, next_i)
if next_ch == "{" { if nc2 == 123 {
// '{' = 123
let is_interp = true let is_interp = true
} }
} }
if is_interp { if is_interp {
// Flush the accumulated literal part (if non-empty) // Flush the accumulated literal part (if non-empty)
let part_len: Int = native_list_len(cur_part) if clean_start < i {
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
}
let part_len: Int = native_list_len(cur_parts)
if part_len > 0 { if part_len > 0 {
let part_text = str_join(cur_part, "") let part_text = str_join(cur_parts, "")
if need_plus { if need_plus {
let out_tokens = native_list_append(out_tokens, make_tok("Plus", "+")) let out_tokens = tok_append(out_tokens, "Plus", "+")
} }
let clean_part = part_text let clean_part = part_text
if looks_like_code(part_text) { if looks_like_code(part_text) {
let clean_part = strip_code_comments(part_text) let clean_part = strip_code_comments(part_text)
} }
let out_tokens = native_list_append(out_tokens, make_tok("Str", clean_part)) let out_tokens = tok_append(out_tokens, "Str", clean_part)
let need_plus = true let need_plus = true
} }
let cur_part = native_list_empty() let cur_parts = native_list_empty()
let has_interp = true let has_interp = true
// Scan brace-balanced expression source // Scan brace-balanced expression source
let brace_result = scan_interp_brace(chars, next_i + 1, total) let brace_result = scan_interp_brace(src, next_i + 1, total)
let expr_src: String = brace_result["text"] let expr_src: String = brace_result["text"]
let new_i: Int = brace_result["pos"] let new_i: Int = brace_result["pos"]
let i = new_i let i = new_i
let clean_start = new_i
// Re-lex the expression and inline the tokens. // Re-lex the expression and inline the tokens.
// Wrap in ( ) so that operators inside ${} (e.g. // Wrap in ( ) so that operators inside ${} (e.g.
// age + 1) are parsed as a grouped sub-expression // age + 1) are parsed as a grouped sub-expression
// rather than merging with the surrounding concat // rather than merging with the surrounding concat
// Plus tokens at the wrong precedence level. // Plus tokens at the wrong precedence level.
let inner_toks: [Map<String, Any>] = lex(expr_src) let inner_toks: [Any] = lex(expr_src)
let inner_len: Int = native_list_len(inner_toks) let inner_len: Int = native_list_len(inner_toks)
if need_plus { if need_plus {
let out_tokens = native_list_append(out_tokens, make_tok("Plus", "+")) let out_tokens = tok_append(out_tokens, "Plus", "+")
} }
// Empty interpolation ${} => empty string segment // Empty interpolation ${} => empty string segment
if inner_len <= 1 { // inner_len <= 2 = only the Eof pair (kind="Eof", value="")
let out_tokens = native_list_append(out_tokens, make_tok("Str", "")) if inner_len <= 2 {
let out_tokens = tok_append(out_tokens, "Str", "")
} else { } else {
let out_tokens = native_list_append(out_tokens, make_tok("LParen", "(")) let out_tokens = tok_append(out_tokens, "LParen", "(")
let out_tokens = interp_tokens_append_all(out_tokens, inner_toks) let out_tokens = interp_tokens_append_all(out_tokens, inner_toks)
let out_tokens = native_list_append(out_tokens, make_tok("RParen", ")")) let out_tokens = tok_append(out_tokens, "RParen", ")")
} }
let need_plus = true let need_plus = true
} else { } else {
// Plain '$' not followed by '{' - treat as literal // Plain '$' not followed by '{' - treat as literal, continue clean run
let cur_part = native_list_append(cur_part, "$")
let i = i + 1 let i = i + 1
} }
} else { } else {
let cur_part = native_list_append(cur_part, ch) // Plain char — extends clean run, no append needed
let i = i + 1 let i = i + 1
} }
} }
@@ -666,8 +718,11 @@ fn scan_interp_string(chars: [String], start: Int, total: Int) -> Map<String, An
} }
// Flush remaining literal segment and build final token list // Flush remaining literal segment and build final token list
let part_text = str_join(cur_part, "") if clean_start < i {
let part_len: Int = native_list_len(cur_part) let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
}
let part_len: Int = native_list_len(cur_parts)
let part_text = str_join(cur_parts, "")
if has_interp { if has_interp {
// Interpolated string: only emit trailing segment if non-empty // Interpolated string: only emit trailing segment if non-empty
if part_len > 0 { if part_len > 0 {
@@ -676,9 +731,9 @@ fn scan_interp_string(chars: [String], start: Int, total: Int) -> Map<String, An
let clean_part = strip_code_comments(part_text) let clean_part = strip_code_comments(part_text)
} }
if need_plus { if need_plus {
let out_tokens = native_list_append(out_tokens, make_tok("Plus", "+")) let out_tokens = tok_append(out_tokens, "Plus", "+")
} }
let out_tokens = native_list_append(out_tokens, make_tok("Str", clean_part)) let out_tokens = tok_append(out_tokens, "Str", clean_part)
} }
} else { } else {
// Plain string with no interpolation - same behaviour as old scan_string // Plain string with no interpolation - same behaviour as old scan_string
@@ -686,42 +741,51 @@ fn scan_interp_string(chars: [String], start: Int, total: Int) -> Map<String, An
if looks_like_code(part_text) { if looks_like_code(part_text) {
let clean_text = strip_code_comments(part_text) let clean_text = strip_code_comments(part_text)
} }
let out_tokens = native_list_append(out_tokens, make_tok("Str", clean_text)) let out_tokens = tok_append(out_tokens, "Str", clean_text)
} }
{ "tokens": out_tokens, "pos": i } { "tokens": out_tokens, "pos": i }
} }
// -- Main lexer ---------------------------------------------------------------- // -- Main lexer ----------------------------------------------------------------
// Char code constants (avoids strdup for single-char comparison)
// '/' = 47, '"' = 34, '0'-'9' = 48-57, 'a'-'z' = 97-122, 'A'-'Z' = 65-90
// '_' = 95, ' '=32, '\t'=9, '\n'=10, '\r'=13
// '=' = 61, '!' = 33, '<' = 60, '>' = 62, '&' = 38, '|' = 124
// '-' = 45, ':' = 58, '+' = 43, '*' = 42, '%' = 37
// '(' = 40, ')' = 41, '{' = 123, '}' = 125, '[' = 91, ']' = 93
// ',' = 44, '.' = 46, ';' = 59, '@' = 64, '?' = 63
fn lex(source: String) -> [Map<String, Any>] { fn lex(source: String) -> [Any] {
let chars: [String] = native_string_chars(source) // Use str_char_code (returns Int) instead of str_char_at (returns strdup String)
let total: Int = native_list_len(chars) // for all character classification in the hot loop. For a 400KB source,
let tokens: [Map<String, Any>] = native_list_empty() // str_char_at allocates ~400K × 16B = ~6.4MB of temporary strings.
let total: Int = str_len(source)
let tokens: [Any] = native_list_empty()
let i: Int = 0 let i: Int = 0
while i < total { while i < total {
let ch: String = native_list_get(chars, i) let c: Int = str_char_code(source, i)
// Skip whitespace // Skip whitespace (space=32, tab=9, newline=10, CR=13)
if lex_is_whitespace(ch) { if is_ws_code(c) {
let i = i + 1 let i = i + 1
} else { } else {
// Line comments: // // Line comments: // (slash=47)
if ch == "/" { if c == 47 {
let next_i = i + 1 let next_i = i + 1
if next_i < total { if next_i < total {
let next_ch: String = native_list_get(chars, next_i) let nc: Int = str_char_code(source, next_i)
if next_ch == "/" { if nc == 47 {
// skip to end of line // skip to end of line (newline=10)
let i = i + 2 let i = i + 2
let running2 = true let running2 = true
while running2 { while running2 {
if i >= total { if i >= total {
let running2 = false let running2 = false
} else { } else {
let lch: String = native_list_get(chars, i) let lc: Int = str_char_code(source, i)
if lch == "\n" { if lc == 10 {
let running2 = false let running2 = false
} else { } else {
let i = i + 1 let i = i + 1
@@ -729,232 +793,254 @@ fn lex(source: String) -> [Map<String, Any>] {
} }
} }
} else { } else {
let tokens = native_list_append(tokens, make_tok("Slash", "/")) let tokens = tok_append(tokens, "Slash", "/")
let i = i + 1 let i = i + 1
} }
} else { } else {
let tokens = native_list_append(tokens, make_tok("Slash", "/")) let tokens = tok_append(tokens, "Slash", "/")
let i = i + 1 let i = i + 1
} }
} else { } else {
// String literal (plain or interpolated with ${expr} syntax). // String literal: '"' = 34
// scan_interp_string handles both cases: plain strings emit a if c == 34 {
// single Str token; interpolated strings emit a flat token let interp_result = scan_interp_string(source, i + 1, total)
// sequence (Str Plus expr-tokens Plus Str ...) that the parser let interp_toks: [Any] = interp_result["tokens"]
// naturally assembles into a BinOp concat tree.
if ch == "\"" {
let interp_result = scan_interp_string(chars, i + 1, total)
let interp_toks: [Map<String, Any>] = interp_result["tokens"]
let new_pos: Int = interp_result["pos"] let new_pos: Int = interp_result["pos"]
let tokens = interp_tokens_append_all(tokens, interp_toks) let tokens = interp_tokens_append_all(tokens, interp_toks)
let i = new_pos let i = new_pos
} else { } else {
// Number literal // Number literal: '0'-'9' = 48-57
if lex_is_digit(ch) { if is_digit_code(c) {
let result = scan_digits(chars, i, total) let result = scan_digits(source, i, total)
let num_text: String = result["text"] let num_text: String = result["text"]
let new_pos: Int = result["pos"] let new_pos: Int = result["pos"]
// check for float (dot followed by digit) // check for float (dot=46 followed by digit)
if new_pos < total { if new_pos < total {
let dot_ch: String = native_list_get(chars, new_pos) let dc: Int = str_char_code(source, new_pos)
if dot_ch == "." { if dc == 46 {
let after_dot = new_pos + 1 let after_dot = new_pos + 1
if after_dot < total { if after_dot < total {
let after_dot_ch: String = native_list_get(chars, after_dot) let adc: Int = str_char_code(source, after_dot)
if lex_is_digit(after_dot_ch) { if is_digit_code(adc) {
let frac_result = scan_digits(chars, after_dot, total) let frac_result = scan_digits(source, after_dot, total)
let frac_text: String = frac_result["text"] let frac_text: String = frac_result["text"]
let frac_pos: Int = frac_result["pos"] let frac_pos: Int = frac_result["pos"]
let tokens = native_list_append(tokens, make_tok("Float", num_text + "." + frac_text)) let tokens = tok_append(tokens, "Float", num_text + "." + frac_text)
let i = frac_pos let i = frac_pos
} else { } else {
let tokens = native_list_append(tokens, make_tok("Int", num_text)) let tokens = tok_append(tokens, "Int", num_text)
let i = new_pos let i = new_pos
} }
} else { } else {
let tokens = native_list_append(tokens, make_tok("Int", num_text)) let tokens = tok_append(tokens, "Int", num_text)
let i = new_pos let i = new_pos
} }
} else { } else {
let tokens = native_list_append(tokens, make_tok("Int", num_text)) let tokens = tok_append(tokens, "Int", num_text)
let i = new_pos let i = new_pos
} }
} else { } else {
let tokens = native_list_append(tokens, make_tok("Int", num_text)) let tokens = tok_append(tokens, "Int", num_text)
let i = new_pos let i = new_pos
} }
} else { } else {
// Identifier or keyword // Identifier or keyword: alpha or '_'=95
if lex_is_alpha(ch) || ch == "_" { if is_alpha_code(c) || c == 95 {
let result = scan_ident(chars, i, total) let result = scan_ident(source, i, total)
let word: String = result["text"] let word: String = result["text"]
let new_pos: Int = result["pos"] let new_pos: Int = result["pos"]
let kw = keyword_kind(word) let kw = keyword_kind(word)
if kw == "" { if kw == "" {
let tokens = native_list_append(tokens, make_tok("Ident", word)) let tokens = tok_append(tokens, "Ident", word)
} else { } else {
let tokens = native_list_append(tokens, make_tok(kw, word)) let tokens = tok_append(tokens, kw, word)
} }
let i = new_pos let i = new_pos
} else { } else {
// Multi-char and single-char operators/delimiters // Multi-char and single-char operators/delimiters
let peek_i = i + 1 let peek_i = i + 1
let peek_ch = "" let peek_c: Int = -1
if peek_i < total { if peek_i < total {
let peek_ch: String = native_list_get(chars, peek_i) let peek_c: Int = str_char_code(source, peek_i)
} }
if ch == "=" { if c == 61 {
if peek_ch == "=" { // '=' = 61
let tokens = native_list_append(tokens, make_tok("EqEq", "==")) if peek_c == 61 {
let tokens = tok_append(tokens, "EqEq", "==")
let i = i + 2 let i = i + 2
} else { } else {
if peek_ch == ">" { if peek_c == 62 {
let tokens = native_list_append(tokens, make_tok("FatArrow", "=>")) // '>' = 62
let tokens = tok_append(tokens, "FatArrow", "=>")
let i = i + 2 let i = i + 2
} else { } else {
let tokens = native_list_append(tokens, make_tok("Eq", "=")) let tokens = tok_append(tokens, "Eq", "=")
let i = i + 1 let i = i + 1
} }
} }
} else { } else {
if ch == "!" { if c == 33 {
if peek_ch == "=" { // '!' = 33
let tokens = native_list_append(tokens, make_tok("NotEq", "!=")) if peek_c == 61 {
let tokens = tok_append(tokens, "NotEq", "!=")
let i = i + 2 let i = i + 2
} else { } else {
let tokens = native_list_append(tokens, make_tok("Not", "!")) let tokens = tok_append(tokens, "Not", "!")
let i = i + 1 let i = i + 1
} }
} else { } else {
if ch == "<" { if c == 60 {
if peek_ch == "=" { // '<' = 60
let tokens = native_list_append(tokens, make_tok("LtEq", "<=")) if peek_c == 61 {
let tokens = tok_append(tokens, "LtEq", "<=")
let i = i + 2 let i = i + 2
} else { } else {
let tokens = native_list_append(tokens, make_tok("Lt", "<")) let tokens = tok_append(tokens, "Lt", "<")
let i = i + 1 let i = i + 1
} }
} else { } else {
if ch == ">" { if c == 62 {
if peek_ch == "=" { // '>' = 62
let tokens = native_list_append(tokens, make_tok("GtEq", ">=")) if peek_c == 61 {
let tokens = tok_append(tokens, "GtEq", ">=")
let i = i + 2 let i = i + 2
} else { } else {
let tokens = native_list_append(tokens, make_tok("Gt", ">")) let tokens = tok_append(tokens, "Gt", ">")
let i = i + 1 let i = i + 1
} }
} else { } else {
if ch == "&" { if c == 38 {
if peek_ch == "&" { // '&' = 38
let tokens = native_list_append(tokens, make_tok("And", "&&")) if peek_c == 38 {
let tokens = tok_append(tokens, "And", "&&")
let i = i + 2 let i = i + 2
} else { } else {
let i = i + 1 let i = i + 1
} }
} else { } else {
if ch == "|" { if c == 124 {
if peek_ch == "|" { // '|' = 124
let tokens = native_list_append(tokens, make_tok("Or", "||")) if peek_c == 124 {
let tokens = tok_append(tokens, "Or", "||")
let i = i + 2 let i = i + 2
} else { } else {
if peek_ch == ">" { if peek_c == 62 {
let tokens = native_list_append(tokens, make_tok("PipeOp", "|>")) // '>' = 62
let tokens = tok_append(tokens, "PipeOp", "|>")
let i = i + 2 let i = i + 2
} else { } else {
let tokens = native_list_append(tokens, make_tok("Pipe", "|")) let tokens = tok_append(tokens, "Pipe", "|")
let i = i + 1 let i = i + 1
} }
} }
} else { } else {
if ch == "-" { if c == 45 {
if peek_ch == ">" { // '-' = 45
let tokens = native_list_append(tokens, make_tok("Arrow", "->")) if peek_c == 62 {
// '>' = 62
let tokens = tok_append(tokens, "Arrow", "->")
let i = i + 2 let i = i + 2
} else { } else {
let tokens = native_list_append(tokens, make_tok("Minus", "-")) let tokens = tok_append(tokens, "Minus", "-")
let i = i + 1 let i = i + 1
} }
} else { } else {
if ch == ":" { if c == 58 {
if peek_ch == ":" { // ':' = 58
let tokens = native_list_append(tokens, make_tok("ColonColon", "::")) if peek_c == 58 {
let tokens = tok_append(tokens, "ColonColon", "::")
let i = i + 2 let i = i + 2
} else { } else {
let tokens = native_list_append(tokens, make_tok("Colon", ":")) let tokens = tok_append(tokens, "Colon", ":")
let i = i + 1 let i = i + 1
} }
} else { } else {
if ch == "+" { if c == 43 {
let tokens = native_list_append(tokens, make_tok("Plus", "+")) // '+' = 43
let tokens = tok_append(tokens, "Plus", "+")
let i = i + 1 let i = i + 1
} else { } else {
if ch == "*" { if c == 42 {
let tokens = native_list_append(tokens, make_tok("Star", "*")) // '*' = 42
let tokens = tok_append(tokens, "Star", "*")
let i = i + 1 let i = i + 1
} else { } else {
if ch == "%" { if c == 37 {
let tokens = native_list_append(tokens, make_tok("Percent", "%")) // '%' = 37
let tokens = tok_append(tokens, "Percent", "%")
let i = i + 1 let i = i + 1
} else { } else {
if ch == "(" { if c == 40 {
let tokens = native_list_append(tokens, make_tok("LParen", "(")) // '(' = 40
let tokens = tok_append(tokens, "LParen", "(")
let i = i + 1 let i = i + 1
} else { } else {
if ch == ")" { if c == 41 {
let tokens = native_list_append(tokens, make_tok("RParen", ")")) // ')' = 41
let tokens = tok_append(tokens, "RParen", ")")
let i = i + 1 let i = i + 1
} else { } else {
if ch == "{" { if c == 123 {
let tokens = native_list_append(tokens, make_tok("LBrace", "{")) // '{' = 123
let tokens = tok_append(tokens, "LBrace", "{")
let i = i + 1 let i = i + 1
} else { } else {
if ch == "}" { if c == 125 {
let tokens = native_list_append(tokens, make_tok("RBrace", "}")) // '}' = 125
let tokens = tok_append(tokens, "RBrace", "}")
let i = i + 1 let i = i + 1
} else { } else {
if ch == "[" { if c == 91 {
let tokens = native_list_append(tokens, make_tok("LBracket", "[")) // '[' = 91
let tokens = tok_append(tokens, "LBracket", "[")
let i = i + 1 let i = i + 1
} else { } else {
if ch == "]" { if c == 93 {
let tokens = native_list_append(tokens, make_tok("RBracket", "]")) // ']' = 93
let tokens = tok_append(tokens, "RBracket", "]")
let i = i + 1 let i = i + 1
} else { } else {
if ch == "," { if c == 44 {
let tokens = native_list_append(tokens, make_tok("Comma", ",")) // ',' = 44
let tokens = tok_append(tokens, "Comma", ",")
let i = i + 1 let i = i + 1
} else { } else {
if ch == "." { if c == 46 {
// Check for ..= (inclusive range) before .. (exclusive range) before single . // '.' = 46: check for ..= or ..
let peek2_i = i + 2 let peek2_i = i + 2
let peek2_ch = "" let peek2_c: Int = -1
if peek2_i < total { if peek2_i < total {
let peek2_ch: String = native_list_get(chars, peek2_i) let peek2_c: Int = str_char_code(source, peek2_i)
} }
if peek_ch == "." { if peek_c == 46 {
if peek2_ch == "=" { // '..' prefix
let tokens = native_list_append(tokens, make_tok("DotDotEq", "..=")) if peek2_c == 61 {
// '..=' = 46 46 61
let tokens = tok_append(tokens, "DotDotEq", "..=")
let i = i + 3 let i = i + 3
} else { } else {
let tokens = native_list_append(tokens, make_tok("DotDot", "..")) let tokens = tok_append(tokens, "DotDot", "..")
let i = i + 2 let i = i + 2
} }
} else { } else {
let tokens = native_list_append(tokens, make_tok("Dot", ".")) let tokens = tok_append(tokens, "Dot", ".")
let i = i + 1 let i = i + 1
} }
} else { } else {
if ch == ";" { if c == 59 {
let tokens = native_list_append(tokens, make_tok("Semicolon", ";")) // ';' = 59
let tokens = tok_append(tokens, "Semicolon", ";")
let i = i + 1 let i = i + 1
} else { } else {
if ch == "@" { if c == 64 {
let tokens = native_list_append(tokens, make_tok("At", "@")) // '@' = 64
let tokens = tok_append(tokens, "At", "@")
let i = i + 1 let i = i + 1
} else { } else {
if ch == "?" { if c == 63 {
let tokens = native_list_append(tokens, make_tok("QuestionMark", "?")) // '?' = 63
let tokens = tok_append(tokens, "QuestionMark", "?")
let i = i + 1 let i = i + 1
} else { } else {
// unknown char - skip // unknown char - skip
@@ -988,6 +1074,6 @@ fn lex(source: String) -> [Map<String, Any>] {
} }
} }
let tokens = native_list_append(tokens, make_tok("Eof", "")) let tokens = tok_append(tokens, "Eof", "")
tokens tokens
} }
lang/el-compiler/src/parser.el
+34 -30
View File
@@ -9,25 +9,28 @@
// The token list is passed as a parameter to all parse functions. // The token list is passed as a parameter to all parse functions.
// native_list_get is used to index into it without cloning. // native_list_get is used to index into it without cloning.
// //
// Entry point: fn parse(tokens: [Map<String, Any>]) -> [Map<String, Any>] // Entry point: fn parse(tokens: [Any]) -> [Map<String, Any>]
// -- Token access helpers ------------------------------------------------------ // -- Token access helpers ------------------------------------------------------
// Tokens is a flat [Any] list: tokens[2*i] = kind, tokens[2*i+1] = value.
// This avoids one ElMap allocation per token (~112B each), saving ~4MB on large
// programs. All callers use these helpers -- only these three need updating.
fn tok_at(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn tok_at(tokens: [Any], pos: Int) -> Map<String, Any> {
native_list_get(tokens, pos) let kind: String = native_list_get(tokens, pos * 2)
let value: String = native_list_get(tokens, pos * 2 + 1)
{ "kind": kind, "value": value }
} }
fn tok_kind(tokens: [Map<String, Any>], pos: Int) -> String { fn tok_kind(tokens: [Any], pos: Int) -> String {
let t = native_list_get(tokens, pos) native_list_get(tokens, pos * 2)
t["kind"]
} }
fn tok_value(tokens: [Map<String, Any>], pos: Int) -> String { fn tok_value(tokens: [Any], pos: Int) -> String {
let t = native_list_get(tokens, pos) native_list_get(tokens, pos * 2 + 1)
t["value"]
} }
fn expect(tokens: [Map<String, Any>], pos: Int, kind: String) -> Int { fn expect(tokens: [Any], pos: Int, kind: String) -> Int {
let k = tok_kind(tokens, pos) let k = tok_kind(tokens, pos)
if k == kind { if k == kind {
return pos + 1 return pos + 1
@@ -46,7 +49,7 @@ fn make_result(node: Map<String, Any>, pos: Int) -> Map<String, Any> {
// Skips over a type annotation, returning the new position. // Skips over a type annotation, returning the new position.
// Types can be: Ident, [Type], Map<K,V>, Type?, Type<Type,...> // Types can be: Ident, [Type], Map<K,V>, Type?, Type<Type,...>
fn skip_type(tokens: [Map<String, Any>], pos: Int) -> Int { fn skip_type(tokens: [Any], pos: Int) -> Int {
let k = tok_kind(tokens, pos) let k = tok_kind(tokens, pos)
// Array type: [Type] // Array type: [Type]
if k == "LBracket" { if k == "LBracket" {
@@ -103,7 +106,7 @@ fn skip_type(tokens: [Map<String, Any>], pos: Int) -> Int {
// -- Parameter list ------------------------------------------------------------ // -- Parameter list ------------------------------------------------------------
// Parses (name: Type, name: Type, ...) - returns { "params": [...], "pos": ... } // Parses (name: Type, name: Type, ...) - returns { "params": [...], "pos": ... }
fn parse_params(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_params(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "LParen") let p = expect(tokens, pos, "LParen")
let params: [Map<String, Any>] = native_list_empty() let params: [Map<String, Any>] = native_list_empty()
let running = true let running = true
@@ -292,7 +295,7 @@ fn is_void_element(name: String) -> Bool {
// Collect tokens as text content until we hit Lt, LBrace, Eof, or a // Collect tokens as text content until we hit Lt, LBrace, Eof, or a
// closing-tag marker (Lt Slash). Returns { "text": "...", "pos": p } // closing-tag marker (Lt Slash). Returns { "text": "...", "pos": p }
fn parse_html_text_tokens(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_html_text_tokens(tokens: [Any], pos: Int) -> Map<String, Any> {
let parts: [String] = native_list_empty() let parts: [String] = native_list_empty()
let p = pos let p = pos
let running = true let running = true
@@ -322,7 +325,7 @@ fn parse_html_text_tokens(tokens: [Map<String, Any>], pos: Int) -> Map<String, A
// Parse an attribute list: (attrname | attrname="val" | attrname={expr})* // Parse an attribute list: (attrname | attrname="val" | attrname={expr})*
// Stops at Gt or Slash (for self-closing />). // Stops at Gt or Slash (for self-closing />).
fn parse_html_attrs(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_html_attrs(tokens: [Any], pos: Int) -> Map<String, Any> {
let attrs: [Map<String, Any>] = native_list_empty() let attrs: [Map<String, Any>] = native_list_empty()
let p = pos let p = pos
let running = true let running = true
@@ -374,7 +377,7 @@ fn parse_html_attrs(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
// Parse the children of an HTML element until we see the closing tag </tag> // Parse the children of an HTML element until we see the closing tag </tag>
// or EOF. Returns { "children": [...], "pos": p_after_closing_tag } // or EOF. Returns { "children": [...], "pos": p_after_closing_tag }
fn parse_html_children(tokens: [Map<String, Any>], pos: Int, parent_tag: String) -> Map<String, Any> { fn parse_html_children(tokens: [Any], pos: Int, parent_tag: String) -> Map<String, Any> {
let children: [Map<String, Any>] = native_list_empty() let children: [Map<String, Any>] = native_list_empty()
let p = pos let p = pos
let running = true let running = true
@@ -514,14 +517,14 @@ fn parse_html_children(tokens: [Map<String, Any>], pos: Int, parent_tag: String)
// Parse body of {#each} until {/each}. Mirrors parse_html_children but // Parse body of {#each} until {/each}. Mirrors parse_html_children but
// stops at the {/each} sentinel rather than a closing element tag. // stops at the {/each} sentinel rather than a closing element tag.
fn parse_html_each_body(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_html_each_body(tokens: [Any], pos: Int) -> Map<String, Any> {
parse_html_children(tokens, pos, "__each__") parse_html_children(tokens, pos, "__each__")
} }
// Parse a single HTML element: <tag attrs> children </tag> // Parse a single HTML element: <tag attrs> children </tag>
// or self-closing: <tag attrs/> // or self-closing: <tag attrs/>
// Pos points to the Lt token. // Pos points to the Lt token.
fn parse_html_element(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_html_element(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = pos let p = pos
// consume < // consume <
let p = expect(tokens, p, "Lt") let p = expect(tokens, p, "Lt")
@@ -558,7 +561,7 @@ fn parse_html_element(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any>
// Entry point for HTML template parsing. // Entry point for HTML template parsing.
// Pos points to Lt (or Lt Not for <!doctype>). // Pos points to Lt (or Lt Not for <!doctype>).
// May parse an optional <!doctype html> prefix followed by the root element. // May parse an optional <!doctype html> prefix followed by the root element.
fn parse_html_template(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_html_template(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = pos let p = pos
// Check for <!doctype html> // Check for <!doctype html>
let doctype = false let doctype = false
@@ -596,7 +599,7 @@ fn parse_html_template(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any>
make_result({ "expr": "HtmlTemplate", "root": root_with_doctype }, p) make_result({ "expr": "HtmlTemplate", "root": root_with_doctype }, p)
} }
fn parse_primary(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_primary(tokens: [Any], pos: Int) -> Map<String, Any> {
let k = tok_kind(tokens, pos) let k = tok_kind(tokens, pos)
let v = tok_value(tokens, pos) let v = tok_value(tokens, pos)
@@ -819,7 +822,7 @@ fn parse_primary(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "expr": "Nil" }, pos + 1) make_result({ "expr": "Nil" }, pos + 1)
} }
fn parse_if(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_if(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "If") let p = expect(tokens, pos, "If")
// Suppress Map-literal parsing in the cond so a stray `{` (the start // Suppress Map-literal parsing in the cond so a stray `{` (the start
// of the then-block) isn't gobbled as a Map. // of the then-block) isn't gobbled as a Map.
@@ -855,7 +858,7 @@ fn parse_if(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "expr": "If", "cond": cond, "then": then_stmts, "else": else_stmts, "has_else": has_else }, p) make_result({ "expr": "If", "cond": cond, "then": then_stmts, "else": else_stmts, "has_else": has_else }, p)
} }
fn parse_match(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_match(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "Match") let p = expect(tokens, pos, "Match")
let prev_no_block: String = state_get("__no_block_expr") let prev_no_block: String = state_get("__no_block_expr")
state_set("__no_block_expr", "1") state_set("__no_block_expr", "1")
@@ -895,7 +898,7 @@ fn parse_match(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "expr": "Match", "subject": subject, "arms": arms }, p) make_result({ "expr": "Match", "subject": subject, "arms": arms }, p)
} }
fn parse_pattern(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_pattern(tokens: [Any], pos: Int) -> Map<String, Any> {
let k = tok_kind(tokens, pos) let k = tok_kind(tokens, pos)
if k == "Ident" { if k == "Ident" {
let v = tok_value(tokens, pos) let v = tok_value(tokens, pos)
@@ -924,7 +927,7 @@ fn parse_pattern(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "pattern": "Wildcard" }, pos + 1) make_result({ "pattern": "Wildcard" }, pos + 1)
} }
fn parse_for_expr(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_for_expr(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "For") let p = expect(tokens, pos, "For")
let item_name = tok_value(tokens, p) let item_name = tok_value(tokens, p)
let p = p + 1 let p = p + 1
@@ -941,7 +944,7 @@ fn parse_for_expr(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
make_result({ "expr": "For", "item": item_name, "list": list_expr, "body": body }, p) make_result({ "expr": "For", "item": item_name, "list": list_expr, "body": body }, p)
} }
fn parse_block(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_block(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = expect(tokens, pos, "LBrace") let p = expect(tokens, pos, "LBrace")
let stmts: [Map<String, Any>] = native_list_empty() let stmts: [Map<String, Any>] = native_list_empty()
let running = true let running = true
@@ -998,7 +1001,7 @@ fn is_duration_unit(name: String) -> Bool {
false false
} }
fn parse_postfix(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_postfix(tokens: [Any], pos: Int) -> Map<String, Any> {
let r = parse_primary(tokens, pos) let r = parse_primary(tokens, pos)
let node = r["node"] let node = r["node"]
let p = r["pos"] let p = r["pos"]
@@ -1115,7 +1118,7 @@ fn is_binop(kind: String) -> Bool {
false false
} }
fn parse_binop(tokens: [Map<String, Any>], pos: Int, min_prec: Int) -> Map<String, Any> { fn parse_binop(tokens: [Any], pos: Int, min_prec: Int) -> Map<String, Any> {
let r = parse_postfix(tokens, pos) let r = parse_postfix(tokens, pos)
let left = r["node"] let left = r["node"]
let p = r["pos"] let p = r["pos"]
@@ -1140,13 +1143,13 @@ fn parse_binop(tokens: [Map<String, Any>], pos: Int, min_prec: Int) -> Map<Strin
make_result(left, p) make_result(left, p)
} }
fn parse_expr(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_expr(tokens: [Any], pos: Int) -> Map<String, Any> {
parse_binop(tokens, pos, 1) parse_binop(tokens, pos, 1)
} }
// -- Statement parsing --------------------------------------------------------- // -- Statement parsing ---------------------------------------------------------
fn parse_stmt(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> { fn parse_stmt(tokens: [Any], pos: Int) -> Map<String, Any> {
let k = tok_kind(tokens, pos) let k = tok_kind(tokens, pos)
// let binding // let binding
@@ -1619,8 +1622,9 @@ fn parse_stmt(tokens: [Map<String, Any>], pos: Int) -> Map<String, Any> {
// -- Top-level parse ------------------------------------------------------------ // -- Top-level parse ------------------------------------------------------------
fn parse(tokens: [Map<String, Any>]) -> [Map<String, Any>] { fn parse(tokens: [Any]) -> [Map<String, Any>] {
let total: Int = native_list_len(tokens) // Flat list: 2 entries per token, so divide by 2 for token count.
let total: Int = native_list_len(tokens) / 2
let stmts: [Map<String, Any>] = native_list_empty() let stmts: [Map<String, Any>] = native_list_empty()
let pos: Int = 0 let pos: Int = 0
let running = true let running = true