Will Anderson
e587bedf30
Combines two orthogonal optimizations: 1. c_escape batching (from alpha): ASCII runs emitted as str_slice segments instead of one str_char_at string per byte. O(N) allocs → O(K) where K = special chars. 2. scan_interp_string batching (from beta): char dispatch via str_char_code (Int) + clean_start tracking to flush plain runs as str_slice. Eliminates per-char string allocations in the string-literal scanning hot path. Result on web/src/main.el: 14.5MB -> 13.4MB peak RSS (-7.6%). Self-hosting: PASS.
1080 lines
52 KiB
EmacsLisp
1080 lines
52 KiB
EmacsLisp
// lexer.el - el self-hosting lexer
|
||
//
|
||
// Tokenises an el source string into a list of token maps.
|
||
// Each token is a Map<String, Any> with keys:
|
||
// "kind" -> String (e.g. "Int", "Ident", "Plus")
|
||
// "value" -> String (the raw text of the token)
|
||
//
|
||
// Entry point: fn lex(source: String) -> [Map<String, Any>]
|
||
//
|
||
// Performance: the hot lexer loop uses str_char_code (returns Int) instead of
|
||
// str_char_at (returns strdup'd String) for character classification.
|
||
// For a 400KB source, str_char_at allocates ~400K × 16B = ~6.4MB of temporary
|
||
// strings for the `ch` variable alone. str_char_code avoids all that.
|
||
|
||
// -- Character helpers (Int-based, no string allocation) ----------------------
|
||
// These operate on char codes (from str_char_code) instead of str_char_at,
|
||
// eliminating one strdup per character in the hot lexer loop.
|
||
|
||
fn is_digit_code(c: Int) -> Bool {
|
||
// '0'=48 .. '9'=57
|
||
if c >= 48 {
|
||
if c <= 57 { return true }
|
||
}
|
||
false
|
||
}
|
||
|
||
fn is_alpha_code(c: Int) -> Bool {
|
||
// 'A'=65..'Z'=90, 'a'=97..'z'=122
|
||
if c >= 65 {
|
||
if c <= 90 { return true }
|
||
}
|
||
if c >= 97 {
|
||
if c <= 122 { return true }
|
||
}
|
||
false
|
||
}
|
||
|
||
fn is_alnum_or_underscore_code(c: Int) -> Bool {
|
||
if is_digit_code(c) { return true }
|
||
if is_alpha_code(c) { return true }
|
||
if c == 95 { return true } // '_'
|
||
false
|
||
}
|
||
|
||
fn is_ws_code(c: Int) -> Bool {
|
||
if c == 32 { return true } // ' '
|
||
if c == 9 { return true } // '\t'
|
||
if c == 10 { return true } // '\n'
|
||
if c == 13 { return true } // '\r'
|
||
false
|
||
}
|
||
|
||
// Legacy String-based helpers kept for scan_interp helpers that use str_char_at.
|
||
fn lex_is_digit(ch: String) -> Bool {
|
||
if ch == "0" { return true }
|
||
if ch == "1" { return true }
|
||
if ch == "2" { return true }
|
||
if ch == "3" { return true }
|
||
if ch == "4" { return true }
|
||
if ch == "5" { return true }
|
||
if ch == "6" { return true }
|
||
if ch == "7" { return true }
|
||
if ch == "8" { return true }
|
||
if ch == "9" { return true }
|
||
false
|
||
}
|
||
|
||
fn lex_is_alpha(ch: String) -> Bool {
|
||
if ch == "a" { return true }
|
||
if ch == "b" { return true }
|
||
if ch == "c" { return true }
|
||
if ch == "d" { return true }
|
||
if ch == "e" { return true }
|
||
if ch == "f" { return true }
|
||
if ch == "g" { return true }
|
||
if ch == "h" { return true }
|
||
if ch == "i" { return true }
|
||
if ch == "j" { return true }
|
||
if ch == "k" { return true }
|
||
if ch == "l" { return true }
|
||
if ch == "m" { return true }
|
||
if ch == "n" { return true }
|
||
if ch == "o" { return true }
|
||
if ch == "p" { return true }
|
||
if ch == "q" { return true }
|
||
if ch == "r" { return true }
|
||
if ch == "s" { return true }
|
||
if ch == "t" { return true }
|
||
if ch == "u" { return true }
|
||
if ch == "v" { return true }
|
||
if ch == "w" { return true }
|
||
if ch == "x" { return true }
|
||
if ch == "y" { return true }
|
||
if ch == "z" { return true }
|
||
if ch == "A" { return true }
|
||
if ch == "B" { return true }
|
||
if ch == "C" { return true }
|
||
if ch == "D" { return true }
|
||
if ch == "E" { return true }
|
||
if ch == "F" { return true }
|
||
if ch == "G" { return true }
|
||
if ch == "H" { return true }
|
||
if ch == "I" { return true }
|
||
if ch == "J" { return true }
|
||
if ch == "K" { return true }
|
||
if ch == "L" { return true }
|
||
if ch == "M" { return true }
|
||
if ch == "N" { return true }
|
||
if ch == "O" { return true }
|
||
if ch == "P" { return true }
|
||
if ch == "Q" { return true }
|
||
if ch == "R" { return true }
|
||
if ch == "S" { return true }
|
||
if ch == "T" { return true }
|
||
if ch == "U" { return true }
|
||
if ch == "V" { return true }
|
||
if ch == "W" { return true }
|
||
if ch == "X" { return true }
|
||
if ch == "Y" { return true }
|
||
if ch == "Z" { return true }
|
||
false
|
||
}
|
||
|
||
fn is_alnum_or_underscore(ch: String) -> Bool {
|
||
if lex_is_digit(ch) { return true }
|
||
if lex_is_alpha(ch) { return true }
|
||
if ch == "_" { return true }
|
||
false
|
||
}
|
||
|
||
fn lex_is_whitespace(ch: String) -> Bool {
|
||
if ch == " " { return true }
|
||
if ch == "\t" { return true }
|
||
if ch == "\n" { return true }
|
||
if ch == "\r" { return true }
|
||
false
|
||
}
|
||
|
||
// tok_append — append a (kind, value) pair to a flat token list.
|
||
// Returns the updated list. Gamma combines flat-list + char-code for max savings.
|
||
fn tok_append(tokens: [Any], kind: String, value: String) -> [Any] {
|
||
let tokens = native_list_append(tokens, kind)
|
||
native_list_append(tokens, value)
|
||
}
|
||
|
||
// -- Keyword lookup ------------------------------------------------------------
|
||
|
||
fn keyword_kind(word: String) -> String {
|
||
if word == "let" { return "Let" }
|
||
if word == "fn" { return "Fn" }
|
||
if word == "type" { return "Type" }
|
||
if word == "enum" { return "Enum" }
|
||
if word == "match" { return "Match" }
|
||
if word == "return" { return "Return" }
|
||
if word == "if" { return "If" }
|
||
if word == "else" { return "Else" }
|
||
if word == "for" { return "For" }
|
||
if word == "in" { return "In" }
|
||
if word == "while" { return "While" }
|
||
if word == "import" { return "Import" }
|
||
if word == "from" { return "From" }
|
||
if word == "as" { return "As" }
|
||
if word == "with" { return "With" }
|
||
if word == "sealed" { return "Sealed" }
|
||
if word == "activate" { return "Activate" }
|
||
if word == "where" { return "Where" }
|
||
if word == "test" { return "Test" }
|
||
if word == "seed" { return "Seed" }
|
||
if word == "assert" { return "Assert" }
|
||
if word == "protocol" { return "Protocol" }
|
||
if word == "impl" { return "Impl" }
|
||
if word == "retry" { return "Retry" }
|
||
if word == "times" { return "Times" }
|
||
if word == "fallback" { return "Fallback" }
|
||
if word == "reason" { return "Reason" }
|
||
if word == "parallel" { return "Parallel" }
|
||
if word == "trace" { return "Trace" }
|
||
if word == "requires" { return "Requires" }
|
||
if word == "deploy" { return "Deploy" }
|
||
if word == "to" { return "To" }
|
||
if word == "via" { return "Via" }
|
||
if word == "target" { return "Target" }
|
||
if word == "true" { return "Bool" }
|
||
if word == "false" { return "Bool" }
|
||
if word == "cgi" { return "Cgi" }
|
||
if word == "service" { return "Service" }
|
||
if word == "manager" { return "Manager" }
|
||
if word == "engine" { return "Engine" }
|
||
if word == "accessor" { return "Accessor" }
|
||
if word == "vessel" { return "Vessel" }
|
||
if word == "extern" { return "Extern" }
|
||
if word == "break" { return "Break" }
|
||
if word == "continue" { return "Continue" }
|
||
""
|
||
}
|
||
|
||
// -- Scan helpers --------------------------------------------------------------
|
||
// All scan helpers receive the chars list and total length.
|
||
|
||
// scan_digits - advance i while chars[i] is a digit
|
||
// Returns { "text": ..., "pos": i }
|
||
fn scan_digits(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let c: Int = str_char_code(src, i)
|
||
if is_digit_code(c) {
|
||
let i = i + 1
|
||
} else {
|
||
let running = false
|
||
}
|
||
}
|
||
}
|
||
// Use str_slice instead of building a parts list — O(1) allocation, O(n) copy.
|
||
{ "text": str_slice(src, start, i), "pos": i }
|
||
}
|
||
|
||
// scan_ident - advance i while chars[i] is alphanumeric or underscore
|
||
fn scan_ident(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let c: Int = str_char_code(src, i)
|
||
if is_alnum_or_underscore_code(c) {
|
||
let i = i + 1
|
||
} else {
|
||
let running = false
|
||
}
|
||
}
|
||
}
|
||
// Use str_slice instead of building a parts list — O(1) allocation, O(n) copy.
|
||
{ "text": str_slice(src, start, i), "pos": i }
|
||
}
|
||
|
||
// -- Code-bearing string detection + comment strip ----------------------------
|
||
// Inline JS/CSS literals embedded in El source (e.g. <script>-</script> blobs
|
||
// or stylesheet payloads inside string literals) carry their own line and
|
||
// block comments. Those comments leak into the served HTML and reveal build
|
||
// notes the visitor should never see. We strip them at the lexer so every
|
||
// downstream consumer (codegen-c, codegen-js, parser) gets the cleaned form.
|
||
//
|
||
// looks_like_code - heuristic gate so we only strip strings that actually
|
||
// embed JS or CSS. Plain prose, hex blobs, JSON, etc. pass through verbatim.
|
||
|
||
fn substr_at(src: String, start: Int, total: Int, needle: String) -> Bool {
|
||
let nlen: Int = str_len(needle)
|
||
if start + nlen > total { return false }
|
||
// Use str_slice comparison instead of char-by-char loop.
|
||
str_eq(str_slice(src, start, start + nlen), needle)
|
||
}
|
||
|
||
fn str_has(s: String, needle: String) -> Bool {
|
||
// Use the built-in str_contains which is implemented in native C — O(n) single pass.
|
||
str_contains(s, needle)
|
||
}
|
||
|
||
fn looks_like_code(s: String) -> Bool {
|
||
if str_has(s, "<script") { return true }
|
||
if str_has(s, "<style") { return true }
|
||
if str_has(s, "function") {
|
||
if str_has(s, ";") { return true }
|
||
}
|
||
false
|
||
}
|
||
|
||
// strip_code_comments - character-by-character walk. Tracks JS string state
|
||
// (single, double, backtick) and never strips inside one. Backslash escapes
|
||
// inside JS strings consume the next char verbatim. URLs like https:// are
|
||
// preserved by checking the previous char before treating // as a line
|
||
// comment opener: if the char immediately before '/' is ':', emit the '/'
|
||
// literally and advance one position.
|
||
fn strip_code_comments(s: String) -> String {
|
||
let total: Int = str_len(s)
|
||
let out_parts: [String] = native_list_empty()
|
||
let i = 0
|
||
let in_squote = false
|
||
let in_dquote = false
|
||
let in_btick = false
|
||
let prev = ""
|
||
while i < total {
|
||
let ch: String = str_char_at(s, i)
|
||
let in_js_string = false
|
||
if in_squote { let in_js_string = true }
|
||
if in_dquote { let in_js_string = true }
|
||
if in_btick { let in_js_string = true }
|
||
|
||
if in_js_string {
|
||
// Backslash escape: consume next char verbatim regardless of which.
|
||
if ch == "\\" {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let nc: String = str_char_at(s, next_i)
|
||
let out_parts = native_list_append(out_parts, nc)
|
||
let prev = nc
|
||
let i = next_i + 1
|
||
} else {
|
||
let prev = ch
|
||
let i = next_i
|
||
}
|
||
} else {
|
||
if in_squote {
|
||
if ch == "'" { let in_squote = false }
|
||
} else {
|
||
if in_dquote {
|
||
if ch == "\"" { let in_dquote = false }
|
||
} else {
|
||
if in_btick {
|
||
if ch == "`" { let in_btick = false }
|
||
}
|
||
}
|
||
}
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
// Not in a JS string. Check for comment openers.
|
||
let next_i = i + 1
|
||
let next_ch = ""
|
||
if next_i < total {
|
||
let next_ch: String = str_char_at(s, next_i)
|
||
}
|
||
|
||
if ch == "/" {
|
||
if next_ch == "/" {
|
||
// URL guard: prev char ':' means this is "://", not a comment.
|
||
if prev == ":" {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
// Skip until newline (newline itself is preserved so
|
||
// surrounding line counts/structure stay sane).
|
||
let i = i + 2
|
||
let scanning = true
|
||
while scanning {
|
||
if i >= total {
|
||
let scanning = false
|
||
} else {
|
||
let lc: String = str_char_at(s, i)
|
||
if lc == "\n" {
|
||
let scanning = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
let prev = ""
|
||
}
|
||
} else {
|
||
if next_ch == "*" {
|
||
// Skip until matching "*/".
|
||
let i = i + 2
|
||
let scanning2 = true
|
||
while scanning2 {
|
||
if i >= total {
|
||
let scanning2 = false
|
||
} else {
|
||
let bc: String = str_char_at(s, i)
|
||
if bc == "*" {
|
||
let after = i + 1
|
||
if after < total {
|
||
let nc2: String = str_char_at(s, after)
|
||
if nc2 == "/" {
|
||
let i = after + 1
|
||
let scanning2 = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
let prev = ""
|
||
} else {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
}
|
||
}
|
||
} else {
|
||
// Open a JS string?
|
||
if ch == "'" {
|
||
let in_squote = true
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
if ch == "\"" {
|
||
let in_dquote = true
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
if ch == "`" {
|
||
let in_btick = true
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
str_join(out_parts, "")
|
||
}
|
||
|
||
// scan_string - scan a quoted string literal, handling \" escapes.
|
||
// Starts AFTER the opening quote. Returns { "text": content, "pos": i_after_close }
|
||
fn scan_string(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let parts: [String] = native_list_empty()
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let ch: String = str_char_at(src, i)
|
||
if ch == "\\" {
|
||
// escape: peek next char
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let next_ch: String = str_char_at(src, next_i)
|
||
if next_ch == "\"" {
|
||
let parts = native_list_append(parts, "\"")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "n" {
|
||
let parts = native_list_append(parts, "\n")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "t" {
|
||
let parts = native_list_append(parts, "\t")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "r" {
|
||
let parts = native_list_append(parts, "\r")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "\\" {
|
||
let parts = native_list_append(parts, "\\")
|
||
let i = next_i + 1
|
||
} else {
|
||
let parts = native_list_append(parts, next_ch)
|
||
let i = next_i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if ch == "\"" {
|
||
let i = i + 1
|
||
let running = false
|
||
} else {
|
||
let parts = native_list_append(parts, ch)
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
{ "text": str_join(parts, ""), "pos": i }
|
||
}
|
||
|
||
// -- String interpolation ------------------------------------------------------
|
||
//
|
||
// scan_interp_brace - scan from `start` (the char after `${`) to the matching
|
||
// `}`, tracking brace depth so inner braces (e.g. fn calls, map literals) are
|
||
// handled correctly. Returns { "text": inner_source, "pos": i_after_close }.
|
||
fn scan_interp_brace(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let depth = 1
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let ch: String = str_char_at(src, i)
|
||
if ch == "{" {
|
||
let depth = depth + 1
|
||
let i = i + 1
|
||
} else {
|
||
if ch == "}" {
|
||
let depth = depth - 1
|
||
if depth <= 0 {
|
||
// Closing brace of the interpolation - stop, do not include it
|
||
let i = i + 1
|
||
let running = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// Use str_slice instead of parts list — the inner source is a contiguous substring.
|
||
{ "text": str_slice(src, start, i - 1), "pos": i }
|
||
}
|
||
|
||
// interp_tokens_append_all - copy every (kind, value) pair from flat src list
|
||
// into flat dst list, skipping the trailing Eof pair that lex() always appends.
|
||
fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] {
|
||
let src_len: Int = native_list_len(src)
|
||
let j = 0
|
||
let result = dst
|
||
while j < src_len {
|
||
let kind: String = native_list_get(src, j)
|
||
if kind == "Eof" {
|
||
let j = src_len
|
||
} else {
|
||
let val: String = native_list_get(src, j + 1)
|
||
let result = native_list_append(result, kind)
|
||
let result = native_list_append(result, val)
|
||
let j = j + 2
|
||
}
|
||
}
|
||
result
|
||
}
|
||
|
||
// scan_interp_string - scan a string literal that may contain ${expr}
|
||
// interpolations. Starts AFTER the opening `"`.
|
||
// Returns { "tokens": [token list to inject], "pos": i_after_close_quote }.
|
||
//
|
||
// For a plain string (no ${}) this emits a single Str token, identical to the
|
||
// old scan_string path. For an interpolated string it emits a flat sequence
|
||
// of tokens equivalent to the string-concat expression, for example:
|
||
//
|
||
// "hello ${name}!"
|
||
// => Str("hello ") Plus <tokens for name> Plus Str("!")
|
||
//
|
||
// Empty literal segments between adjacent ${ } blocks are omitted. The
|
||
// resulting token stream is consumed by the existing parse_binop / parse_primary
|
||
// path in the parser with zero parser changes required.
|
||
//
|
||
// Supported escape sequences: \" \n \t \r \\ \$ (literal dollar sign).
|
||
// Nested quotes inside ${} are not supported; use a variable instead.
|
||
//
|
||
// Performance: uses str_char_code (Int) for all character dispatch, eliminating
|
||
// per-character strdup. Plain runs are batched into str_slice segments instead
|
||
// of accumulating single-char strings, reducing list appends from O(N) to O(K)
|
||
// where K = number of escape/special chars in the literal.
|
||
// Char codes: '\' = 92, '"' = 34, '$' = 36, '{' = 123
|
||
fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let out_tokens: [Any] = native_list_empty()
|
||
let cur_parts: [String] = native_list_empty()
|
||
let clean_start = start
|
||
let has_interp = false
|
||
let need_plus = false
|
||
let running = true
|
||
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let c: Int = str_char_code(src, i)
|
||
|
||
if c == 92 {
|
||
// '\\' = 92 — escape sequence: flush clean run, append resolved char
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let nc: Int = str_char_code(src, next_i)
|
||
if nc == 36 {
|
||
// '\$' => literal '$' (36 = '$')
|
||
let cur_parts = native_list_append(cur_parts, "$")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 34 {
|
||
// '\"' => literal '"' (34 = '"')
|
||
let cur_parts = native_list_append(cur_parts, "\"")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 110 {
|
||
// '\n' (110 = 'n')
|
||
let cur_parts = native_list_append(cur_parts, "\n")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 116 {
|
||
// '\t' (116 = 't')
|
||
let cur_parts = native_list_append(cur_parts, "\t")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 114 {
|
||
// '\r' (114 = 'r')
|
||
let cur_parts = native_list_append(cur_parts, "\r")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 92 {
|
||
// '\\' (92)
|
||
let cur_parts = native_list_append(cur_parts, "\\")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
// Unknown escape: emit the escaped char verbatim
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, next_i, next_i + 1))
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
let clean_start = next_i
|
||
let i = next_i
|
||
}
|
||
} else {
|
||
if c == 34 {
|
||
// '"' = 34 — closing quote: flush clean run, stop
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let i = i + 1
|
||
let clean_start = i
|
||
let running = false
|
||
} else {
|
||
if c == 36 {
|
||
// '$' = 36 — possible interpolation start
|
||
let next_i = i + 1
|
||
let is_interp = false
|
||
if next_i < total {
|
||
let nc2: Int = str_char_code(src, next_i)
|
||
if nc2 == 123 {
|
||
// '{' = 123
|
||
let is_interp = true
|
||
}
|
||
}
|
||
if is_interp {
|
||
// Flush the accumulated literal part (if non-empty)
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let part_len: Int = native_list_len(cur_parts)
|
||
if part_len > 0 {
|
||
let part_text = str_join(cur_parts, "")
|
||
if need_plus {
|
||
let out_tokens = tok_append(out_tokens, "Plus", "+")
|
||
}
|
||
let clean_part = part_text
|
||
if looks_like_code(part_text) {
|
||
let clean_part = strip_code_comments(part_text)
|
||
}
|
||
let out_tokens = tok_append(out_tokens, "Str", clean_part)
|
||
let need_plus = true
|
||
}
|
||
let cur_parts = native_list_empty()
|
||
let has_interp = true
|
||
|
||
// Scan brace-balanced expression source
|
||
let brace_result = scan_interp_brace(src, next_i + 1, total)
|
||
let expr_src: String = brace_result["text"]
|
||
let new_i: Int = brace_result["pos"]
|
||
let i = new_i
|
||
let clean_start = new_i
|
||
|
||
// Re-lex the expression and inline the tokens.
|
||
// Wrap in ( ) so that operators inside ${} (e.g.
|
||
// age + 1) are parsed as a grouped sub-expression
|
||
// rather than merging with the surrounding concat
|
||
// Plus tokens at the wrong precedence level.
|
||
let inner_toks: [Any] = lex(expr_src)
|
||
let inner_len: Int = native_list_len(inner_toks)
|
||
|
||
if need_plus {
|
||
let out_tokens = tok_append(out_tokens, "Plus", "+")
|
||
}
|
||
// Empty interpolation ${} => empty string segment
|
||
// inner_len <= 2 = only the Eof pair (kind="Eof", value="")
|
||
if inner_len <= 2 {
|
||
let out_tokens = tok_append(out_tokens, "Str", "")
|
||
} else {
|
||
let out_tokens = tok_append(out_tokens, "LParen", "(")
|
||
let out_tokens = interp_tokens_append_all(out_tokens, inner_toks)
|
||
let out_tokens = tok_append(out_tokens, "RParen", ")")
|
||
}
|
||
let need_plus = true
|
||
} else {
|
||
// Plain '$' not followed by '{' - treat as literal, continue clean run
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
// Plain char — extends clean run, no append needed
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Flush remaining literal segment and build final token list
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let part_len: Int = native_list_len(cur_parts)
|
||
let part_text = str_join(cur_parts, "")
|
||
if has_interp {
|
||
// Interpolated string: only emit trailing segment if non-empty
|
||
if part_len > 0 {
|
||
let clean_part = part_text
|
||
if looks_like_code(part_text) {
|
||
let clean_part = strip_code_comments(part_text)
|
||
}
|
||
if need_plus {
|
||
let out_tokens = tok_append(out_tokens, "Plus", "+")
|
||
}
|
||
let out_tokens = tok_append(out_tokens, "Str", clean_part)
|
||
}
|
||
} else {
|
||
// Plain string with no interpolation - same behaviour as old scan_string
|
||
let clean_text = part_text
|
||
if looks_like_code(part_text) {
|
||
let clean_text = strip_code_comments(part_text)
|
||
}
|
||
let out_tokens = tok_append(out_tokens, "Str", clean_text)
|
||
}
|
||
|
||
{ "tokens": out_tokens, "pos": i }
|
||
}
|
||
|
||
// -- Main lexer ----------------------------------------------------------------
|
||
// Char code constants (avoids strdup for single-char comparison)
|
||
// '/' = 47, '"' = 34, '0'-'9' = 48-57, 'a'-'z' = 97-122, 'A'-'Z' = 65-90
|
||
// '_' = 95, ' '=32, '\t'=9, '\n'=10, '\r'=13
|
||
// '=' = 61, '!' = 33, '<' = 60, '>' = 62, '&' = 38, '|' = 124
|
||
// '-' = 45, ':' = 58, '+' = 43, '*' = 42, '%' = 37
|
||
// '(' = 40, ')' = 41, '{' = 123, '}' = 125, '[' = 91, ']' = 93
|
||
// ',' = 44, '.' = 46, ';' = 59, '@' = 64, '?' = 63
|
||
|
||
fn lex(source: String) -> [Any] {
|
||
// Use str_char_code (returns Int) instead of str_char_at (returns strdup String)
|
||
// for all character classification in the hot loop. For a 400KB source,
|
||
// str_char_at allocates ~400K × 16B = ~6.4MB of temporary strings.
|
||
let total: Int = str_len(source)
|
||
let tokens: [Any] = native_list_empty()
|
||
let i: Int = 0
|
||
|
||
while i < total {
|
||
let c: Int = str_char_code(source, i)
|
||
|
||
// Skip whitespace (space=32, tab=9, newline=10, CR=13)
|
||
if is_ws_code(c) {
|
||
let i = i + 1
|
||
} else {
|
||
// Line comments: // (slash=47)
|
||
if c == 47 {
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let nc: Int = str_char_code(source, next_i)
|
||
if nc == 47 {
|
||
// skip to end of line (newline=10)
|
||
let i = i + 2
|
||
let running2 = true
|
||
while running2 {
|
||
if i >= total {
|
||
let running2 = false
|
||
} else {
|
||
let lc: Int = str_char_code(source, i)
|
||
if lc == 10 {
|
||
let running2 = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Slash", "/")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Slash", "/")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
// String literal: '"' = 34
|
||
if c == 34 {
|
||
let interp_result = scan_interp_string(source, i + 1, total)
|
||
let interp_toks: [Any] = interp_result["tokens"]
|
||
let new_pos: Int = interp_result["pos"]
|
||
let tokens = interp_tokens_append_all(tokens, interp_toks)
|
||
let i = new_pos
|
||
} else {
|
||
// Number literal: '0'-'9' = 48-57
|
||
if is_digit_code(c) {
|
||
let result = scan_digits(source, i, total)
|
||
let num_text: String = result["text"]
|
||
let new_pos: Int = result["pos"]
|
||
// check for float (dot=46 followed by digit)
|
||
if new_pos < total {
|
||
let dc: Int = str_char_code(source, new_pos)
|
||
if dc == 46 {
|
||
let after_dot = new_pos + 1
|
||
if after_dot < total {
|
||
let adc: Int = str_char_code(source, after_dot)
|
||
if is_digit_code(adc) {
|
||
let frac_result = scan_digits(source, after_dot, total)
|
||
let frac_text: String = frac_result["text"]
|
||
let frac_pos: Int = frac_result["pos"]
|
||
let tokens = tok_append(tokens, "Float", num_text + "." + frac_text)
|
||
let i = frac_pos
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
// Identifier or keyword: alpha or '_'=95
|
||
if is_alpha_code(c) || c == 95 {
|
||
let result = scan_ident(source, i, total)
|
||
let word: String = result["text"]
|
||
let new_pos: Int = result["pos"]
|
||
let kw = keyword_kind(word)
|
||
if kw == "" {
|
||
let tokens = tok_append(tokens, "Ident", word)
|
||
} else {
|
||
let tokens = tok_append(tokens, kw, word)
|
||
}
|
||
let i = new_pos
|
||
} else {
|
||
// Multi-char and single-char operators/delimiters
|
||
let peek_i = i + 1
|
||
let peek_c: Int = -1
|
||
if peek_i < total {
|
||
let peek_c: Int = str_char_code(source, peek_i)
|
||
}
|
||
|
||
if c == 61 {
|
||
// '=' = 61
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "EqEq", "==")
|
||
let i = i + 2
|
||
} else {
|
||
if peek_c == 62 {
|
||
// '>' = 62
|
||
let tokens = tok_append(tokens, "FatArrow", "=>")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Eq", "=")
|
||
let i = i + 1
|
||
}
|
||
}
|
||
} else {
|
||
if c == 33 {
|
||
// '!' = 33
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "NotEq", "!=")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Not", "!")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 60 {
|
||
// '<' = 60
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "LtEq", "<=")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Lt", "<")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 62 {
|
||
// '>' = 62
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "GtEq", ">=")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Gt", ">")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 38 {
|
||
// '&' = 38
|
||
if peek_c == 38 {
|
||
let tokens = tok_append(tokens, "And", "&&")
|
||
let i = i + 2
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 124 {
|
||
// '|' = 124
|
||
if peek_c == 124 {
|
||
let tokens = tok_append(tokens, "Or", "||")
|
||
let i = i + 2
|
||
} else {
|
||
if peek_c == 62 {
|
||
// '>' = 62
|
||
let tokens = tok_append(tokens, "PipeOp", "|>")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Pipe", "|")
|
||
let i = i + 1
|
||
}
|
||
}
|
||
} else {
|
||
if c == 45 {
|
||
// '-' = 45
|
||
if peek_c == 62 {
|
||
// '>' = 62
|
||
let tokens = tok_append(tokens, "Arrow", "->")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Minus", "-")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 58 {
|
||
// ':' = 58
|
||
if peek_c == 58 {
|
||
let tokens = tok_append(tokens, "ColonColon", "::")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Colon", ":")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 43 {
|
||
// '+' = 43
|
||
let tokens = tok_append(tokens, "Plus", "+")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 42 {
|
||
// '*' = 42
|
||
let tokens = tok_append(tokens, "Star", "*")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 37 {
|
||
// '%' = 37
|
||
let tokens = tok_append(tokens, "Percent", "%")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 40 {
|
||
// '(' = 40
|
||
let tokens = tok_append(tokens, "LParen", "(")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 41 {
|
||
// ')' = 41
|
||
let tokens = tok_append(tokens, "RParen", ")")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 123 {
|
||
// '{' = 123
|
||
let tokens = tok_append(tokens, "LBrace", "{")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 125 {
|
||
// '}' = 125
|
||
let tokens = tok_append(tokens, "RBrace", "}")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 91 {
|
||
// '[' = 91
|
||
let tokens = tok_append(tokens, "LBracket", "[")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 93 {
|
||
// ']' = 93
|
||
let tokens = tok_append(tokens, "RBracket", "]")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 44 {
|
||
// ',' = 44
|
||
let tokens = tok_append(tokens, "Comma", ",")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 46 {
|
||
// '.' = 46: check for ..= or ..
|
||
let peek2_i = i + 2
|
||
let peek2_c: Int = -1
|
||
if peek2_i < total {
|
||
let peek2_c: Int = str_char_code(source, peek2_i)
|
||
}
|
||
if peek_c == 46 {
|
||
// '..' prefix
|
||
if peek2_c == 61 {
|
||
// '..=' = 46 46 61
|
||
let tokens = tok_append(tokens, "DotDotEq", "..=")
|
||
let i = i + 3
|
||
} else {
|
||
let tokens = tok_append(tokens, "DotDot", "..")
|
||
let i = i + 2
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Dot", ".")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 59 {
|
||
// ';' = 59
|
||
let tokens = tok_append(tokens, "Semicolon", ";")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 64 {
|
||
// '@' = 64
|
||
let tokens = tok_append(tokens, "At", "@")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 63 {
|
||
// '?' = 63
|
||
let tokens = tok_append(tokens, "QuestionMark", "?")
|
||
let i = i + 1
|
||
} else {
|
||
// unknown char - skip
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
let tokens = tok_append(tokens, "Eof", "")
|
||
tokens
|
||
}
|