Archived
bigmerge
6c975b1d50
The module question ended with a limit: textual inlining destroys file
provenance, so a duplicate-definition message could name the symbol but not the
files. Threading it exposed a bigger absence first.
TOKENS HAD NO POSITION AT ALL. A token was a flat (kind, value) pair, so NO
diagnostic in El could name a place -- every error named a symbol and never a
line. That is the prerequisite the module question was resting on.
THE CHAIN, end to end
lexer counts newlines; tok_append mints (kind, value, line)
parser stride 2 -> 3; tok_line added; FnDef carries its line
codegen records <fn> defines_at:<line>
resolve_imports publishes <file> spans <start> <end> for the combined source
checker maps a combined line back to file:line-within-that-file
duplicate definition: 'helper' is defined 2 times — El has no namespacing,
so imported modules share one global scope
/tmp/modtest/a.el:1
/tmp/modtest/b.el:1
PREDICTIONS AND RESULTS
P1 15 stride sites, encapsulated in tok_kind/tok_value TRUE, but see below
P2 adding a line field is mechanical TRUE
P3 the lexer must count newlines TRUE
P4 resolve_imports can record per-file line ranges TRUE
P5 the message can then name both files TRUE
P6 token memory grows TRUE, 25.0 -> 33.9 MB (+36%)
FOUR DEFECTS, EACH FOUND BY RUNNING AND NOT BY READING
1. interp_tokens_append_all walks the token list DIRECTLY with its own copy of
the stride. Gen1 built fine and gen2 emitted corrupt C, because the
compiler's own source uses string interpolation. My search missed it because
I grepped for the variable name `tokens`; it is called `dst`/`result`.
Searching by name instead of by shape -- third time today.
2. tok_count in test_compiler.el carried the stride too. I had scoped the search
to compiler sources and it had escaped into the tests.
3. Nested resolve_imports calls accumulated spans into shared state, so each
republished meaningless line ranges under the parent's name. Making the
buffer local fixed it; guarding the WRITE did not, which is what I tried
first.
4. The first working version reported b.el:3 -- the COMBINED line against a
filename that has no line 3. A file:line that does not match the file is
worse than no line at all.
105/105 native, 37/37 integration, fixpoint ok, compiler self-checks clean.
1115 lines
54 KiB
EmacsLisp
1115 lines
54 KiB
EmacsLisp
// lexer.el - el self-hosting lexer
|
||
//
|
||
// Tokenises an el source string into a list of token maps.
|
||
// Each token is a Map<String, Any> with keys:
|
||
// "kind" -> String (e.g. "Int", "Ident", "Plus")
|
||
// "value" -> String (the raw text of the token)
|
||
//
|
||
// Entry point: fn lex(source: String) -> [Map<String, Any>]
|
||
//
|
||
// Performance: the hot lexer loop uses str_char_code (returns Int) instead of
|
||
// str_char_at (returns strdup'd String) for character classification.
|
||
// For a 400KB source, str_char_at allocates ~400K × 16B = ~6.4MB of temporary
|
||
// strings for the `ch` variable alone. str_char_code avoids all that.
|
||
|
||
// -- Character helpers (Int-based, no string allocation) ----------------------
|
||
// These operate on char codes (from str_char_code) instead of str_char_at,
|
||
// eliminating one strdup per character in the hot lexer loop.
|
||
|
||
fn is_digit_code(c: Int) -> Bool {
|
||
// '0'=48 .. '9'=57
|
||
if c >= 48 {
|
||
if c <= 57 { return true }
|
||
}
|
||
false
|
||
}
|
||
|
||
fn is_alpha_code(c: Int) -> Bool {
|
||
// 'A'=65..'Z'=90, 'a'=97..'z'=122
|
||
if c >= 65 {
|
||
if c <= 90 { return true }
|
||
}
|
||
if c >= 97 {
|
||
if c <= 122 { return true }
|
||
}
|
||
false
|
||
}
|
||
|
||
fn is_alnum_or_underscore_code(c: Int) -> Bool {
|
||
if is_digit_code(c) { return true }
|
||
if is_alpha_code(c) { return true }
|
||
if c == 95 { return true } // '_'
|
||
false
|
||
}
|
||
|
||
fn is_ws_code(c: Int) -> Bool {
|
||
if c == 32 { return true } // ' '
|
||
if c == 9 { return true } // '\t'
|
||
if c == 10 { return true } // '\n'
|
||
if c == 13 { return true } // '\r'
|
||
false
|
||
}
|
||
|
||
// Legacy String-based helpers kept for scan_interp helpers that use str_char_at.
|
||
fn lex_is_digit(ch: String) -> Bool {
|
||
if ch == "0" { return true }
|
||
if ch == "1" { return true }
|
||
if ch == "2" { return true }
|
||
if ch == "3" { return true }
|
||
if ch == "4" { return true }
|
||
if ch == "5" { return true }
|
||
if ch == "6" { return true }
|
||
if ch == "7" { return true }
|
||
if ch == "8" { return true }
|
||
if ch == "9" { return true }
|
||
false
|
||
}
|
||
|
||
fn lex_is_alpha(ch: String) -> Bool {
|
||
if ch == "a" { return true }
|
||
if ch == "b" { return true }
|
||
if ch == "c" { return true }
|
||
if ch == "d" { return true }
|
||
if ch == "e" { return true }
|
||
if ch == "f" { return true }
|
||
if ch == "g" { return true }
|
||
if ch == "h" { return true }
|
||
if ch == "i" { return true }
|
||
if ch == "j" { return true }
|
||
if ch == "k" { return true }
|
||
if ch == "l" { return true }
|
||
if ch == "m" { return true }
|
||
if ch == "n" { return true }
|
||
if ch == "o" { return true }
|
||
if ch == "p" { return true }
|
||
if ch == "q" { return true }
|
||
if ch == "r" { return true }
|
||
if ch == "s" { return true }
|
||
if ch == "t" { return true }
|
||
if ch == "u" { return true }
|
||
if ch == "v" { return true }
|
||
if ch == "w" { return true }
|
||
if ch == "x" { return true }
|
||
if ch == "y" { return true }
|
||
if ch == "z" { return true }
|
||
if ch == "A" { return true }
|
||
if ch == "B" { return true }
|
||
if ch == "C" { return true }
|
||
if ch == "D" { return true }
|
||
if ch == "E" { return true }
|
||
if ch == "F" { return true }
|
||
if ch == "G" { return true }
|
||
if ch == "H" { return true }
|
||
if ch == "I" { return true }
|
||
if ch == "J" { return true }
|
||
if ch == "K" { return true }
|
||
if ch == "L" { return true }
|
||
if ch == "M" { return true }
|
||
if ch == "N" { return true }
|
||
if ch == "O" { return true }
|
||
if ch == "P" { return true }
|
||
if ch == "Q" { return true }
|
||
if ch == "R" { return true }
|
||
if ch == "S" { return true }
|
||
if ch == "T" { return true }
|
||
if ch == "U" { return true }
|
||
if ch == "V" { return true }
|
||
if ch == "W" { return true }
|
||
if ch == "X" { return true }
|
||
if ch == "Y" { return true }
|
||
if ch == "Z" { return true }
|
||
false
|
||
}
|
||
|
||
fn is_alnum_or_underscore(ch: String) -> Bool {
|
||
if lex_is_digit(ch) { return true }
|
||
if lex_is_alpha(ch) { return true }
|
||
if ch == "_" { return true }
|
||
false
|
||
}
|
||
|
||
fn lex_is_whitespace(ch: String) -> Bool {
|
||
if ch == " " { return true }
|
||
if ch == "\t" { return true }
|
||
if ch == "\n" { return true }
|
||
if ch == "\r" { return true }
|
||
false
|
||
}
|
||
|
||
// tok_append — append a (kind, value) pair to a flat token list.
|
||
// Returns the updated list. Gamma combines flat-list + char-code for max savings.
|
||
// A token is (kind, value, line). The line comes from state rather than a
|
||
// parameter so the ~200 existing tok_append call sites are untouched -- the
|
||
// lexer advances __lex_line as it walks, and every token minted takes the line
|
||
// it was minted on.
|
||
//
|
||
// WHY AT ALL: before this a token carried no position, so no diagnostic in El
|
||
// could name a place. Every error named a symbol and never a line, and after
|
||
// textual inlining there was no way to say which FILE a definition came from.
|
||
fn tok_append(tokens: [Any], kind: String, value: String) -> [Any] {
|
||
let tokens = native_list_append(tokens, kind)
|
||
let tokens = native_list_append(tokens, value)
|
||
native_list_append(tokens, state_get("__lex_line"))
|
||
}
|
||
|
||
// -- Keyword lookup ------------------------------------------------------------
|
||
|
||
// keyword_kind — the language's reserved spellings.
|
||
//
|
||
// A grammar is a BASIS: `fn` means function-start because someone said so, and
|
||
// nothing derives it. But unlike the other tables moved out this session, this
|
||
// one stays code, and the SHOULD gate is why. The keyword set is closed by the
|
||
// language definition -- it does not leak the way an allowlist does -- and the
|
||
// lexer runs before the program is understood, so a program can never declare
|
||
// its own keywords. Externalising it would cost file I/O on every compile and
|
||
// buy nothing.
|
||
//
|
||
// Removed 2026-08-17: sealed, activate, seed, protocol, impl. Reserved in the
|
||
// lexer, consumed by no parser or codegen path, and each one stole an
|
||
// identifier from users for nothing. `test` LOOKED inert by the same measure
|
||
// and is not -- codegen consumes it at 4135 for --test mode, 408 uses in the
|
||
// tree. The first measurement checked only parser.el and would have broken all
|
||
// of them.
|
||
fn keyword_kind(word: String) -> String {
|
||
if word == "let" { return "Let" }
|
||
if word == "fn" { return "Fn" }
|
||
if word == "type" { return "Type" }
|
||
if word == "enum" { return "Enum" }
|
||
if word == "match" { return "Match" }
|
||
if word == "return" { return "Return" }
|
||
if word == "if" { return "If" }
|
||
if word == "else" { return "Else" }
|
||
if word == "for" { return "For" }
|
||
if word == "in" { return "In" }
|
||
if word == "while" { return "While" }
|
||
if word == "import" { return "Import" }
|
||
if word == "from" { return "From" }
|
||
if word == "as" { return "As" }
|
||
if word == "with" { return "With" }
|
||
if word == "where" { return "Where" }
|
||
if word == "test" { return "Test" }
|
||
if word == "assert" { return "Assert" }
|
||
if word == "retry" { return "Retry" }
|
||
if word == "times" { return "Times" }
|
||
if word == "fallback" { return "Fallback" }
|
||
if word == "reason" { return "Reason" }
|
||
if word == "parallel" { return "Parallel" }
|
||
if word == "trace" { return "Trace" }
|
||
if word == "requires" { return "Requires" }
|
||
if word == "deploy" { return "Deploy" }
|
||
if word == "to" { return "To" }
|
||
if word == "via" { return "Via" }
|
||
if word == "target" { return "Target" }
|
||
if word == "true" { return "Bool" }
|
||
if word == "false" { return "Bool" }
|
||
if word == "cgi" { return "Cgi" }
|
||
if word == "service" { return "Service" }
|
||
if word == "program" { return "Program" }
|
||
if word == "manager" { return "Manager" }
|
||
if word == "engine" { return "Engine" }
|
||
if word == "accessor" { return "Accessor" }
|
||
if word == "vessel" { return "Vessel" }
|
||
if word == "extern" { return "Extern" }
|
||
if word == "break" { return "Break" }
|
||
if word == "continue" { return "Continue" }
|
||
""
|
||
}
|
||
|
||
// -- Scan helpers --------------------------------------------------------------
|
||
// All scan helpers receive the chars list and total length.
|
||
|
||
// scan_digits - advance i while chars[i] is a digit
|
||
// Returns { "text": ..., "pos": i }
|
||
fn scan_digits(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let c: Int = str_char_code(src, i)
|
||
if is_digit_code(c) {
|
||
let i = i + 1
|
||
} else {
|
||
let running = false
|
||
}
|
||
}
|
||
}
|
||
// Use str_slice instead of building a parts list — O(1) allocation, O(n) copy.
|
||
{ "text": str_slice(src, start, i), "pos": i }
|
||
}
|
||
|
||
// scan_ident - advance i while chars[i] is alphanumeric or underscore
|
||
fn scan_ident(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let c: Int = str_char_code(src, i)
|
||
if is_alnum_or_underscore_code(c) {
|
||
let i = i + 1
|
||
} else {
|
||
let running = false
|
||
}
|
||
}
|
||
}
|
||
// Use str_slice instead of building a parts list — O(1) allocation, O(n) copy.
|
||
{ "text": str_slice(src, start, i), "pos": i }
|
||
}
|
||
|
||
// -- Code-bearing string detection + comment strip ----------------------------
|
||
// Inline JS/CSS literals embedded in El source (e.g. <script>-</script> blobs
|
||
// or stylesheet payloads inside string literals) carry their own line and
|
||
// block comments. Those comments leak into the served HTML and reveal build
|
||
// notes the visitor should never see. We strip them at the lexer so every
|
||
// downstream consumer (codegen-c, codegen-js, parser) gets the cleaned form.
|
||
//
|
||
// looks_like_code - heuristic gate so we only strip strings that actually
|
||
// embed JS or CSS. Plain prose, hex blobs, JSON, etc. pass through verbatim.
|
||
|
||
fn substr_at(src: String, start: Int, total: Int, needle: String) -> Bool {
|
||
let nlen: Int = str_len(needle)
|
||
if start + nlen > total { return false }
|
||
// Use str_slice comparison instead of char-by-char loop.
|
||
str_eq(str_slice(src, start, start + nlen), needle)
|
||
}
|
||
|
||
fn str_has(s: String, needle: String) -> Bool {
|
||
// Use the built-in str_contains which is implemented in native C — O(n) single pass.
|
||
str_contains(s, needle)
|
||
}
|
||
|
||
fn looks_like_code(s: String) -> Bool {
|
||
if str_has(s, "<script") { return true }
|
||
if str_has(s, "<style") { return true }
|
||
if str_has(s, "function") {
|
||
if str_has(s, ";") { return true }
|
||
}
|
||
false
|
||
}
|
||
|
||
// strip_code_comments - character-by-character walk. Tracks JS string state
|
||
// (single, double, backtick) and never strips inside one. Backslash escapes
|
||
// inside JS strings consume the next char verbatim. URLs like https:// are
|
||
// preserved by checking the previous char before treating // as a line
|
||
// comment opener: if the char immediately before '/' is ':', emit the '/'
|
||
// literally and advance one position.
|
||
fn strip_code_comments(s: String) -> String {
|
||
let total: Int = str_len(s)
|
||
let out_parts: [String] = native_list_empty()
|
||
let i = 0
|
||
let in_squote = false
|
||
let in_dquote = false
|
||
let in_btick = false
|
||
let prev = ""
|
||
while i < total {
|
||
let ch: String = str_char_at(s, i)
|
||
let in_js_string = false
|
||
if in_squote { let in_js_string = true }
|
||
if in_dquote { let in_js_string = true }
|
||
if in_btick { let in_js_string = true }
|
||
|
||
if in_js_string {
|
||
// Backslash escape: consume next char verbatim regardless of which.
|
||
if ch == "\\" {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let nc: String = str_char_at(s, next_i)
|
||
let out_parts = native_list_append(out_parts, nc)
|
||
let prev = nc
|
||
let i = next_i + 1
|
||
} else {
|
||
let prev = ch
|
||
let i = next_i
|
||
}
|
||
} else {
|
||
if in_squote {
|
||
if ch == "'" { let in_squote = false }
|
||
} else {
|
||
if in_dquote {
|
||
if ch == "\"" { let in_dquote = false }
|
||
} else {
|
||
if in_btick {
|
||
if ch == "`" { let in_btick = false }
|
||
}
|
||
}
|
||
}
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
// Not in a JS string. Check for comment openers.
|
||
let next_i = i + 1
|
||
let next_ch = ""
|
||
if next_i < total {
|
||
let next_ch: String = str_char_at(s, next_i)
|
||
}
|
||
|
||
if ch == "/" {
|
||
if next_ch == "/" {
|
||
// URL guard: prev char ':' means this is "://", not a comment.
|
||
if prev == ":" {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
// Skip until newline (newline itself is preserved so
|
||
// surrounding line counts/structure stay sane).
|
||
let i = i + 2
|
||
let scanning = true
|
||
while scanning {
|
||
if i >= total {
|
||
let scanning = false
|
||
} else {
|
||
let lc: String = str_char_at(s, i)
|
||
if lc == "\n" {
|
||
let scanning = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
let prev = ""
|
||
}
|
||
} else {
|
||
if next_ch == "*" {
|
||
// Skip until matching "*/".
|
||
let i = i + 2
|
||
let scanning2 = true
|
||
while scanning2 {
|
||
if i >= total {
|
||
let scanning2 = false
|
||
} else {
|
||
let bc: String = str_char_at(s, i)
|
||
if bc == "*" {
|
||
let after = i + 1
|
||
if after < total {
|
||
let nc2: String = str_char_at(s, after)
|
||
if nc2 == "/" {
|
||
let i = after + 1
|
||
let scanning2 = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
let prev = ""
|
||
} else {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
}
|
||
}
|
||
} else {
|
||
// Open a JS string?
|
||
if ch == "'" {
|
||
let in_squote = true
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
if ch == "\"" {
|
||
let in_dquote = true
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
if ch == "`" {
|
||
let in_btick = true
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
str_join(out_parts, "")
|
||
}
|
||
|
||
// scan_string - scan a quoted string literal, handling \" escapes.
|
||
// Starts AFTER the opening quote. Returns { "text": content, "pos": i_after_close }
|
||
fn scan_string(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let parts: [String] = native_list_empty()
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let ch: String = str_char_at(src, i)
|
||
if ch == "\\" {
|
||
// escape: peek next char
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let next_ch: String = str_char_at(src, next_i)
|
||
if next_ch == "\"" {
|
||
let parts = native_list_append(parts, "\"")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "n" {
|
||
let parts = native_list_append(parts, "\n")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "t" {
|
||
let parts = native_list_append(parts, "\t")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "r" {
|
||
let parts = native_list_append(parts, "\r")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "\\" {
|
||
let parts = native_list_append(parts, "\\")
|
||
let i = next_i + 1
|
||
} else {
|
||
let parts = native_list_append(parts, next_ch)
|
||
let i = next_i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if ch == "\"" {
|
||
let i = i + 1
|
||
let running = false
|
||
} else {
|
||
let parts = native_list_append(parts, ch)
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
{ "text": str_join(parts, ""), "pos": i }
|
||
}
|
||
|
||
// -- String interpolation ------------------------------------------------------
|
||
//
|
||
// scan_interp_brace - scan from `start` (the char after `${`) to the matching
|
||
// `}`, tracking brace depth so inner braces (e.g. fn calls, map literals) are
|
||
// handled correctly. Returns { "text": inner_source, "pos": i_after_close }.
|
||
fn scan_interp_brace(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let depth = 1
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let ch: String = str_char_at(src, i)
|
||
if ch == "{" {
|
||
let depth = depth + 1
|
||
let i = i + 1
|
||
} else {
|
||
if ch == "}" {
|
||
let depth = depth - 1
|
||
if depth <= 0 {
|
||
// Closing brace of the interpolation - stop, do not include it
|
||
let i = i + 1
|
||
let running = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// Use str_slice instead of parts list — the inner source is a contiguous substring.
|
||
{ "text": str_slice(src, start, i - 1), "pos": i }
|
||
}
|
||
|
||
// interp_tokens_append_all - copy every (kind, value) pair from flat src list
|
||
// into flat dst list, skipping the trailing Eof pair that lex() always appends.
|
||
// Splices re-lexed interpolation tokens into the stream. This walks the token
|
||
// list DIRECTLY rather than through tok_append, so it carries its own copy of
|
||
// the stride -- which is why giving tokens a line broke the compiler's second
|
||
// generation and not its first: the compiler's own source uses string
|
||
// interpolation, so gen1 (built by the old compiler) was fine and gen2 emitted
|
||
// a corrupted stream.
|
||
fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] {
|
||
let src_len: Int = native_list_len(src)
|
||
let j = 0
|
||
let result = dst
|
||
while j < src_len {
|
||
let kind: String = native_list_get(src, j)
|
||
if kind == "Eof" {
|
||
let j = src_len
|
||
} else {
|
||
let val: String = native_list_get(src, j + 1)
|
||
let ln: String = native_list_get(src, j + 2)
|
||
let result = native_list_append(result, kind)
|
||
let result = native_list_append(result, val)
|
||
let result = native_list_append(result, ln)
|
||
let j = j + 3
|
||
}
|
||
}
|
||
result
|
||
}
|
||
|
||
// scan_interp_string - scan a string literal that may contain ${expr}
|
||
// interpolations. Starts AFTER the opening `"`.
|
||
// Returns { "tokens": [token list to inject], "pos": i_after_close_quote }.
|
||
//
|
||
// For a plain string (no ${}) this emits a single Str token, identical to the
|
||
// old scan_string path. For an interpolated string it emits a flat sequence
|
||
// of tokens equivalent to the string-concat expression, for example:
|
||
//
|
||
// "hello ${name}!"
|
||
// => Str("hello ") Plus <tokens for name> Plus Str("!")
|
||
//
|
||
// Empty literal segments between adjacent ${ } blocks are omitted. The
|
||
// resulting token stream is consumed by the existing parse_binop / parse_primary
|
||
// path in the parser with zero parser changes required.
|
||
//
|
||
// Supported escape sequences: \" \n \t \r \\ \$ (literal dollar sign).
|
||
// Nested quotes inside ${} are not supported; use a variable instead.
|
||
//
|
||
// Performance: uses str_char_code (Int) for all character dispatch, eliminating
|
||
// per-character strdup. Plain runs are batched into str_slice segments instead
|
||
// of accumulating single-char strings, reducing list appends from O(N) to O(K)
|
||
// where K = number of escape/special chars in the literal.
|
||
// Char codes: '\' = 92, '"' = 34, '$' = 36, '{' = 123
|
||
fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let out_tokens: [Any] = native_list_empty()
|
||
let cur_parts: [String] = native_list_empty()
|
||
let clean_start = start
|
||
let has_interp = false
|
||
let need_plus = false
|
||
let running = true
|
||
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let c: Int = str_char_code(src, i)
|
||
|
||
if c == 92 {
|
||
// '\\' = 92 — escape sequence: flush clean run, append resolved char
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let nc: Int = str_char_code(src, next_i)
|
||
if nc == 36 {
|
||
// '\$' => literal '$' (36 = '$')
|
||
let cur_parts = native_list_append(cur_parts, "$")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 34 {
|
||
// '\"' => literal '"' (34 = '"')
|
||
let cur_parts = native_list_append(cur_parts, "\"")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 110 {
|
||
// '\n' (110 = 'n')
|
||
let cur_parts = native_list_append(cur_parts, "\n")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 116 {
|
||
// '\t' (116 = 't')
|
||
let cur_parts = native_list_append(cur_parts, "\t")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 114 {
|
||
// '\r' (114 = 'r')
|
||
let cur_parts = native_list_append(cur_parts, "\r")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 92 {
|
||
// '\\' (92)
|
||
let cur_parts = native_list_append(cur_parts, "\\")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
// Unknown escape: emit the escaped char verbatim
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, next_i, next_i + 1))
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
let clean_start = next_i
|
||
let i = next_i
|
||
}
|
||
} else {
|
||
if c == 34 {
|
||
// '"' = 34 — closing quote: flush clean run, stop
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let i = i + 1
|
||
let clean_start = i
|
||
let running = false
|
||
} else {
|
||
if c == 36 {
|
||
// '$' = 36 — possible interpolation start
|
||
let next_i = i + 1
|
||
let is_interp = false
|
||
if next_i < total {
|
||
let nc2: Int = str_char_code(src, next_i)
|
||
if nc2 == 123 {
|
||
// '{' = 123
|
||
let is_interp = true
|
||
}
|
||
}
|
||
if is_interp {
|
||
// Flush the accumulated literal part (if non-empty)
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let part_len: Int = native_list_len(cur_parts)
|
||
if part_len > 0 {
|
||
let part_text = str_join(cur_parts, "")
|
||
if need_plus {
|
||
let out_tokens = tok_append(out_tokens, "Plus", "+")
|
||
}
|
||
let clean_part = part_text
|
||
if looks_like_code(part_text) {
|
||
let clean_part = strip_code_comments(part_text)
|
||
}
|
||
let out_tokens = tok_append(out_tokens, "Str", clean_part)
|
||
let need_plus = true
|
||
}
|
||
let cur_parts = native_list_empty()
|
||
let has_interp = true
|
||
|
||
// Scan brace-balanced expression source
|
||
let brace_result = scan_interp_brace(src, next_i + 1, total)
|
||
let expr_src: String = brace_result["text"]
|
||
let new_i: Int = brace_result["pos"]
|
||
let i = new_i
|
||
let clean_start = new_i
|
||
|
||
// Re-lex the expression and inline the tokens.
|
||
// Wrap in ( ) so that operators inside ${} (e.g.
|
||
// age + 1) are parsed as a grouped sub-expression
|
||
// rather than merging with the surrounding concat
|
||
// Plus tokens at the wrong precedence level.
|
||
let inner_toks: [Any] = lex(expr_src)
|
||
let inner_len: Int = native_list_len(inner_toks)
|
||
|
||
if need_plus {
|
||
let out_tokens = tok_append(out_tokens, "Plus", "+")
|
||
}
|
||
// Empty interpolation ${} => empty string segment
|
||
// inner_len <= 2 = only the Eof pair (kind="Eof", value="")
|
||
if inner_len <= 2 {
|
||
let out_tokens = tok_append(out_tokens, "Str", "")
|
||
} else {
|
||
let out_tokens = tok_append(out_tokens, "LParen", "(")
|
||
let out_tokens = interp_tokens_append_all(out_tokens, inner_toks)
|
||
let out_tokens = tok_append(out_tokens, "RParen", ")")
|
||
}
|
||
let need_plus = true
|
||
} else {
|
||
// Plain '$' not followed by '{' - treat as literal, continue clean run
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
// Plain char — extends clean run, no append needed
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Flush remaining literal segment and build final token list
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let part_len: Int = native_list_len(cur_parts)
|
||
let part_text = str_join(cur_parts, "")
|
||
if has_interp {
|
||
// Interpolated string: only emit trailing segment if non-empty
|
||
if part_len > 0 {
|
||
let clean_part = part_text
|
||
if looks_like_code(part_text) {
|
||
let clean_part = strip_code_comments(part_text)
|
||
}
|
||
if need_plus {
|
||
let out_tokens = tok_append(out_tokens, "Plus", "+")
|
||
}
|
||
let out_tokens = tok_append(out_tokens, "Str", clean_part)
|
||
}
|
||
} else {
|
||
// Plain string with no interpolation - same behaviour as old scan_string
|
||
let clean_text = part_text
|
||
if looks_like_code(part_text) {
|
||
let clean_text = strip_code_comments(part_text)
|
||
}
|
||
let out_tokens = tok_append(out_tokens, "Str", clean_text)
|
||
}
|
||
|
||
{ "tokens": out_tokens, "pos": i }
|
||
}
|
||
|
||
// -- Main lexer ----------------------------------------------------------------
|
||
// Char code constants (avoids strdup for single-char comparison)
|
||
// '/' = 47, '"' = 34, '0'-'9' = 48-57, 'a'-'z' = 97-122, 'A'-'Z' = 65-90
|
||
// '_' = 95, ' '=32, '\t'=9, '\n'=10, '\r'=13
|
||
// '=' = 61, '!' = 33, '<' = 60, '>' = 62, '&' = 38, '|' = 124
|
||
// '-' = 45, ':' = 58, '+' = 43, '*' = 42, '%' = 37
|
||
// '(' = 40, ')' = 41, '{' = 123, '}' = 125, '[' = 91, ']' = 93
|
||
// ',' = 44, '.' = 46, ';' = 59, '@' = 64, '?' = 63
|
||
|
||
fn lex(source: String) -> [Any] {
|
||
// Use str_char_code (returns Int) instead of str_char_at (returns strdup String)
|
||
// for all character classification in the hot loop. For a 400KB source,
|
||
// str_char_at allocates ~400K × 16B = ~6.4MB of temporary strings.
|
||
let total: Int = str_len(source)
|
||
let tokens: [Any] = native_list_empty()
|
||
let i: Int = 0
|
||
state_set("__lex_line", "1")
|
||
let line_no: Int = 1
|
||
|
||
while i < total {
|
||
if str_eq(str_slice(source, i, i + 1), "\n") {
|
||
let line_no = line_no + 1
|
||
state_set("__lex_line", native_int_to_str(line_no))
|
||
}
|
||
let c: Int = str_char_code(source, i)
|
||
|
||
// Skip whitespace (space=32, tab=9, newline=10, CR=13)
|
||
if is_ws_code(c) {
|
||
let i = i + 1
|
||
} else {
|
||
// Line comments: // (slash=47)
|
||
if c == 47 {
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let nc: Int = str_char_code(source, next_i)
|
||
if nc == 47 {
|
||
// skip to end of line (newline=10)
|
||
let i = i + 2
|
||
let running2 = true
|
||
while running2 {
|
||
if i >= total {
|
||
let running2 = false
|
||
} else {
|
||
let lc: Int = str_char_code(source, i)
|
||
if lc == 10 {
|
||
let running2 = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Slash", "/")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Slash", "/")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
// String literal: '"' = 34
|
||
if c == 34 {
|
||
let interp_result = scan_interp_string(source, i + 1, total)
|
||
let interp_toks: [Any] = interp_result["tokens"]
|
||
let new_pos: Int = interp_result["pos"]
|
||
let tokens = interp_tokens_append_all(tokens, interp_toks)
|
||
let i = new_pos
|
||
} else {
|
||
// Number literal: '0'-'9' = 48-57
|
||
if is_digit_code(c) {
|
||
let result = scan_digits(source, i, total)
|
||
let num_text: String = result["text"]
|
||
let new_pos: Int = result["pos"]
|
||
// check for float (dot=46 followed by digit)
|
||
if new_pos < total {
|
||
let dc: Int = str_char_code(source, new_pos)
|
||
if dc == 46 {
|
||
let after_dot = new_pos + 1
|
||
if after_dot < total {
|
||
let adc: Int = str_char_code(source, after_dot)
|
||
if is_digit_code(adc) {
|
||
let frac_result = scan_digits(source, after_dot, total)
|
||
let frac_text: String = frac_result["text"]
|
||
let frac_pos: Int = frac_result["pos"]
|
||
let tokens = tok_append(tokens, "Float", num_text + "." + frac_text)
|
||
let i = frac_pos
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
// Identifier or keyword: alpha or '_'=95
|
||
if is_alpha_code(c) || c == 95 {
|
||
let result = scan_ident(source, i, total)
|
||
let word: String = result["text"]
|
||
let new_pos: Int = result["pos"]
|
||
let kw = keyword_kind(word)
|
||
if kw == "" {
|
||
let tokens = tok_append(tokens, "Ident", word)
|
||
} else {
|
||
let tokens = tok_append(tokens, kw, word)
|
||
}
|
||
let i = new_pos
|
||
} else {
|
||
// Multi-char and single-char operators/delimiters
|
||
let peek_i = i + 1
|
||
let peek_c: Int = -1
|
||
if peek_i < total {
|
||
let peek_c: Int = str_char_code(source, peek_i)
|
||
}
|
||
|
||
if c == 61 {
|
||
// '=' = 61
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "EqEq", "==")
|
||
let i = i + 2
|
||
} else {
|
||
if peek_c == 62 {
|
||
// '>' = 62
|
||
let tokens = tok_append(tokens, "FatArrow", "=>")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Eq", "=")
|
||
let i = i + 1
|
||
}
|
||
}
|
||
} else {
|
||
if c == 33 {
|
||
// '!' = 33
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "NotEq", "!=")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Not", "!")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 60 {
|
||
// '<' = 60
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "LtEq", "<=")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Lt", "<")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 62 {
|
||
// '>' = 62
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "GtEq", ">=")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Gt", ">")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 38 {
|
||
// '&' = 38
|
||
if peek_c == 38 {
|
||
let tokens = tok_append(tokens, "And", "&&")
|
||
let i = i + 2
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 124 {
|
||
// '|' = 124
|
||
if peek_c == 124 {
|
||
let tokens = tok_append(tokens, "Or", "||")
|
||
let i = i + 2
|
||
} else {
|
||
if peek_c == 62 {
|
||
// '>' = 62
|
||
let tokens = tok_append(tokens, "PipeOp", "|>")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Pipe", "|")
|
||
let i = i + 1
|
||
}
|
||
}
|
||
} else {
|
||
if c == 45 {
|
||
// '-' = 45
|
||
if peek_c == 62 {
|
||
// '>' = 62
|
||
let tokens = tok_append(tokens, "Arrow", "->")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Minus", "-")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 58 {
|
||
// ':' = 58
|
||
if peek_c == 58 {
|
||
let tokens = tok_append(tokens, "ColonColon", "::")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Colon", ":")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 43 {
|
||
// '+' = 43
|
||
let tokens = tok_append(tokens, "Plus", "+")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 42 {
|
||
// '*' = 42
|
||
let tokens = tok_append(tokens, "Star", "*")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 37 {
|
||
// '%' = 37
|
||
let tokens = tok_append(tokens, "Percent", "%")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 40 {
|
||
// '(' = 40
|
||
let tokens = tok_append(tokens, "LParen", "(")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 41 {
|
||
// ')' = 41
|
||
let tokens = tok_append(tokens, "RParen", ")")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 123 {
|
||
// '{' = 123
|
||
let tokens = tok_append(tokens, "LBrace", "{")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 125 {
|
||
// '}' = 125
|
||
let tokens = tok_append(tokens, "RBrace", "}")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 91 {
|
||
// '[' = 91
|
||
let tokens = tok_append(tokens, "LBracket", "[")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 93 {
|
||
// ']' = 93
|
||
let tokens = tok_append(tokens, "RBracket", "]")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 44 {
|
||
// ',' = 44
|
||
let tokens = tok_append(tokens, "Comma", ",")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 46 {
|
||
// '.' = 46: check for ..= or ..
|
||
let peek2_i = i + 2
|
||
let peek2_c: Int = -1
|
||
if peek2_i < total {
|
||
let peek2_c: Int = str_char_code(source, peek2_i)
|
||
}
|
||
if peek_c == 46 {
|
||
// '..' prefix
|
||
if peek2_c == 61 {
|
||
// '..=' = 46 46 61
|
||
let tokens = tok_append(tokens, "DotDotEq", "..=")
|
||
let i = i + 3
|
||
} else {
|
||
let tokens = tok_append(tokens, "DotDot", "..")
|
||
let i = i + 2
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Dot", ".")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 59 {
|
||
// ';' = 59
|
||
let tokens = tok_append(tokens, "Semicolon", ";")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 64 {
|
||
// '@' = 64
|
||
let tokens = tok_append(tokens, "At", "@")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 63 {
|
||
// '?' = 63
|
||
let tokens = tok_append(tokens, "QuestionMark", "?")
|
||
let i = i + 1
|
||
} else {
|
||
// unknown char - skip
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
let tokens = tok_append(tokens, "Eof", "")
|
||
tokens
|
||
}
|