Archived
bigmerge
0143cc458a
Both, at different layers -- and it is the same split as serialization: the
convention is the BASIS, never the ACT.
lexeme -> token `fn` means function-start because someone said so CONVENTION
shape recognition given tokens, which construct is this REGION
source -> structure parsing is transduction onto that basis GEOMETRY
byte traversal something must read them in order IRREDUCIBLE
Three things push the ACT toward region rather than convention: ambiguity
(a * b needs context; a grammar resolves it with the lexer hack, a region by
neighbourhood), error recovery (nearest-region is free), and precedence, which
is ordering along an axis with a conventional parameter.
AND THE SHOULD GATE SAYS NO TO THE OBVIOUS MOVE
Every other table this session moved to data. This one stays code. The keyword
set is CLOSED by the language definition -- it does not leak the way an
allowlist does -- and the lexer runs before the program is understood, so a
program can never declare its own keywords. Externalising it costs file I/O on
every compile and buys nothing. Same verdict as is_digit in ASCII.
WHAT WAS ACTUALLY WRONG: five of 46 keywords were consumed by no parser or
codegen path. sealed, activate, seed, protocol, impl. Each stole an identifier
from users for nothing.
SECOND SILENT MISCOMPILATION OF THE DAY. Using one did not fail to parse:
let seed = 42
let impl = seed + 1
compiled CLEAN -- zero cc errors -- and printed 0 instead of 44. No diagnostic
at any layer. Fixed by removing the five.
A DEFECT IN MY OWN MEASUREMENT, caught before it did damage: my first pass
checked only parser.el and reported `test` as inert too. codegen consumes it at
4135 for --test mode, and the tree has 408 uses. Removing it would have broken
every test in the suite. The measurement was re-run across all four consumers.
100/100 native + 2 new, 31/31 integration, fixpoint ok.
1092 lines
53 KiB
EmacsLisp
1092 lines
53 KiB
EmacsLisp
// lexer.el - el self-hosting lexer
|
||
//
|
||
// Tokenises an el source string into a list of token maps.
|
||
// Each token is a Map<String, Any> with keys:
|
||
// "kind" -> String (e.g. "Int", "Ident", "Plus")
|
||
// "value" -> String (the raw text of the token)
|
||
//
|
||
// Entry point: fn lex(source: String) -> [Map<String, Any>]
|
||
//
|
||
// Performance: the hot lexer loop uses str_char_code (returns Int) instead of
|
||
// str_char_at (returns strdup'd String) for character classification.
|
||
// For a 400KB source, str_char_at allocates ~400K × 16B = ~6.4MB of temporary
|
||
// strings for the `ch` variable alone. str_char_code avoids all that.
|
||
|
||
// -- Character helpers (Int-based, no string allocation) ----------------------
|
||
// These operate on char codes (from str_char_code) instead of str_char_at,
|
||
// eliminating one strdup per character in the hot lexer loop.
|
||
|
||
fn is_digit_code(c: Int) -> Bool {
|
||
// '0'=48 .. '9'=57
|
||
if c >= 48 {
|
||
if c <= 57 { return true }
|
||
}
|
||
false
|
||
}
|
||
|
||
fn is_alpha_code(c: Int) -> Bool {
|
||
// 'A'=65..'Z'=90, 'a'=97..'z'=122
|
||
if c >= 65 {
|
||
if c <= 90 { return true }
|
||
}
|
||
if c >= 97 {
|
||
if c <= 122 { return true }
|
||
}
|
||
false
|
||
}
|
||
|
||
fn is_alnum_or_underscore_code(c: Int) -> Bool {
|
||
if is_digit_code(c) { return true }
|
||
if is_alpha_code(c) { return true }
|
||
if c == 95 { return true } // '_'
|
||
false
|
||
}
|
||
|
||
fn is_ws_code(c: Int) -> Bool {
|
||
if c == 32 { return true } // ' '
|
||
if c == 9 { return true } // '\t'
|
||
if c == 10 { return true } // '\n'
|
||
if c == 13 { return true } // '\r'
|
||
false
|
||
}
|
||
|
||
// Legacy String-based helpers kept for scan_interp helpers that use str_char_at.
|
||
fn lex_is_digit(ch: String) -> Bool {
|
||
if ch == "0" { return true }
|
||
if ch == "1" { return true }
|
||
if ch == "2" { return true }
|
||
if ch == "3" { return true }
|
||
if ch == "4" { return true }
|
||
if ch == "5" { return true }
|
||
if ch == "6" { return true }
|
||
if ch == "7" { return true }
|
||
if ch == "8" { return true }
|
||
if ch == "9" { return true }
|
||
false
|
||
}
|
||
|
||
fn lex_is_alpha(ch: String) -> Bool {
|
||
if ch == "a" { return true }
|
||
if ch == "b" { return true }
|
||
if ch == "c" { return true }
|
||
if ch == "d" { return true }
|
||
if ch == "e" { return true }
|
||
if ch == "f" { return true }
|
||
if ch == "g" { return true }
|
||
if ch == "h" { return true }
|
||
if ch == "i" { return true }
|
||
if ch == "j" { return true }
|
||
if ch == "k" { return true }
|
||
if ch == "l" { return true }
|
||
if ch == "m" { return true }
|
||
if ch == "n" { return true }
|
||
if ch == "o" { return true }
|
||
if ch == "p" { return true }
|
||
if ch == "q" { return true }
|
||
if ch == "r" { return true }
|
||
if ch == "s" { return true }
|
||
if ch == "t" { return true }
|
||
if ch == "u" { return true }
|
||
if ch == "v" { return true }
|
||
if ch == "w" { return true }
|
||
if ch == "x" { return true }
|
||
if ch == "y" { return true }
|
||
if ch == "z" { return true }
|
||
if ch == "A" { return true }
|
||
if ch == "B" { return true }
|
||
if ch == "C" { return true }
|
||
if ch == "D" { return true }
|
||
if ch == "E" { return true }
|
||
if ch == "F" { return true }
|
||
if ch == "G" { return true }
|
||
if ch == "H" { return true }
|
||
if ch == "I" { return true }
|
||
if ch == "J" { return true }
|
||
if ch == "K" { return true }
|
||
if ch == "L" { return true }
|
||
if ch == "M" { return true }
|
||
if ch == "N" { return true }
|
||
if ch == "O" { return true }
|
||
if ch == "P" { return true }
|
||
if ch == "Q" { return true }
|
||
if ch == "R" { return true }
|
||
if ch == "S" { return true }
|
||
if ch == "T" { return true }
|
||
if ch == "U" { return true }
|
||
if ch == "V" { return true }
|
||
if ch == "W" { return true }
|
||
if ch == "X" { return true }
|
||
if ch == "Y" { return true }
|
||
if ch == "Z" { return true }
|
||
false
|
||
}
|
||
|
||
fn is_alnum_or_underscore(ch: String) -> Bool {
|
||
if lex_is_digit(ch) { return true }
|
||
if lex_is_alpha(ch) { return true }
|
||
if ch == "_" { return true }
|
||
false
|
||
}
|
||
|
||
fn lex_is_whitespace(ch: String) -> Bool {
|
||
if ch == " " { return true }
|
||
if ch == "\t" { return true }
|
||
if ch == "\n" { return true }
|
||
if ch == "\r" { return true }
|
||
false
|
||
}
|
||
|
||
// tok_append — append a (kind, value) pair to a flat token list.
|
||
// Returns the updated list. Gamma combines flat-list + char-code for max savings.
|
||
fn tok_append(tokens: [Any], kind: String, value: String) -> [Any] {
|
||
let tokens = native_list_append(tokens, kind)
|
||
native_list_append(tokens, value)
|
||
}
|
||
|
||
// -- Keyword lookup ------------------------------------------------------------
|
||
|
||
// keyword_kind — the language's reserved spellings.
|
||
//
|
||
// A grammar is a BASIS: `fn` means function-start because someone said so, and
|
||
// nothing derives it. But unlike the other tables moved out this session, this
|
||
// one stays code, and the SHOULD gate is why. The keyword set is closed by the
|
||
// language definition -- it does not leak the way an allowlist does -- and the
|
||
// lexer runs before the program is understood, so a program can never declare
|
||
// its own keywords. Externalising it would cost file I/O on every compile and
|
||
// buy nothing.
|
||
//
|
||
// Removed 2026-08-17: sealed, activate, seed, protocol, impl. Reserved in the
|
||
// lexer, consumed by no parser or codegen path, and each one stole an
|
||
// identifier from users for nothing. `test` LOOKED inert by the same measure
|
||
// and is not -- codegen consumes it at 4135 for --test mode, 408 uses in the
|
||
// tree. The first measurement checked only parser.el and would have broken all
|
||
// of them.
|
||
fn keyword_kind(word: String) -> String {
|
||
if word == "let" { return "Let" }
|
||
if word == "fn" { return "Fn" }
|
||
if word == "type" { return "Type" }
|
||
if word == "enum" { return "Enum" }
|
||
if word == "match" { return "Match" }
|
||
if word == "return" { return "Return" }
|
||
if word == "if" { return "If" }
|
||
if word == "else" { return "Else" }
|
||
if word == "for" { return "For" }
|
||
if word == "in" { return "In" }
|
||
if word == "while" { return "While" }
|
||
if word == "import" { return "Import" }
|
||
if word == "from" { return "From" }
|
||
if word == "as" { return "As" }
|
||
if word == "with" { return "With" }
|
||
if word == "where" { return "Where" }
|
||
if word == "test" { return "Test" }
|
||
if word == "assert" { return "Assert" }
|
||
if word == "retry" { return "Retry" }
|
||
if word == "times" { return "Times" }
|
||
if word == "fallback" { return "Fallback" }
|
||
if word == "reason" { return "Reason" }
|
||
if word == "parallel" { return "Parallel" }
|
||
if word == "trace" { return "Trace" }
|
||
if word == "requires" { return "Requires" }
|
||
if word == "deploy" { return "Deploy" }
|
||
if word == "to" { return "To" }
|
||
if word == "via" { return "Via" }
|
||
if word == "target" { return "Target" }
|
||
if word == "true" { return "Bool" }
|
||
if word == "false" { return "Bool" }
|
||
if word == "cgi" { return "Cgi" }
|
||
if word == "service" { return "Service" }
|
||
if word == "program" { return "Program" }
|
||
if word == "manager" { return "Manager" }
|
||
if word == "engine" { return "Engine" }
|
||
if word == "accessor" { return "Accessor" }
|
||
if word == "vessel" { return "Vessel" }
|
||
if word == "extern" { return "Extern" }
|
||
if word == "break" { return "Break" }
|
||
if word == "continue" { return "Continue" }
|
||
""
|
||
}
|
||
|
||
// -- Scan helpers --------------------------------------------------------------
|
||
// All scan helpers receive the chars list and total length.
|
||
|
||
// scan_digits - advance i while chars[i] is a digit
|
||
// Returns { "text": ..., "pos": i }
|
||
fn scan_digits(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let c: Int = str_char_code(src, i)
|
||
if is_digit_code(c) {
|
||
let i = i + 1
|
||
} else {
|
||
let running = false
|
||
}
|
||
}
|
||
}
|
||
// Use str_slice instead of building a parts list — O(1) allocation, O(n) copy.
|
||
{ "text": str_slice(src, start, i), "pos": i }
|
||
}
|
||
|
||
// scan_ident - advance i while chars[i] is alphanumeric or underscore
|
||
fn scan_ident(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let c: Int = str_char_code(src, i)
|
||
if is_alnum_or_underscore_code(c) {
|
||
let i = i + 1
|
||
} else {
|
||
let running = false
|
||
}
|
||
}
|
||
}
|
||
// Use str_slice instead of building a parts list — O(1) allocation, O(n) copy.
|
||
{ "text": str_slice(src, start, i), "pos": i }
|
||
}
|
||
|
||
// -- Code-bearing string detection + comment strip ----------------------------
|
||
// Inline JS/CSS literals embedded in El source (e.g. <script>-</script> blobs
|
||
// or stylesheet payloads inside string literals) carry their own line and
|
||
// block comments. Those comments leak into the served HTML and reveal build
|
||
// notes the visitor should never see. We strip them at the lexer so every
|
||
// downstream consumer (codegen-c, codegen-js, parser) gets the cleaned form.
|
||
//
|
||
// looks_like_code - heuristic gate so we only strip strings that actually
|
||
// embed JS or CSS. Plain prose, hex blobs, JSON, etc. pass through verbatim.
|
||
|
||
fn substr_at(src: String, start: Int, total: Int, needle: String) -> Bool {
|
||
let nlen: Int = str_len(needle)
|
||
if start + nlen > total { return false }
|
||
// Use str_slice comparison instead of char-by-char loop.
|
||
str_eq(str_slice(src, start, start + nlen), needle)
|
||
}
|
||
|
||
fn str_has(s: String, needle: String) -> Bool {
|
||
// Use the built-in str_contains which is implemented in native C — O(n) single pass.
|
||
str_contains(s, needle)
|
||
}
|
||
|
||
fn looks_like_code(s: String) -> Bool {
|
||
if str_has(s, "<script") { return true }
|
||
if str_has(s, "<style") { return true }
|
||
if str_has(s, "function") {
|
||
if str_has(s, ";") { return true }
|
||
}
|
||
false
|
||
}
|
||
|
||
// strip_code_comments - character-by-character walk. Tracks JS string state
|
||
// (single, double, backtick) and never strips inside one. Backslash escapes
|
||
// inside JS strings consume the next char verbatim. URLs like https:// are
|
||
// preserved by checking the previous char before treating // as a line
|
||
// comment opener: if the char immediately before '/' is ':', emit the '/'
|
||
// literally and advance one position.
|
||
fn strip_code_comments(s: String) -> String {
|
||
let total: Int = str_len(s)
|
||
let out_parts: [String] = native_list_empty()
|
||
let i = 0
|
||
let in_squote = false
|
||
let in_dquote = false
|
||
let in_btick = false
|
||
let prev = ""
|
||
while i < total {
|
||
let ch: String = str_char_at(s, i)
|
||
let in_js_string = false
|
||
if in_squote { let in_js_string = true }
|
||
if in_dquote { let in_js_string = true }
|
||
if in_btick { let in_js_string = true }
|
||
|
||
if in_js_string {
|
||
// Backslash escape: consume next char verbatim regardless of which.
|
||
if ch == "\\" {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let nc: String = str_char_at(s, next_i)
|
||
let out_parts = native_list_append(out_parts, nc)
|
||
let prev = nc
|
||
let i = next_i + 1
|
||
} else {
|
||
let prev = ch
|
||
let i = next_i
|
||
}
|
||
} else {
|
||
if in_squote {
|
||
if ch == "'" { let in_squote = false }
|
||
} else {
|
||
if in_dquote {
|
||
if ch == "\"" { let in_dquote = false }
|
||
} else {
|
||
if in_btick {
|
||
if ch == "`" { let in_btick = false }
|
||
}
|
||
}
|
||
}
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
// Not in a JS string. Check for comment openers.
|
||
let next_i = i + 1
|
||
let next_ch = ""
|
||
if next_i < total {
|
||
let next_ch: String = str_char_at(s, next_i)
|
||
}
|
||
|
||
if ch == "/" {
|
||
if next_ch == "/" {
|
||
// URL guard: prev char ':' means this is "://", not a comment.
|
||
if prev == ":" {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
// Skip until newline (newline itself is preserved so
|
||
// surrounding line counts/structure stay sane).
|
||
let i = i + 2
|
||
let scanning = true
|
||
while scanning {
|
||
if i >= total {
|
||
let scanning = false
|
||
} else {
|
||
let lc: String = str_char_at(s, i)
|
||
if lc == "\n" {
|
||
let scanning = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
let prev = ""
|
||
}
|
||
} else {
|
||
if next_ch == "*" {
|
||
// Skip until matching "*/".
|
||
let i = i + 2
|
||
let scanning2 = true
|
||
while scanning2 {
|
||
if i >= total {
|
||
let scanning2 = false
|
||
} else {
|
||
let bc: String = str_char_at(s, i)
|
||
if bc == "*" {
|
||
let after = i + 1
|
||
if after < total {
|
||
let nc2: String = str_char_at(s, after)
|
||
if nc2 == "/" {
|
||
let i = after + 1
|
||
let scanning2 = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
let prev = ""
|
||
} else {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
}
|
||
}
|
||
} else {
|
||
// Open a JS string?
|
||
if ch == "'" {
|
||
let in_squote = true
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
if ch == "\"" {
|
||
let in_dquote = true
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
if ch == "`" {
|
||
let in_btick = true
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
str_join(out_parts, "")
|
||
}
|
||
|
||
// scan_string - scan a quoted string literal, handling \" escapes.
|
||
// Starts AFTER the opening quote. Returns { "text": content, "pos": i_after_close }
|
||
fn scan_string(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let parts: [String] = native_list_empty()
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let ch: String = str_char_at(src, i)
|
||
if ch == "\\" {
|
||
// escape: peek next char
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let next_ch: String = str_char_at(src, next_i)
|
||
if next_ch == "\"" {
|
||
let parts = native_list_append(parts, "\"")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "n" {
|
||
let parts = native_list_append(parts, "\n")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "t" {
|
||
let parts = native_list_append(parts, "\t")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "r" {
|
||
let parts = native_list_append(parts, "\r")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "\\" {
|
||
let parts = native_list_append(parts, "\\")
|
||
let i = next_i + 1
|
||
} else {
|
||
let parts = native_list_append(parts, next_ch)
|
||
let i = next_i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if ch == "\"" {
|
||
let i = i + 1
|
||
let running = false
|
||
} else {
|
||
let parts = native_list_append(parts, ch)
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
{ "text": str_join(parts, ""), "pos": i }
|
||
}
|
||
|
||
// -- String interpolation ------------------------------------------------------
|
||
//
|
||
// scan_interp_brace - scan from `start` (the char after `${`) to the matching
|
||
// `}`, tracking brace depth so inner braces (e.g. fn calls, map literals) are
|
||
// handled correctly. Returns { "text": inner_source, "pos": i_after_close }.
|
||
fn scan_interp_brace(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let depth = 1
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let ch: String = str_char_at(src, i)
|
||
if ch == "{" {
|
||
let depth = depth + 1
|
||
let i = i + 1
|
||
} else {
|
||
if ch == "}" {
|
||
let depth = depth - 1
|
||
if depth <= 0 {
|
||
// Closing brace of the interpolation - stop, do not include it
|
||
let i = i + 1
|
||
let running = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// Use str_slice instead of parts list — the inner source is a contiguous substring.
|
||
{ "text": str_slice(src, start, i - 1), "pos": i }
|
||
}
|
||
|
||
// interp_tokens_append_all - copy every (kind, value) pair from flat src list
|
||
// into flat dst list, skipping the trailing Eof pair that lex() always appends.
|
||
fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] {
|
||
let src_len: Int = native_list_len(src)
|
||
let j = 0
|
||
let result = dst
|
||
while j < src_len {
|
||
let kind: String = native_list_get(src, j)
|
||
if kind == "Eof" {
|
||
let j = src_len
|
||
} else {
|
||
let val: String = native_list_get(src, j + 1)
|
||
let result = native_list_append(result, kind)
|
||
let result = native_list_append(result, val)
|
||
let j = j + 2
|
||
}
|
||
}
|
||
result
|
||
}
|
||
|
||
// scan_interp_string - scan a string literal that may contain ${expr}
|
||
// interpolations. Starts AFTER the opening `"`.
|
||
// Returns { "tokens": [token list to inject], "pos": i_after_close_quote }.
|
||
//
|
||
// For a plain string (no ${}) this emits a single Str token, identical to the
|
||
// old scan_string path. For an interpolated string it emits a flat sequence
|
||
// of tokens equivalent to the string-concat expression, for example:
|
||
//
|
||
// "hello ${name}!"
|
||
// => Str("hello ") Plus <tokens for name> Plus Str("!")
|
||
//
|
||
// Empty literal segments between adjacent ${ } blocks are omitted. The
|
||
// resulting token stream is consumed by the existing parse_binop / parse_primary
|
||
// path in the parser with zero parser changes required.
|
||
//
|
||
// Supported escape sequences: \" \n \t \r \\ \$ (literal dollar sign).
|
||
// Nested quotes inside ${} are not supported; use a variable instead.
|
||
//
|
||
// Performance: uses str_char_code (Int) for all character dispatch, eliminating
|
||
// per-character strdup. Plain runs are batched into str_slice segments instead
|
||
// of accumulating single-char strings, reducing list appends from O(N) to O(K)
|
||
// where K = number of escape/special chars in the literal.
|
||
// Char codes: '\' = 92, '"' = 34, '$' = 36, '{' = 123
|
||
fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let out_tokens: [Any] = native_list_empty()
|
||
let cur_parts: [String] = native_list_empty()
|
||
let clean_start = start
|
||
let has_interp = false
|
||
let need_plus = false
|
||
let running = true
|
||
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let c: Int = str_char_code(src, i)
|
||
|
||
if c == 92 {
|
||
// '\\' = 92 — escape sequence: flush clean run, append resolved char
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let nc: Int = str_char_code(src, next_i)
|
||
if nc == 36 {
|
||
// '\$' => literal '$' (36 = '$')
|
||
let cur_parts = native_list_append(cur_parts, "$")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 34 {
|
||
// '\"' => literal '"' (34 = '"')
|
||
let cur_parts = native_list_append(cur_parts, "\"")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 110 {
|
||
// '\n' (110 = 'n')
|
||
let cur_parts = native_list_append(cur_parts, "\n")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 116 {
|
||
// '\t' (116 = 't')
|
||
let cur_parts = native_list_append(cur_parts, "\t")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 114 {
|
||
// '\r' (114 = 'r')
|
||
let cur_parts = native_list_append(cur_parts, "\r")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 92 {
|
||
// '\\' (92)
|
||
let cur_parts = native_list_append(cur_parts, "\\")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
// Unknown escape: emit the escaped char verbatim
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, next_i, next_i + 1))
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
let clean_start = next_i
|
||
let i = next_i
|
||
}
|
||
} else {
|
||
if c == 34 {
|
||
// '"' = 34 — closing quote: flush clean run, stop
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let i = i + 1
|
||
let clean_start = i
|
||
let running = false
|
||
} else {
|
||
if c == 36 {
|
||
// '$' = 36 — possible interpolation start
|
||
let next_i = i + 1
|
||
let is_interp = false
|
||
if next_i < total {
|
||
let nc2: Int = str_char_code(src, next_i)
|
||
if nc2 == 123 {
|
||
// '{' = 123
|
||
let is_interp = true
|
||
}
|
||
}
|
||
if is_interp {
|
||
// Flush the accumulated literal part (if non-empty)
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let part_len: Int = native_list_len(cur_parts)
|
||
if part_len > 0 {
|
||
let part_text = str_join(cur_parts, "")
|
||
if need_plus {
|
||
let out_tokens = tok_append(out_tokens, "Plus", "+")
|
||
}
|
||
let clean_part = part_text
|
||
if looks_like_code(part_text) {
|
||
let clean_part = strip_code_comments(part_text)
|
||
}
|
||
let out_tokens = tok_append(out_tokens, "Str", clean_part)
|
||
let need_plus = true
|
||
}
|
||
let cur_parts = native_list_empty()
|
||
let has_interp = true
|
||
|
||
// Scan brace-balanced expression source
|
||
let brace_result = scan_interp_brace(src, next_i + 1, total)
|
||
let expr_src: String = brace_result["text"]
|
||
let new_i: Int = brace_result["pos"]
|
||
let i = new_i
|
||
let clean_start = new_i
|
||
|
||
// Re-lex the expression and inline the tokens.
|
||
// Wrap in ( ) so that operators inside ${} (e.g.
|
||
// age + 1) are parsed as a grouped sub-expression
|
||
// rather than merging with the surrounding concat
|
||
// Plus tokens at the wrong precedence level.
|
||
let inner_toks: [Any] = lex(expr_src)
|
||
let inner_len: Int = native_list_len(inner_toks)
|
||
|
||
if need_plus {
|
||
let out_tokens = tok_append(out_tokens, "Plus", "+")
|
||
}
|
||
// Empty interpolation ${} => empty string segment
|
||
// inner_len <= 2 = only the Eof pair (kind="Eof", value="")
|
||
if inner_len <= 2 {
|
||
let out_tokens = tok_append(out_tokens, "Str", "")
|
||
} else {
|
||
let out_tokens = tok_append(out_tokens, "LParen", "(")
|
||
let out_tokens = interp_tokens_append_all(out_tokens, inner_toks)
|
||
let out_tokens = tok_append(out_tokens, "RParen", ")")
|
||
}
|
||
let need_plus = true
|
||
} else {
|
||
// Plain '$' not followed by '{' - treat as literal, continue clean run
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
// Plain char — extends clean run, no append needed
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Flush remaining literal segment and build final token list
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let part_len: Int = native_list_len(cur_parts)
|
||
let part_text = str_join(cur_parts, "")
|
||
if has_interp {
|
||
// Interpolated string: only emit trailing segment if non-empty
|
||
if part_len > 0 {
|
||
let clean_part = part_text
|
||
if looks_like_code(part_text) {
|
||
let clean_part = strip_code_comments(part_text)
|
||
}
|
||
if need_plus {
|
||
let out_tokens = tok_append(out_tokens, "Plus", "+")
|
||
}
|
||
let out_tokens = tok_append(out_tokens, "Str", clean_part)
|
||
}
|
||
} else {
|
||
// Plain string with no interpolation - same behaviour as old scan_string
|
||
let clean_text = part_text
|
||
if looks_like_code(part_text) {
|
||
let clean_text = strip_code_comments(part_text)
|
||
}
|
||
let out_tokens = tok_append(out_tokens, "Str", clean_text)
|
||
}
|
||
|
||
{ "tokens": out_tokens, "pos": i }
|
||
}
|
||
|
||
// -- Main lexer ----------------------------------------------------------------
|
||
// Char code constants (avoids strdup for single-char comparison)
|
||
// '/' = 47, '"' = 34, '0'-'9' = 48-57, 'a'-'z' = 97-122, 'A'-'Z' = 65-90
|
||
// '_' = 95, ' '=32, '\t'=9, '\n'=10, '\r'=13
|
||
// '=' = 61, '!' = 33, '<' = 60, '>' = 62, '&' = 38, '|' = 124
|
||
// '-' = 45, ':' = 58, '+' = 43, '*' = 42, '%' = 37
|
||
// '(' = 40, ')' = 41, '{' = 123, '}' = 125, '[' = 91, ']' = 93
|
||
// ',' = 44, '.' = 46, ';' = 59, '@' = 64, '?' = 63
|
||
|
||
fn lex(source: String) -> [Any] {
|
||
// Use str_char_code (returns Int) instead of str_char_at (returns strdup String)
|
||
// for all character classification in the hot loop. For a 400KB source,
|
||
// str_char_at allocates ~400K × 16B = ~6.4MB of temporary strings.
|
||
let total: Int = str_len(source)
|
||
let tokens: [Any] = native_list_empty()
|
||
let i: Int = 0
|
||
|
||
while i < total {
|
||
let c: Int = str_char_code(source, i)
|
||
|
||
// Skip whitespace (space=32, tab=9, newline=10, CR=13)
|
||
if is_ws_code(c) {
|
||
let i = i + 1
|
||
} else {
|
||
// Line comments: // (slash=47)
|
||
if c == 47 {
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let nc: Int = str_char_code(source, next_i)
|
||
if nc == 47 {
|
||
// skip to end of line (newline=10)
|
||
let i = i + 2
|
||
let running2 = true
|
||
while running2 {
|
||
if i >= total {
|
||
let running2 = false
|
||
} else {
|
||
let lc: Int = str_char_code(source, i)
|
||
if lc == 10 {
|
||
let running2 = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Slash", "/")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Slash", "/")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
// String literal: '"' = 34
|
||
if c == 34 {
|
||
let interp_result = scan_interp_string(source, i + 1, total)
|
||
let interp_toks: [Any] = interp_result["tokens"]
|
||
let new_pos: Int = interp_result["pos"]
|
||
let tokens = interp_tokens_append_all(tokens, interp_toks)
|
||
let i = new_pos
|
||
} else {
|
||
// Number literal: '0'-'9' = 48-57
|
||
if is_digit_code(c) {
|
||
let result = scan_digits(source, i, total)
|
||
let num_text: String = result["text"]
|
||
let new_pos: Int = result["pos"]
|
||
// check for float (dot=46 followed by digit)
|
||
if new_pos < total {
|
||
let dc: Int = str_char_code(source, new_pos)
|
||
if dc == 46 {
|
||
let after_dot = new_pos + 1
|
||
if after_dot < total {
|
||
let adc: Int = str_char_code(source, after_dot)
|
||
if is_digit_code(adc) {
|
||
let frac_result = scan_digits(source, after_dot, total)
|
||
let frac_text: String = frac_result["text"]
|
||
let frac_pos: Int = frac_result["pos"]
|
||
let tokens = tok_append(tokens, "Float", num_text + "." + frac_text)
|
||
let i = frac_pos
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
// Identifier or keyword: alpha or '_'=95
|
||
if is_alpha_code(c) || c == 95 {
|
||
let result = scan_ident(source, i, total)
|
||
let word: String = result["text"]
|
||
let new_pos: Int = result["pos"]
|
||
let kw = keyword_kind(word)
|
||
if kw == "" {
|
||
let tokens = tok_append(tokens, "Ident", word)
|
||
} else {
|
||
let tokens = tok_append(tokens, kw, word)
|
||
}
|
||
let i = new_pos
|
||
} else {
|
||
// Multi-char and single-char operators/delimiters
|
||
let peek_i = i + 1
|
||
let peek_c: Int = -1
|
||
if peek_i < total {
|
||
let peek_c: Int = str_char_code(source, peek_i)
|
||
}
|
||
|
||
if c == 61 {
|
||
// '=' = 61
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "EqEq", "==")
|
||
let i = i + 2
|
||
} else {
|
||
if peek_c == 62 {
|
||
// '>' = 62
|
||
let tokens = tok_append(tokens, "FatArrow", "=>")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Eq", "=")
|
||
let i = i + 1
|
||
}
|
||
}
|
||
} else {
|
||
if c == 33 {
|
||
// '!' = 33
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "NotEq", "!=")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Not", "!")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 60 {
|
||
// '<' = 60
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "LtEq", "<=")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Lt", "<")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 62 {
|
||
// '>' = 62
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "GtEq", ">=")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Gt", ">")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 38 {
|
||
// '&' = 38
|
||
if peek_c == 38 {
|
||
let tokens = tok_append(tokens, "And", "&&")
|
||
let i = i + 2
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 124 {
|
||
// '|' = 124
|
||
if peek_c == 124 {
|
||
let tokens = tok_append(tokens, "Or", "||")
|
||
let i = i + 2
|
||
} else {
|
||
if peek_c == 62 {
|
||
// '>' = 62
|
||
let tokens = tok_append(tokens, "PipeOp", "|>")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Pipe", "|")
|
||
let i = i + 1
|
||
}
|
||
}
|
||
} else {
|
||
if c == 45 {
|
||
// '-' = 45
|
||
if peek_c == 62 {
|
||
// '>' = 62
|
||
let tokens = tok_append(tokens, "Arrow", "->")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Minus", "-")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 58 {
|
||
// ':' = 58
|
||
if peek_c == 58 {
|
||
let tokens = tok_append(tokens, "ColonColon", "::")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Colon", ":")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 43 {
|
||
// '+' = 43
|
||
let tokens = tok_append(tokens, "Plus", "+")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 42 {
|
||
// '*' = 42
|
||
let tokens = tok_append(tokens, "Star", "*")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 37 {
|
||
// '%' = 37
|
||
let tokens = tok_append(tokens, "Percent", "%")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 40 {
|
||
// '(' = 40
|
||
let tokens = tok_append(tokens, "LParen", "(")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 41 {
|
||
// ')' = 41
|
||
let tokens = tok_append(tokens, "RParen", ")")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 123 {
|
||
// '{' = 123
|
||
let tokens = tok_append(tokens, "LBrace", "{")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 125 {
|
||
// '}' = 125
|
||
let tokens = tok_append(tokens, "RBrace", "}")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 91 {
|
||
// '[' = 91
|
||
let tokens = tok_append(tokens, "LBracket", "[")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 93 {
|
||
// ']' = 93
|
||
let tokens = tok_append(tokens, "RBracket", "]")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 44 {
|
||
// ',' = 44
|
||
let tokens = tok_append(tokens, "Comma", ",")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 46 {
|
||
// '.' = 46: check for ..= or ..
|
||
let peek2_i = i + 2
|
||
let peek2_c: Int = -1
|
||
if peek2_i < total {
|
||
let peek2_c: Int = str_char_code(source, peek2_i)
|
||
}
|
||
if peek_c == 46 {
|
||
// '..' prefix
|
||
if peek2_c == 61 {
|
||
// '..=' = 46 46 61
|
||
let tokens = tok_append(tokens, "DotDotEq", "..=")
|
||
let i = i + 3
|
||
} else {
|
||
let tokens = tok_append(tokens, "DotDot", "..")
|
||
let i = i + 2
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Dot", ".")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 59 {
|
||
// ';' = 59
|
||
let tokens = tok_append(tokens, "Semicolon", ";")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 64 {
|
||
// '@' = 64
|
||
let tokens = tok_append(tokens, "At", "@")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 63 {
|
||
// '?' = 63
|
||
let tokens = tok_append(tokens, "QuestionMark", "?")
|
||
let i = i + 1
|
||
} else {
|
||
// unknown char - skip
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
let tokens = tok_append(tokens, "Eof", "")
|
||
tokens
|
||
}
|