bigmerge
8ae163e8e5
El's units of encapsulation are the function and the module. Neither can hold
a concern that belongs to the process, so each one had been expressed the only
way it could be -- as a convention: call this at every site. Conventions of
that shape do not hold. Measured here: zero process-identity guards at any
layer, 20 environment variables each with its default written inline at the
read site, 62 persist call sites, 10 per-route auth checks. One absence, four
times.
Step 0 first, because the premise was wrong. El was believed to have no
middleware or effect mechanism. It has one, and it is already load-bearing:
codegen injects engram_boundary_beat at the entry of every @manager/@accessor
fn, decorators take arguments and stack, dharma_emit from a non-@manager fn is
a #error, and the cgi block injects el_cgi_init at the head of main(). So the
correct move was not to invent a mechanism but to generalize the seam that
already existed. The real gap is narrower and is now recorded: the seam is
prologue-only and its callee is a fixed builtin.
Adds a `program` block -- the third program-level declarative block. cgi and
service declare what a program may do; program declares what it is.
program "engram" {
singleton: "engram"
env ENGRAM_BIND: String = ":8742"
env GUIDE_PORT: Int = "8771"
}
singleton takes an exclusive flock before any user statement runs and refuses a
second start, reporting the holder's pid. It is a lock rather than a pidfile so
the kernel releases it on death including SIGKILL -- no stale state, and so no
"delete the lock file to get unstuck" ritual, which would itself be a
convention. It reports the pid because "already running" is not actionable; a
pid is. That is the direct answer to a stale process surviving a pkill and
going on answering probes.
env entries resolve once at startup -- environment wins, declaration supplies
the fallback -- and validate as a whole, reporting every problem at once rather
than costing one restart per variable. config("X") for an undeclared X is
fatal, because an advisory schema is just another convention. Programs without
a program block are unaffected, so migration is per-program.
Only one keyword is added. `config` and `env` could not become keywords -- both
are real identifiers in the tree -- so the block's fields are read as
identifier token values by its own parse loop and stay usable everywhere else.
The init function is emitted at the block site and called from main() rather
than inlined into main(). The live backend is codegen_streaming, which emits in
source order and cannot hold the entry list alive until main(); this way only a
single bool has to survive.
Also fixes: config() was defined in el_runtime.c but never prototyped in
el_runtime.h, so any el program calling it failed to compile under C99.
Spec: section 18 documents what shipped. Section 9 is corrected -- it claimed
decorators had no structural meaning, which has not been true for some time.
Section 19 designs durability-as-an-epilogue-effect and route authorization
and states plainly why neither is implemented here: both land in files under
concurrent modification, and the prerequisite for both is lifting the seam
from prologue-only to prologue/epilogue.
Self-hosting fixpoint verified byte-identical.
1081 lines
52 KiB
EmacsLisp
1081 lines
52 KiB
EmacsLisp
// lexer.el - el self-hosting lexer
|
||
//
|
||
// Tokenises an el source string into a list of token maps.
|
||
// Each token is a Map<String, Any> with keys:
|
||
// "kind" -> String (e.g. "Int", "Ident", "Plus")
|
||
// "value" -> String (the raw text of the token)
|
||
//
|
||
// Entry point: fn lex(source: String) -> [Map<String, Any>]
|
||
//
|
||
// Performance: the hot lexer loop uses str_char_code (returns Int) instead of
|
||
// str_char_at (returns strdup'd String) for character classification.
|
||
// For a 400KB source, str_char_at allocates ~400K × 16B = ~6.4MB of temporary
|
||
// strings for the `ch` variable alone. str_char_code avoids all that.
|
||
|
||
// -- Character helpers (Int-based, no string allocation) ----------------------
|
||
// These operate on char codes (from str_char_code) instead of str_char_at,
|
||
// eliminating one strdup per character in the hot lexer loop.
|
||
|
||
fn is_digit_code(c: Int) -> Bool {
|
||
// '0'=48 .. '9'=57
|
||
if c >= 48 {
|
||
if c <= 57 { return true }
|
||
}
|
||
false
|
||
}
|
||
|
||
fn is_alpha_code(c: Int) -> Bool {
|
||
// 'A'=65..'Z'=90, 'a'=97..'z'=122
|
||
if c >= 65 {
|
||
if c <= 90 { return true }
|
||
}
|
||
if c >= 97 {
|
||
if c <= 122 { return true }
|
||
}
|
||
false
|
||
}
|
||
|
||
fn is_alnum_or_underscore_code(c: Int) -> Bool {
|
||
if is_digit_code(c) { return true }
|
||
if is_alpha_code(c) { return true }
|
||
if c == 95 { return true } // '_'
|
||
false
|
||
}
|
||
|
||
fn is_ws_code(c: Int) -> Bool {
|
||
if c == 32 { return true } // ' '
|
||
if c == 9 { return true } // '\t'
|
||
if c == 10 { return true } // '\n'
|
||
if c == 13 { return true } // '\r'
|
||
false
|
||
}
|
||
|
||
// Legacy String-based helpers kept for scan_interp helpers that use str_char_at.
|
||
fn lex_is_digit(ch: String) -> Bool {
|
||
if ch == "0" { return true }
|
||
if ch == "1" { return true }
|
||
if ch == "2" { return true }
|
||
if ch == "3" { return true }
|
||
if ch == "4" { return true }
|
||
if ch == "5" { return true }
|
||
if ch == "6" { return true }
|
||
if ch == "7" { return true }
|
||
if ch == "8" { return true }
|
||
if ch == "9" { return true }
|
||
false
|
||
}
|
||
|
||
fn lex_is_alpha(ch: String) -> Bool {
|
||
if ch == "a" { return true }
|
||
if ch == "b" { return true }
|
||
if ch == "c" { return true }
|
||
if ch == "d" { return true }
|
||
if ch == "e" { return true }
|
||
if ch == "f" { return true }
|
||
if ch == "g" { return true }
|
||
if ch == "h" { return true }
|
||
if ch == "i" { return true }
|
||
if ch == "j" { return true }
|
||
if ch == "k" { return true }
|
||
if ch == "l" { return true }
|
||
if ch == "m" { return true }
|
||
if ch == "n" { return true }
|
||
if ch == "o" { return true }
|
||
if ch == "p" { return true }
|
||
if ch == "q" { return true }
|
||
if ch == "r" { return true }
|
||
if ch == "s" { return true }
|
||
if ch == "t" { return true }
|
||
if ch == "u" { return true }
|
||
if ch == "v" { return true }
|
||
if ch == "w" { return true }
|
||
if ch == "x" { return true }
|
||
if ch == "y" { return true }
|
||
if ch == "z" { return true }
|
||
if ch == "A" { return true }
|
||
if ch == "B" { return true }
|
||
if ch == "C" { return true }
|
||
if ch == "D" { return true }
|
||
if ch == "E" { return true }
|
||
if ch == "F" { return true }
|
||
if ch == "G" { return true }
|
||
if ch == "H" { return true }
|
||
if ch == "I" { return true }
|
||
if ch == "J" { return true }
|
||
if ch == "K" { return true }
|
||
if ch == "L" { return true }
|
||
if ch == "M" { return true }
|
||
if ch == "N" { return true }
|
||
if ch == "O" { return true }
|
||
if ch == "P" { return true }
|
||
if ch == "Q" { return true }
|
||
if ch == "R" { return true }
|
||
if ch == "S" { return true }
|
||
if ch == "T" { return true }
|
||
if ch == "U" { return true }
|
||
if ch == "V" { return true }
|
||
if ch == "W" { return true }
|
||
if ch == "X" { return true }
|
||
if ch == "Y" { return true }
|
||
if ch == "Z" { return true }
|
||
false
|
||
}
|
||
|
||
fn is_alnum_or_underscore(ch: String) -> Bool {
|
||
if lex_is_digit(ch) { return true }
|
||
if lex_is_alpha(ch) { return true }
|
||
if ch == "_" { return true }
|
||
false
|
||
}
|
||
|
||
fn lex_is_whitespace(ch: String) -> Bool {
|
||
if ch == " " { return true }
|
||
if ch == "\t" { return true }
|
||
if ch == "\n" { return true }
|
||
if ch == "\r" { return true }
|
||
false
|
||
}
|
||
|
||
// tok_append — append a (kind, value) pair to a flat token list.
|
||
// Returns the updated list. Gamma combines flat-list + char-code for max savings.
|
||
fn tok_append(tokens: [Any], kind: String, value: String) -> [Any] {
|
||
let tokens = native_list_append(tokens, kind)
|
||
native_list_append(tokens, value)
|
||
}
|
||
|
||
// -- Keyword lookup ------------------------------------------------------------
|
||
|
||
fn keyword_kind(word: String) -> String {
|
||
if word == "let" { return "Let" }
|
||
if word == "fn" { return "Fn" }
|
||
if word == "type" { return "Type" }
|
||
if word == "enum" { return "Enum" }
|
||
if word == "match" { return "Match" }
|
||
if word == "return" { return "Return" }
|
||
if word == "if" { return "If" }
|
||
if word == "else" { return "Else" }
|
||
if word == "for" { return "For" }
|
||
if word == "in" { return "In" }
|
||
if word == "while" { return "While" }
|
||
if word == "import" { return "Import" }
|
||
if word == "from" { return "From" }
|
||
if word == "as" { return "As" }
|
||
if word == "with" { return "With" }
|
||
if word == "sealed" { return "Sealed" }
|
||
if word == "activate" { return "Activate" }
|
||
if word == "where" { return "Where" }
|
||
if word == "test" { return "Test" }
|
||
if word == "seed" { return "Seed" }
|
||
if word == "assert" { return "Assert" }
|
||
if word == "protocol" { return "Protocol" }
|
||
if word == "impl" { return "Impl" }
|
||
if word == "retry" { return "Retry" }
|
||
if word == "times" { return "Times" }
|
||
if word == "fallback" { return "Fallback" }
|
||
if word == "reason" { return "Reason" }
|
||
if word == "parallel" { return "Parallel" }
|
||
if word == "trace" { return "Trace" }
|
||
if word == "requires" { return "Requires" }
|
||
if word == "deploy" { return "Deploy" }
|
||
if word == "to" { return "To" }
|
||
if word == "via" { return "Via" }
|
||
if word == "target" { return "Target" }
|
||
if word == "true" { return "Bool" }
|
||
if word == "false" { return "Bool" }
|
||
if word == "cgi" { return "Cgi" }
|
||
if word == "service" { return "Service" }
|
||
if word == "program" { return "Program" }
|
||
if word == "manager" { return "Manager" }
|
||
if word == "engine" { return "Engine" }
|
||
if word == "accessor" { return "Accessor" }
|
||
if word == "vessel" { return "Vessel" }
|
||
if word == "extern" { return "Extern" }
|
||
if word == "break" { return "Break" }
|
||
if word == "continue" { return "Continue" }
|
||
""
|
||
}
|
||
|
||
// -- Scan helpers --------------------------------------------------------------
|
||
// All scan helpers receive the chars list and total length.
|
||
|
||
// scan_digits - advance i while chars[i] is a digit
|
||
// Returns { "text": ..., "pos": i }
|
||
fn scan_digits(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let c: Int = str_char_code(src, i)
|
||
if is_digit_code(c) {
|
||
let i = i + 1
|
||
} else {
|
||
let running = false
|
||
}
|
||
}
|
||
}
|
||
// Use str_slice instead of building a parts list — O(1) allocation, O(n) copy.
|
||
{ "text": str_slice(src, start, i), "pos": i }
|
||
}
|
||
|
||
// scan_ident - advance i while chars[i] is alphanumeric or underscore
|
||
fn scan_ident(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let c: Int = str_char_code(src, i)
|
||
if is_alnum_or_underscore_code(c) {
|
||
let i = i + 1
|
||
} else {
|
||
let running = false
|
||
}
|
||
}
|
||
}
|
||
// Use str_slice instead of building a parts list — O(1) allocation, O(n) copy.
|
||
{ "text": str_slice(src, start, i), "pos": i }
|
||
}
|
||
|
||
// -- Code-bearing string detection + comment strip ----------------------------
|
||
// Inline JS/CSS literals embedded in El source (e.g. <script>-</script> blobs
|
||
// or stylesheet payloads inside string literals) carry their own line and
|
||
// block comments. Those comments leak into the served HTML and reveal build
|
||
// notes the visitor should never see. We strip them at the lexer so every
|
||
// downstream consumer (codegen-c, codegen-js, parser) gets the cleaned form.
|
||
//
|
||
// looks_like_code - heuristic gate so we only strip strings that actually
|
||
// embed JS or CSS. Plain prose, hex blobs, JSON, etc. pass through verbatim.
|
||
|
||
fn substr_at(src: String, start: Int, total: Int, needle: String) -> Bool {
|
||
let nlen: Int = str_len(needle)
|
||
if start + nlen > total { return false }
|
||
// Use str_slice comparison instead of char-by-char loop.
|
||
str_eq(str_slice(src, start, start + nlen), needle)
|
||
}
|
||
|
||
fn str_has(s: String, needle: String) -> Bool {
|
||
// Use the built-in str_contains which is implemented in native C — O(n) single pass.
|
||
str_contains(s, needle)
|
||
}
|
||
|
||
fn looks_like_code(s: String) -> Bool {
|
||
if str_has(s, "<script") { return true }
|
||
if str_has(s, "<style") { return true }
|
||
if str_has(s, "function") {
|
||
if str_has(s, ";") { return true }
|
||
}
|
||
false
|
||
}
|
||
|
||
// strip_code_comments - character-by-character walk. Tracks JS string state
|
||
// (single, double, backtick) and never strips inside one. Backslash escapes
|
||
// inside JS strings consume the next char verbatim. URLs like https:// are
|
||
// preserved by checking the previous char before treating // as a line
|
||
// comment opener: if the char immediately before '/' is ':', emit the '/'
|
||
// literally and advance one position.
|
||
fn strip_code_comments(s: String) -> String {
|
||
let total: Int = str_len(s)
|
||
let out_parts: [String] = native_list_empty()
|
||
let i = 0
|
||
let in_squote = false
|
||
let in_dquote = false
|
||
let in_btick = false
|
||
let prev = ""
|
||
while i < total {
|
||
let ch: String = str_char_at(s, i)
|
||
let in_js_string = false
|
||
if in_squote { let in_js_string = true }
|
||
if in_dquote { let in_js_string = true }
|
||
if in_btick { let in_js_string = true }
|
||
|
||
if in_js_string {
|
||
// Backslash escape: consume next char verbatim regardless of which.
|
||
if ch == "\\" {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let nc: String = str_char_at(s, next_i)
|
||
let out_parts = native_list_append(out_parts, nc)
|
||
let prev = nc
|
||
let i = next_i + 1
|
||
} else {
|
||
let prev = ch
|
||
let i = next_i
|
||
}
|
||
} else {
|
||
if in_squote {
|
||
if ch == "'" { let in_squote = false }
|
||
} else {
|
||
if in_dquote {
|
||
if ch == "\"" { let in_dquote = false }
|
||
} else {
|
||
if in_btick {
|
||
if ch == "`" { let in_btick = false }
|
||
}
|
||
}
|
||
}
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
// Not in a JS string. Check for comment openers.
|
||
let next_i = i + 1
|
||
let next_ch = ""
|
||
if next_i < total {
|
||
let next_ch: String = str_char_at(s, next_i)
|
||
}
|
||
|
||
if ch == "/" {
|
||
if next_ch == "/" {
|
||
// URL guard: prev char ':' means this is "://", not a comment.
|
||
if prev == ":" {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
// Skip until newline (newline itself is preserved so
|
||
// surrounding line counts/structure stay sane).
|
||
let i = i + 2
|
||
let scanning = true
|
||
while scanning {
|
||
if i >= total {
|
||
let scanning = false
|
||
} else {
|
||
let lc: String = str_char_at(s, i)
|
||
if lc == "\n" {
|
||
let scanning = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
let prev = ""
|
||
}
|
||
} else {
|
||
if next_ch == "*" {
|
||
// Skip until matching "*/".
|
||
let i = i + 2
|
||
let scanning2 = true
|
||
while scanning2 {
|
||
if i >= total {
|
||
let scanning2 = false
|
||
} else {
|
||
let bc: String = str_char_at(s, i)
|
||
if bc == "*" {
|
||
let after = i + 1
|
||
if after < total {
|
||
let nc2: String = str_char_at(s, after)
|
||
if nc2 == "/" {
|
||
let i = after + 1
|
||
let scanning2 = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
let prev = ""
|
||
} else {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
}
|
||
}
|
||
} else {
|
||
// Open a JS string?
|
||
if ch == "'" {
|
||
let in_squote = true
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
if ch == "\"" {
|
||
let in_dquote = true
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
if ch == "`" {
|
||
let in_btick = true
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
} else {
|
||
let out_parts = native_list_append(out_parts, ch)
|
||
let prev = ch
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
str_join(out_parts, "")
|
||
}
|
||
|
||
// scan_string - scan a quoted string literal, handling \" escapes.
|
||
// Starts AFTER the opening quote. Returns { "text": content, "pos": i_after_close }
|
||
fn scan_string(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let parts: [String] = native_list_empty()
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let ch: String = str_char_at(src, i)
|
||
if ch == "\\" {
|
||
// escape: peek next char
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let next_ch: String = str_char_at(src, next_i)
|
||
if next_ch == "\"" {
|
||
let parts = native_list_append(parts, "\"")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "n" {
|
||
let parts = native_list_append(parts, "\n")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "t" {
|
||
let parts = native_list_append(parts, "\t")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "r" {
|
||
let parts = native_list_append(parts, "\r")
|
||
let i = next_i + 1
|
||
} else {
|
||
if next_ch == "\\" {
|
||
let parts = native_list_append(parts, "\\")
|
||
let i = next_i + 1
|
||
} else {
|
||
let parts = native_list_append(parts, next_ch)
|
||
let i = next_i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if ch == "\"" {
|
||
let i = i + 1
|
||
let running = false
|
||
} else {
|
||
let parts = native_list_append(parts, ch)
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
{ "text": str_join(parts, ""), "pos": i }
|
||
}
|
||
|
||
// -- String interpolation ------------------------------------------------------
|
||
//
|
||
// scan_interp_brace - scan from `start` (the char after `${`) to the matching
|
||
// `}`, tracking brace depth so inner braces (e.g. fn calls, map literals) are
|
||
// handled correctly. Returns { "text": inner_source, "pos": i_after_close }.
|
||
fn scan_interp_brace(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let depth = 1
|
||
let running = true
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let ch: String = str_char_at(src, i)
|
||
if ch == "{" {
|
||
let depth = depth + 1
|
||
let i = i + 1
|
||
} else {
|
||
if ch == "}" {
|
||
let depth = depth - 1
|
||
if depth <= 0 {
|
||
// Closing brace of the interpolation - stop, do not include it
|
||
let i = i + 1
|
||
let running = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
// Use str_slice instead of parts list — the inner source is a contiguous substring.
|
||
{ "text": str_slice(src, start, i - 1), "pos": i }
|
||
}
|
||
|
||
// interp_tokens_append_all - copy every (kind, value) pair from flat src list
|
||
// into flat dst list, skipping the trailing Eof pair that lex() always appends.
|
||
fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] {
|
||
let src_len: Int = native_list_len(src)
|
||
let j = 0
|
||
let result = dst
|
||
while j < src_len {
|
||
let kind: String = native_list_get(src, j)
|
||
if kind == "Eof" {
|
||
let j = src_len
|
||
} else {
|
||
let val: String = native_list_get(src, j + 1)
|
||
let result = native_list_append(result, kind)
|
||
let result = native_list_append(result, val)
|
||
let j = j + 2
|
||
}
|
||
}
|
||
result
|
||
}
|
||
|
||
// scan_interp_string - scan a string literal that may contain ${expr}
|
||
// interpolations. Starts AFTER the opening `"`.
|
||
// Returns { "tokens": [token list to inject], "pos": i_after_close_quote }.
|
||
//
|
||
// For a plain string (no ${}) this emits a single Str token, identical to the
|
||
// old scan_string path. For an interpolated string it emits a flat sequence
|
||
// of tokens equivalent to the string-concat expression, for example:
|
||
//
|
||
// "hello ${name}!"
|
||
// => Str("hello ") Plus <tokens for name> Plus Str("!")
|
||
//
|
||
// Empty literal segments between adjacent ${ } blocks are omitted. The
|
||
// resulting token stream is consumed by the existing parse_binop / parse_primary
|
||
// path in the parser with zero parser changes required.
|
||
//
|
||
// Supported escape sequences: \" \n \t \r \\ \$ (literal dollar sign).
|
||
// Nested quotes inside ${} are not supported; use a variable instead.
|
||
//
|
||
// Performance: uses str_char_code (Int) for all character dispatch, eliminating
|
||
// per-character strdup. Plain runs are batched into str_slice segments instead
|
||
// of accumulating single-char strings, reducing list appends from O(N) to O(K)
|
||
// where K = number of escape/special chars in the literal.
|
||
// Char codes: '\' = 92, '"' = 34, '$' = 36, '{' = 123
|
||
fn scan_interp_string(src: String, start: Int, total: Int) -> Map<String, Any> {
|
||
let i = start
|
||
let out_tokens: [Any] = native_list_empty()
|
||
let cur_parts: [String] = native_list_empty()
|
||
let clean_start = start
|
||
let has_interp = false
|
||
let need_plus = false
|
||
let running = true
|
||
|
||
while running {
|
||
if i >= total {
|
||
let running = false
|
||
} else {
|
||
let c: Int = str_char_code(src, i)
|
||
|
||
if c == 92 {
|
||
// '\\' = 92 — escape sequence: flush clean run, append resolved char
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let nc: Int = str_char_code(src, next_i)
|
||
if nc == 36 {
|
||
// '\$' => literal '$' (36 = '$')
|
||
let cur_parts = native_list_append(cur_parts, "$")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 34 {
|
||
// '\"' => literal '"' (34 = '"')
|
||
let cur_parts = native_list_append(cur_parts, "\"")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 110 {
|
||
// '\n' (110 = 'n')
|
||
let cur_parts = native_list_append(cur_parts, "\n")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 116 {
|
||
// '\t' (116 = 't')
|
||
let cur_parts = native_list_append(cur_parts, "\t")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 114 {
|
||
// '\r' (114 = 'r')
|
||
let cur_parts = native_list_append(cur_parts, "\r")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
if nc == 92 {
|
||
// '\\' (92)
|
||
let cur_parts = native_list_append(cur_parts, "\\")
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
} else {
|
||
// Unknown escape: emit the escaped char verbatim
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, next_i, next_i + 1))
|
||
let clean_start = next_i + 1
|
||
let i = next_i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
let clean_start = next_i
|
||
let i = next_i
|
||
}
|
||
} else {
|
||
if c == 34 {
|
||
// '"' = 34 — closing quote: flush clean run, stop
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let i = i + 1
|
||
let clean_start = i
|
||
let running = false
|
||
} else {
|
||
if c == 36 {
|
||
// '$' = 36 — possible interpolation start
|
||
let next_i = i + 1
|
||
let is_interp = false
|
||
if next_i < total {
|
||
let nc2: Int = str_char_code(src, next_i)
|
||
if nc2 == 123 {
|
||
// '{' = 123
|
||
let is_interp = true
|
||
}
|
||
}
|
||
if is_interp {
|
||
// Flush the accumulated literal part (if non-empty)
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let part_len: Int = native_list_len(cur_parts)
|
||
if part_len > 0 {
|
||
let part_text = str_join(cur_parts, "")
|
||
if need_plus {
|
||
let out_tokens = tok_append(out_tokens, "Plus", "+")
|
||
}
|
||
let clean_part = part_text
|
||
if looks_like_code(part_text) {
|
||
let clean_part = strip_code_comments(part_text)
|
||
}
|
||
let out_tokens = tok_append(out_tokens, "Str", clean_part)
|
||
let need_plus = true
|
||
}
|
||
let cur_parts = native_list_empty()
|
||
let has_interp = true
|
||
|
||
// Scan brace-balanced expression source
|
||
let brace_result = scan_interp_brace(src, next_i + 1, total)
|
||
let expr_src: String = brace_result["text"]
|
||
let new_i: Int = brace_result["pos"]
|
||
let i = new_i
|
||
let clean_start = new_i
|
||
|
||
// Re-lex the expression and inline the tokens.
|
||
// Wrap in ( ) so that operators inside ${} (e.g.
|
||
// age + 1) are parsed as a grouped sub-expression
|
||
// rather than merging with the surrounding concat
|
||
// Plus tokens at the wrong precedence level.
|
||
let inner_toks: [Any] = lex(expr_src)
|
||
let inner_len: Int = native_list_len(inner_toks)
|
||
|
||
if need_plus {
|
||
let out_tokens = tok_append(out_tokens, "Plus", "+")
|
||
}
|
||
// Empty interpolation ${} => empty string segment
|
||
// inner_len <= 2 = only the Eof pair (kind="Eof", value="")
|
||
if inner_len <= 2 {
|
||
let out_tokens = tok_append(out_tokens, "Str", "")
|
||
} else {
|
||
let out_tokens = tok_append(out_tokens, "LParen", "(")
|
||
let out_tokens = interp_tokens_append_all(out_tokens, inner_toks)
|
||
let out_tokens = tok_append(out_tokens, "RParen", ")")
|
||
}
|
||
let need_plus = true
|
||
} else {
|
||
// Plain '$' not followed by '{' - treat as literal, continue clean run
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
// Plain char — extends clean run, no append needed
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Flush remaining literal segment and build final token list
|
||
if clean_start < i {
|
||
let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i))
|
||
}
|
||
let part_len: Int = native_list_len(cur_parts)
|
||
let part_text = str_join(cur_parts, "")
|
||
if has_interp {
|
||
// Interpolated string: only emit trailing segment if non-empty
|
||
if part_len > 0 {
|
||
let clean_part = part_text
|
||
if looks_like_code(part_text) {
|
||
let clean_part = strip_code_comments(part_text)
|
||
}
|
||
if need_plus {
|
||
let out_tokens = tok_append(out_tokens, "Plus", "+")
|
||
}
|
||
let out_tokens = tok_append(out_tokens, "Str", clean_part)
|
||
}
|
||
} else {
|
||
// Plain string with no interpolation - same behaviour as old scan_string
|
||
let clean_text = part_text
|
||
if looks_like_code(part_text) {
|
||
let clean_text = strip_code_comments(part_text)
|
||
}
|
||
let out_tokens = tok_append(out_tokens, "Str", clean_text)
|
||
}
|
||
|
||
{ "tokens": out_tokens, "pos": i }
|
||
}
|
||
|
||
// -- Main lexer ----------------------------------------------------------------
|
||
// Char code constants (avoids strdup for single-char comparison)
|
||
// '/' = 47, '"' = 34, '0'-'9' = 48-57, 'a'-'z' = 97-122, 'A'-'Z' = 65-90
|
||
// '_' = 95, ' '=32, '\t'=9, '\n'=10, '\r'=13
|
||
// '=' = 61, '!' = 33, '<' = 60, '>' = 62, '&' = 38, '|' = 124
|
||
// '-' = 45, ':' = 58, '+' = 43, '*' = 42, '%' = 37
|
||
// '(' = 40, ')' = 41, '{' = 123, '}' = 125, '[' = 91, ']' = 93
|
||
// ',' = 44, '.' = 46, ';' = 59, '@' = 64, '?' = 63
|
||
|
||
fn lex(source: String) -> [Any] {
|
||
// Use str_char_code (returns Int) instead of str_char_at (returns strdup String)
|
||
// for all character classification in the hot loop. For a 400KB source,
|
||
// str_char_at allocates ~400K × 16B = ~6.4MB of temporary strings.
|
||
let total: Int = str_len(source)
|
||
let tokens: [Any] = native_list_empty()
|
||
let i: Int = 0
|
||
|
||
while i < total {
|
||
let c: Int = str_char_code(source, i)
|
||
|
||
// Skip whitespace (space=32, tab=9, newline=10, CR=13)
|
||
if is_ws_code(c) {
|
||
let i = i + 1
|
||
} else {
|
||
// Line comments: // (slash=47)
|
||
if c == 47 {
|
||
let next_i = i + 1
|
||
if next_i < total {
|
||
let nc: Int = str_char_code(source, next_i)
|
||
if nc == 47 {
|
||
// skip to end of line (newline=10)
|
||
let i = i + 2
|
||
let running2 = true
|
||
while running2 {
|
||
if i >= total {
|
||
let running2 = false
|
||
} else {
|
||
let lc: Int = str_char_code(source, i)
|
||
if lc == 10 {
|
||
let running2 = false
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Slash", "/")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Slash", "/")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
// String literal: '"' = 34
|
||
if c == 34 {
|
||
let interp_result = scan_interp_string(source, i + 1, total)
|
||
let interp_toks: [Any] = interp_result["tokens"]
|
||
let new_pos: Int = interp_result["pos"]
|
||
let tokens = interp_tokens_append_all(tokens, interp_toks)
|
||
let i = new_pos
|
||
} else {
|
||
// Number literal: '0'-'9' = 48-57
|
||
if is_digit_code(c) {
|
||
let result = scan_digits(source, i, total)
|
||
let num_text: String = result["text"]
|
||
let new_pos: Int = result["pos"]
|
||
// check for float (dot=46 followed by digit)
|
||
if new_pos < total {
|
||
let dc: Int = str_char_code(source, new_pos)
|
||
if dc == 46 {
|
||
let after_dot = new_pos + 1
|
||
if after_dot < total {
|
||
let adc: Int = str_char_code(source, after_dot)
|
||
if is_digit_code(adc) {
|
||
let frac_result = scan_digits(source, after_dot, total)
|
||
let frac_text: String = frac_result["text"]
|
||
let frac_pos: Int = frac_result["pos"]
|
||
let tokens = tok_append(tokens, "Float", num_text + "." + frac_text)
|
||
let i = frac_pos
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Int", num_text)
|
||
let i = new_pos
|
||
}
|
||
} else {
|
||
// Identifier or keyword: alpha or '_'=95
|
||
if is_alpha_code(c) || c == 95 {
|
||
let result = scan_ident(source, i, total)
|
||
let word: String = result["text"]
|
||
let new_pos: Int = result["pos"]
|
||
let kw = keyword_kind(word)
|
||
if kw == "" {
|
||
let tokens = tok_append(tokens, "Ident", word)
|
||
} else {
|
||
let tokens = tok_append(tokens, kw, word)
|
||
}
|
||
let i = new_pos
|
||
} else {
|
||
// Multi-char and single-char operators/delimiters
|
||
let peek_i = i + 1
|
||
let peek_c: Int = -1
|
||
if peek_i < total {
|
||
let peek_c: Int = str_char_code(source, peek_i)
|
||
}
|
||
|
||
if c == 61 {
|
||
// '=' = 61
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "EqEq", "==")
|
||
let i = i + 2
|
||
} else {
|
||
if peek_c == 62 {
|
||
// '>' = 62
|
||
let tokens = tok_append(tokens, "FatArrow", "=>")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Eq", "=")
|
||
let i = i + 1
|
||
}
|
||
}
|
||
} else {
|
||
if c == 33 {
|
||
// '!' = 33
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "NotEq", "!=")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Not", "!")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 60 {
|
||
// '<' = 60
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "LtEq", "<=")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Lt", "<")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 62 {
|
||
// '>' = 62
|
||
if peek_c == 61 {
|
||
let tokens = tok_append(tokens, "GtEq", ">=")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Gt", ">")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 38 {
|
||
// '&' = 38
|
||
if peek_c == 38 {
|
||
let tokens = tok_append(tokens, "And", "&&")
|
||
let i = i + 2
|
||
} else {
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 124 {
|
||
// '|' = 124
|
||
if peek_c == 124 {
|
||
let tokens = tok_append(tokens, "Or", "||")
|
||
let i = i + 2
|
||
} else {
|
||
if peek_c == 62 {
|
||
// '>' = 62
|
||
let tokens = tok_append(tokens, "PipeOp", "|>")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Pipe", "|")
|
||
let i = i + 1
|
||
}
|
||
}
|
||
} else {
|
||
if c == 45 {
|
||
// '-' = 45
|
||
if peek_c == 62 {
|
||
// '>' = 62
|
||
let tokens = tok_append(tokens, "Arrow", "->")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Minus", "-")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 58 {
|
||
// ':' = 58
|
||
if peek_c == 58 {
|
||
let tokens = tok_append(tokens, "ColonColon", "::")
|
||
let i = i + 2
|
||
} else {
|
||
let tokens = tok_append(tokens, "Colon", ":")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 43 {
|
||
// '+' = 43
|
||
let tokens = tok_append(tokens, "Plus", "+")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 42 {
|
||
// '*' = 42
|
||
let tokens = tok_append(tokens, "Star", "*")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 37 {
|
||
// '%' = 37
|
||
let tokens = tok_append(tokens, "Percent", "%")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 40 {
|
||
// '(' = 40
|
||
let tokens = tok_append(tokens, "LParen", "(")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 41 {
|
||
// ')' = 41
|
||
let tokens = tok_append(tokens, "RParen", ")")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 123 {
|
||
// '{' = 123
|
||
let tokens = tok_append(tokens, "LBrace", "{")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 125 {
|
||
// '}' = 125
|
||
let tokens = tok_append(tokens, "RBrace", "}")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 91 {
|
||
// '[' = 91
|
||
let tokens = tok_append(tokens, "LBracket", "[")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 93 {
|
||
// ']' = 93
|
||
let tokens = tok_append(tokens, "RBracket", "]")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 44 {
|
||
// ',' = 44
|
||
let tokens = tok_append(tokens, "Comma", ",")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 46 {
|
||
// '.' = 46: check for ..= or ..
|
||
let peek2_i = i + 2
|
||
let peek2_c: Int = -1
|
||
if peek2_i < total {
|
||
let peek2_c: Int = str_char_code(source, peek2_i)
|
||
}
|
||
if peek_c == 46 {
|
||
// '..' prefix
|
||
if peek2_c == 61 {
|
||
// '..=' = 46 46 61
|
||
let tokens = tok_append(tokens, "DotDotEq", "..=")
|
||
let i = i + 3
|
||
} else {
|
||
let tokens = tok_append(tokens, "DotDot", "..")
|
||
let i = i + 2
|
||
}
|
||
} else {
|
||
let tokens = tok_append(tokens, "Dot", ".")
|
||
let i = i + 1
|
||
}
|
||
} else {
|
||
if c == 59 {
|
||
// ';' = 59
|
||
let tokens = tok_append(tokens, "Semicolon", ";")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 64 {
|
||
// '@' = 64
|
||
let tokens = tok_append(tokens, "At", "@")
|
||
let i = i + 1
|
||
} else {
|
||
if c == 63 {
|
||
// '?' = 63
|
||
let tokens = tok_append(tokens, "QuestionMark", "?")
|
||
let i = i + 1
|
||
} else {
|
||
// unknown char - skip
|
||
let i = i + 1
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
let tokens = tok_append(tokens, "Eof", "")
|
||
tokens
|
||
}
|