// lexer.el - el self-hosting lexer // // Tokenises an el source string into a list of token maps. // Each token is a Map with keys: // "kind" -> String (e.g. "Int", "Ident", "Plus") // "value" -> String (the raw text of the token) // // Entry point: fn lex(source: String) -> [Map] // // Performance: the hot lexer loop uses str_char_code (returns Int) instead of // str_char_at (returns strdup'd String) for character classification. // For a 400KB source, str_char_at allocates ~400K × 16B = ~6.4MB of temporary // strings for the `ch` variable alone. str_char_code avoids all that. // -- Character helpers (Int-based, no string allocation) ---------------------- // These operate on char codes (from str_char_code) instead of str_char_at, // eliminating one strdup per character in the hot lexer loop. fn is_digit_code(c: Int) -> Bool { // '0'=48 .. '9'=57 if c >= 48 { if c <= 57 { return true } } false } fn is_alpha_code(c: Int) -> Bool { // 'A'=65..'Z'=90, 'a'=97..'z'=122 if c >= 65 { if c <= 90 { return true } } if c >= 97 { if c <= 122 { return true } } false } fn is_alnum_or_underscore_code(c: Int) -> Bool { if is_digit_code(c) { return true } if is_alpha_code(c) { return true } if c == 95 { return true } // '_' false } fn is_ws_code(c: Int) -> Bool { if c == 32 { return true } // ' ' if c == 9 { return true } // '\t' if c == 10 { return true } // '\n' if c == 13 { return true } // '\r' false } // Legacy String-based helpers kept for scan_interp helpers that use str_char_at. fn lex_is_digit(ch: String) -> Bool { if ch == "0" { return true } if ch == "1" { return true } if ch == "2" { return true } if ch == "3" { return true } if ch == "4" { return true } if ch == "5" { return true } if ch == "6" { return true } if ch == "7" { return true } if ch == "8" { return true } if ch == "9" { return true } false } fn lex_is_alpha(ch: String) -> Bool { if ch == "a" { return true } if ch == "b" { return true } if ch == "c" { return true } if ch == "d" { return true } if ch == "e" { return true } if ch == "f" { return true } if ch == "g" { return true } if ch == "h" { return true } if ch == "i" { return true } if ch == "j" { return true } if ch == "k" { return true } if ch == "l" { return true } if ch == "m" { return true } if ch == "n" { return true } if ch == "o" { return true } if ch == "p" { return true } if ch == "q" { return true } if ch == "r" { return true } if ch == "s" { return true } if ch == "t" { return true } if ch == "u" { return true } if ch == "v" { return true } if ch == "w" { return true } if ch == "x" { return true } if ch == "y" { return true } if ch == "z" { return true } if ch == "A" { return true } if ch == "B" { return true } if ch == "C" { return true } if ch == "D" { return true } if ch == "E" { return true } if ch == "F" { return true } if ch == "G" { return true } if ch == "H" { return true } if ch == "I" { return true } if ch == "J" { return true } if ch == "K" { return true } if ch == "L" { return true } if ch == "M" { return true } if ch == "N" { return true } if ch == "O" { return true } if ch == "P" { return true } if ch == "Q" { return true } if ch == "R" { return true } if ch == "S" { return true } if ch == "T" { return true } if ch == "U" { return true } if ch == "V" { return true } if ch == "W" { return true } if ch == "X" { return true } if ch == "Y" { return true } if ch == "Z" { return true } false } fn is_alnum_or_underscore(ch: String) -> Bool { if lex_is_digit(ch) { return true } if lex_is_alpha(ch) { return true } if ch == "_" { return true } false } fn lex_is_whitespace(ch: String) -> Bool { if ch == " " { return true } if ch == "\t" { return true } if ch == "\n" { return true } if ch == "\r" { return true } false } // tok_append — append a (kind, value) pair to a flat token list. // Returns the updated list. Gamma combines flat-list + char-code for max savings. fn tok_append(tokens: [Any], kind: String, value: String) -> [Any] { let tokens = native_list_append(tokens, kind) native_list_append(tokens, value) } // -- Keyword lookup ------------------------------------------------------------ fn keyword_kind(word: String) -> String { if word == "let" { return "Let" } if word == "fn" { return "Fn" } if word == "type" { return "Type" } if word == "enum" { return "Enum" } if word == "match" { return "Match" } if word == "return" { return "Return" } if word == "if" { return "If" } if word == "else" { return "Else" } if word == "for" { return "For" } if word == "in" { return "In" } if word == "while" { return "While" } if word == "import" { return "Import" } if word == "from" { return "From" } if word == "as" { return "As" } if word == "with" { return "With" } if word == "sealed" { return "Sealed" } if word == "activate" { return "Activate" } if word == "where" { return "Where" } if word == "test" { return "Test" } if word == "seed" { return "Seed" } if word == "assert" { return "Assert" } if word == "protocol" { return "Protocol" } if word == "impl" { return "Impl" } if word == "retry" { return "Retry" } if word == "times" { return "Times" } if word == "fallback" { return "Fallback" } if word == "reason" { return "Reason" } if word == "parallel" { return "Parallel" } if word == "trace" { return "Trace" } if word == "requires" { return "Requires" } if word == "deploy" { return "Deploy" } if word == "to" { return "To" } if word == "via" { return "Via" } if word == "target" { return "Target" } if word == "true" { return "Bool" } if word == "false" { return "Bool" } if word == "cgi" { return "Cgi" } if word == "service" { return "Service" } if word == "manager" { return "Manager" } if word == "engine" { return "Engine" } if word == "accessor" { return "Accessor" } if word == "vessel" { return "Vessel" } if word == "extern" { return "Extern" } if word == "break" { return "Break" } if word == "continue" { return "Continue" } "" } // -- Scan helpers -------------------------------------------------------------- // All scan helpers receive the chars list and total length. // scan_digits - advance i while chars[i] is a digit // Returns { "text": ..., "pos": i } fn scan_digits(src: String, start: Int, total: Int) -> Map { let i = start let running = true while running { if i >= total { let running = false } else { let c: Int = str_char_code(src, i) if is_digit_code(c) { let i = i + 1 } else { let running = false } } } // Use str_slice instead of building a parts list — O(1) allocation, O(n) copy. { "text": str_slice(src, start, i), "pos": i } } // scan_ident - advance i while chars[i] is alphanumeric or underscore fn scan_ident(src: String, start: Int, total: Int) -> Map { let i = start let running = true while running { if i >= total { let running = false } else { let c: Int = str_char_code(src, i) if is_alnum_or_underscore_code(c) { let i = i + 1 } else { let running = false } } } // Use str_slice instead of building a parts list — O(1) allocation, O(n) copy. { "text": str_slice(src, start, i), "pos": i } } // -- Code-bearing string detection + comment strip ---------------------------- // Inline JS/CSS literals embedded in El source (e.g. blobs // or stylesheet payloads inside string literals) carry their own line and // block comments. Those comments leak into the served HTML and reveal build // notes the visitor should never see. We strip them at the lexer so every // downstream consumer (codegen-c, codegen-js, parser) gets the cleaned form. // // looks_like_code - heuristic gate so we only strip strings that actually // embed JS or CSS. Plain prose, hex blobs, JSON, etc. pass through verbatim. fn substr_at(src: String, start: Int, total: Int, needle: String) -> Bool { let nlen: Int = str_len(needle) if start + nlen > total { return false } // Use str_slice comparison instead of char-by-char loop. str_eq(str_slice(src, start, start + nlen), needle) } fn str_has(s: String, needle: String) -> Bool { // Use the built-in str_contains which is implemented in native C — O(n) single pass. str_contains(s, needle) } fn looks_like_code(s: String) -> Bool { if str_has(s, " String { let total: Int = str_len(s) let out_parts: [String] = native_list_empty() let i = 0 let in_squote = false let in_dquote = false let in_btick = false let prev = "" while i < total { let ch: String = str_char_at(s, i) let in_js_string = false if in_squote { let in_js_string = true } if in_dquote { let in_js_string = true } if in_btick { let in_js_string = true } if in_js_string { // Backslash escape: consume next char verbatim regardless of which. if ch == "\\" { let out_parts = native_list_append(out_parts, ch) let next_i = i + 1 if next_i < total { let nc: String = str_char_at(s, next_i) let out_parts = native_list_append(out_parts, nc) let prev = nc let i = next_i + 1 } else { let prev = ch let i = next_i } } else { if in_squote { if ch == "'" { let in_squote = false } } else { if in_dquote { if ch == "\"" { let in_dquote = false } } else { if in_btick { if ch == "`" { let in_btick = false } } } } let out_parts = native_list_append(out_parts, ch) let prev = ch let i = i + 1 } } else { // Not in a JS string. Check for comment openers. let next_i = i + 1 let next_ch = "" if next_i < total { let next_ch: String = str_char_at(s, next_i) } if ch == "/" { if next_ch == "/" { // URL guard: prev char ':' means this is "://", not a comment. if prev == ":" { let out_parts = native_list_append(out_parts, ch) let prev = ch let i = i + 1 } else { // Skip until newline (newline itself is preserved so // surrounding line counts/structure stay sane). let i = i + 2 let scanning = true while scanning { if i >= total { let scanning = false } else { let lc: String = str_char_at(s, i) if lc == "\n" { let scanning = false } else { let i = i + 1 } } } let prev = "" } } else { if next_ch == "*" { // Skip until matching "*/". let i = i + 2 let scanning2 = true while scanning2 { if i >= total { let scanning2 = false } else { let bc: String = str_char_at(s, i) if bc == "*" { let after = i + 1 if after < total { let nc2: String = str_char_at(s, after) if nc2 == "/" { let i = after + 1 let scanning2 = false } else { let i = i + 1 } } else { let i = i + 1 } } else { let i = i + 1 } } } let prev = "" } else { let out_parts = native_list_append(out_parts, ch) let prev = ch let i = i + 1 } } } else { // Open a JS string? if ch == "'" { let in_squote = true let out_parts = native_list_append(out_parts, ch) let prev = ch let i = i + 1 } else { if ch == "\"" { let in_dquote = true let out_parts = native_list_append(out_parts, ch) let prev = ch let i = i + 1 } else { if ch == "`" { let in_btick = true let out_parts = native_list_append(out_parts, ch) let prev = ch let i = i + 1 } else { let out_parts = native_list_append(out_parts, ch) let prev = ch let i = i + 1 } } } } } } str_join(out_parts, "") } // scan_string - scan a quoted string literal, handling \" escapes. // Starts AFTER the opening quote. Returns { "text": content, "pos": i_after_close } fn scan_string(src: String, start: Int, total: Int) -> Map { let i = start let parts: [String] = native_list_empty() let running = true while running { if i >= total { let running = false } else { let ch: String = str_char_at(src, i) if ch == "\\" { // escape: peek next char let next_i = i + 1 if next_i < total { let next_ch: String = str_char_at(src, next_i) if next_ch == "\"" { let parts = native_list_append(parts, "\"") let i = next_i + 1 } else { if next_ch == "n" { let parts = native_list_append(parts, "\n") let i = next_i + 1 } else { if next_ch == "t" { let parts = native_list_append(parts, "\t") let i = next_i + 1 } else { if next_ch == "r" { let parts = native_list_append(parts, "\r") let i = next_i + 1 } else { if next_ch == "\\" { let parts = native_list_append(parts, "\\") let i = next_i + 1 } else { let parts = native_list_append(parts, next_ch) let i = next_i + 1 } } } } } } else { let i = i + 1 } } else { if ch == "\"" { let i = i + 1 let running = false } else { let parts = native_list_append(parts, ch) let i = i + 1 } } } } { "text": str_join(parts, ""), "pos": i } } // -- String interpolation ------------------------------------------------------ // // scan_interp_brace - scan from `start` (the char after `${`) to the matching // `}`, tracking brace depth so inner braces (e.g. fn calls, map literals) are // handled correctly. Returns { "text": inner_source, "pos": i_after_close }. fn scan_interp_brace(src: String, start: Int, total: Int) -> Map { let i = start let depth = 1 let running = true while running { if i >= total { let running = false } else { let ch: String = str_char_at(src, i) if ch == "{" { let depth = depth + 1 let i = i + 1 } else { if ch == "}" { let depth = depth - 1 if depth <= 0 { // Closing brace of the interpolation - stop, do not include it let i = i + 1 let running = false } else { let i = i + 1 } } else { let i = i + 1 } } } } // Use str_slice instead of parts list — the inner source is a contiguous substring. { "text": str_slice(src, start, i - 1), "pos": i } } // interp_tokens_append_all - copy every (kind, value) pair from flat src list // into flat dst list, skipping the trailing Eof pair that lex() always appends. fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] { let src_len: Int = native_list_len(src) let j = 0 let result = dst while j < src_len { let kind: String = native_list_get(src, j) if kind == "Eof" { let j = src_len } else { let val: String = native_list_get(src, j + 1) let result = native_list_append(result, kind) let result = native_list_append(result, val) let j = j + 2 } } result } // scan_interp_string - scan a string literal that may contain ${expr} // interpolations. Starts AFTER the opening `"`. // Returns { "tokens": [token list to inject], "pos": i_after_close_quote }. // // For a plain string (no ${}) this emits a single Str token, identical to the // old scan_string path. For an interpolated string it emits a flat sequence // of tokens equivalent to the string-concat expression, for example: // // "hello ${name}!" // => Str("hello ") Plus Plus Str("!") // // Empty literal segments between adjacent ${ } blocks are omitted. The // resulting token stream is consumed by the existing parse_binop / parse_primary // path in the parser with zero parser changes required. // // Supported escape sequences: \" \n \t \r \\ \$ (literal dollar sign). // Nested quotes inside ${} are not supported; use a variable instead. // // Performance: uses str_char_code (Int) for all character dispatch, eliminating // per-character strdup. Plain runs are batched into str_slice segments instead // of accumulating single-char strings, reducing list appends from O(N) to O(K) // where K = number of escape/special chars in the literal. // Char codes: '\' = 92, '"' = 34, '$' = 36, '{' = 123 fn scan_interp_string(src: String, start: Int, total: Int) -> Map { let i = start let out_tokens: [Any] = native_list_empty() let cur_parts: [String] = native_list_empty() let clean_start = start let has_interp = false let need_plus = false let running = true while running { if i >= total { let running = false } else { let c: Int = str_char_code(src, i) if c == 92 { // '\\' = 92 — escape sequence: flush clean run, append resolved char if clean_start < i { let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i)) } let next_i = i + 1 if next_i < total { let nc: Int = str_char_code(src, next_i) if nc == 36 { // '\$' => literal '$' (36 = '$') let cur_parts = native_list_append(cur_parts, "$") let clean_start = next_i + 1 let i = next_i + 1 } else { if nc == 34 { // '\"' => literal '"' (34 = '"') let cur_parts = native_list_append(cur_parts, "\"") let clean_start = next_i + 1 let i = next_i + 1 } else { if nc == 110 { // '\n' (110 = 'n') let cur_parts = native_list_append(cur_parts, "\n") let clean_start = next_i + 1 let i = next_i + 1 } else { if nc == 116 { // '\t' (116 = 't') let cur_parts = native_list_append(cur_parts, "\t") let clean_start = next_i + 1 let i = next_i + 1 } else { if nc == 114 { // '\r' (114 = 'r') let cur_parts = native_list_append(cur_parts, "\r") let clean_start = next_i + 1 let i = next_i + 1 } else { if nc == 92 { // '\\' (92) let cur_parts = native_list_append(cur_parts, "\\") let clean_start = next_i + 1 let i = next_i + 1 } else { // Unknown escape: emit the escaped char verbatim let cur_parts = native_list_append(cur_parts, str_slice(src, next_i, next_i + 1)) let clean_start = next_i + 1 let i = next_i + 1 } } } } } } } else { let clean_start = next_i let i = next_i } } else { if c == 34 { // '"' = 34 — closing quote: flush clean run, stop if clean_start < i { let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i)) } let i = i + 1 let clean_start = i let running = false } else { if c == 36 { // '$' = 36 — possible interpolation start let next_i = i + 1 let is_interp = false if next_i < total { let nc2: Int = str_char_code(src, next_i) if nc2 == 123 { // '{' = 123 let is_interp = true } } if is_interp { // Flush the accumulated literal part (if non-empty) if clean_start < i { let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i)) } let part_len: Int = native_list_len(cur_parts) if part_len > 0 { let part_text = str_join(cur_parts, "") if need_plus { let out_tokens = tok_append(out_tokens, "Plus", "+") } let clean_part = part_text if looks_like_code(part_text) { let clean_part = strip_code_comments(part_text) } let out_tokens = tok_append(out_tokens, "Str", clean_part) let need_plus = true } let cur_parts = native_list_empty() let has_interp = true // Scan brace-balanced expression source let brace_result = scan_interp_brace(src, next_i + 1, total) let expr_src: String = brace_result["text"] let new_i: Int = brace_result["pos"] let i = new_i let clean_start = new_i // Re-lex the expression and inline the tokens. // Wrap in ( ) so that operators inside ${} (e.g. // age + 1) are parsed as a grouped sub-expression // rather than merging with the surrounding concat // Plus tokens at the wrong precedence level. let inner_toks: [Any] = lex(expr_src) let inner_len: Int = native_list_len(inner_toks) if need_plus { let out_tokens = tok_append(out_tokens, "Plus", "+") } // Empty interpolation ${} => empty string segment // inner_len <= 2 = only the Eof pair (kind="Eof", value="") if inner_len <= 2 { let out_tokens = tok_append(out_tokens, "Str", "") } else { let out_tokens = tok_append(out_tokens, "LParen", "(") let out_tokens = interp_tokens_append_all(out_tokens, inner_toks) let out_tokens = tok_append(out_tokens, "RParen", ")") } let need_plus = true } else { // Plain '$' not followed by '{' - treat as literal, continue clean run let i = i + 1 } } else { // Plain char — extends clean run, no append needed let i = i + 1 } } } } } // Flush remaining literal segment and build final token list if clean_start < i { let cur_parts = native_list_append(cur_parts, str_slice(src, clean_start, i)) } let part_len: Int = native_list_len(cur_parts) let part_text = str_join(cur_parts, "") if has_interp { // Interpolated string: only emit trailing segment if non-empty if part_len > 0 { let clean_part = part_text if looks_like_code(part_text) { let clean_part = strip_code_comments(part_text) } if need_plus { let out_tokens = tok_append(out_tokens, "Plus", "+") } let out_tokens = tok_append(out_tokens, "Str", clean_part) } } else { // Plain string with no interpolation - same behaviour as old scan_string let clean_text = part_text if looks_like_code(part_text) { let clean_text = strip_code_comments(part_text) } let out_tokens = tok_append(out_tokens, "Str", clean_text) } { "tokens": out_tokens, "pos": i } } // -- Main lexer ---------------------------------------------------------------- // Char code constants (avoids strdup for single-char comparison) // '/' = 47, '"' = 34, '0'-'9' = 48-57, 'a'-'z' = 97-122, 'A'-'Z' = 65-90 // '_' = 95, ' '=32, '\t'=9, '\n'=10, '\r'=13 // '=' = 61, '!' = 33, '<' = 60, '>' = 62, '&' = 38, '|' = 124 // '-' = 45, ':' = 58, '+' = 43, '*' = 42, '%' = 37 // '(' = 40, ')' = 41, '{' = 123, '}' = 125, '[' = 91, ']' = 93 // ',' = 44, '.' = 46, ';' = 59, '@' = 64, '?' = 63 fn lex(source: String) -> [Any] { // Use str_char_code (returns Int) instead of str_char_at (returns strdup String) // for all character classification in the hot loop. For a 400KB source, // str_char_at allocates ~400K × 16B = ~6.4MB of temporary strings. let total: Int = str_len(source) let tokens: [Any] = native_list_empty() let i: Int = 0 while i < total { let c: Int = str_char_code(source, i) // Skip whitespace (space=32, tab=9, newline=10, CR=13) if is_ws_code(c) { let i = i + 1 } else { // Line comments: // (slash=47) if c == 47 { let next_i = i + 1 if next_i < total { let nc: Int = str_char_code(source, next_i) if nc == 47 { // skip to end of line (newline=10) let i = i + 2 let running2 = true while running2 { if i >= total { let running2 = false } else { let lc: Int = str_char_code(source, i) if lc == 10 { let running2 = false } else { let i = i + 1 } } } } else { let tokens = tok_append(tokens, "Slash", "/") let i = i + 1 } } else { let tokens = tok_append(tokens, "Slash", "/") let i = i + 1 } } else { // String literal: '"' = 34 if c == 34 { let interp_result = scan_interp_string(source, i + 1, total) let interp_toks: [Any] = interp_result["tokens"] let new_pos: Int = interp_result["pos"] let tokens = interp_tokens_append_all(tokens, interp_toks) let i = new_pos } else { // Number literal: '0'-'9' = 48-57 if is_digit_code(c) { let result = scan_digits(source, i, total) let num_text: String = result["text"] let new_pos: Int = result["pos"] // check for float (dot=46 followed by digit) if new_pos < total { let dc: Int = str_char_code(source, new_pos) if dc == 46 { let after_dot = new_pos + 1 if after_dot < total { let adc: Int = str_char_code(source, after_dot) if is_digit_code(adc) { let frac_result = scan_digits(source, after_dot, total) let frac_text: String = frac_result["text"] let frac_pos: Int = frac_result["pos"] let tokens = tok_append(tokens, "Float", num_text + "." + frac_text) let i = frac_pos } else { let tokens = tok_append(tokens, "Int", num_text) let i = new_pos } } else { let tokens = tok_append(tokens, "Int", num_text) let i = new_pos } } else { let tokens = tok_append(tokens, "Int", num_text) let i = new_pos } } else { let tokens = tok_append(tokens, "Int", num_text) let i = new_pos } } else { // Identifier or keyword: alpha or '_'=95 if is_alpha_code(c) || c == 95 { let result = scan_ident(source, i, total) let word: String = result["text"] let new_pos: Int = result["pos"] let kw = keyword_kind(word) if kw == "" { let tokens = tok_append(tokens, "Ident", word) } else { let tokens = tok_append(tokens, kw, word) } let i = new_pos } else { // Multi-char and single-char operators/delimiters let peek_i = i + 1 let peek_c: Int = -1 if peek_i < total { let peek_c: Int = str_char_code(source, peek_i) } if c == 61 { // '=' = 61 if peek_c == 61 { let tokens = tok_append(tokens, "EqEq", "==") let i = i + 2 } else { if peek_c == 62 { // '>' = 62 let tokens = tok_append(tokens, "FatArrow", "=>") let i = i + 2 } else { let tokens = tok_append(tokens, "Eq", "=") let i = i + 1 } } } else { if c == 33 { // '!' = 33 if peek_c == 61 { let tokens = tok_append(tokens, "NotEq", "!=") let i = i + 2 } else { let tokens = tok_append(tokens, "Not", "!") let i = i + 1 } } else { if c == 60 { // '<' = 60 if peek_c == 61 { let tokens = tok_append(tokens, "LtEq", "<=") let i = i + 2 } else { let tokens = tok_append(tokens, "Lt", "<") let i = i + 1 } } else { if c == 62 { // '>' = 62 if peek_c == 61 { let tokens = tok_append(tokens, "GtEq", ">=") let i = i + 2 } else { let tokens = tok_append(tokens, "Gt", ">") let i = i + 1 } } else { if c == 38 { // '&' = 38 if peek_c == 38 { let tokens = tok_append(tokens, "And", "&&") let i = i + 2 } else { let i = i + 1 } } else { if c == 124 { // '|' = 124 if peek_c == 124 { let tokens = tok_append(tokens, "Or", "||") let i = i + 2 } else { if peek_c == 62 { // '>' = 62 let tokens = tok_append(tokens, "PipeOp", "|>") let i = i + 2 } else { let tokens = tok_append(tokens, "Pipe", "|") let i = i + 1 } } } else { if c == 45 { // '-' = 45 if peek_c == 62 { // '>' = 62 let tokens = tok_append(tokens, "Arrow", "->") let i = i + 2 } else { let tokens = tok_append(tokens, "Minus", "-") let i = i + 1 } } else { if c == 58 { // ':' = 58 if peek_c == 58 { let tokens = tok_append(tokens, "ColonColon", "::") let i = i + 2 } else { let tokens = tok_append(tokens, "Colon", ":") let i = i + 1 } } else { if c == 43 { // '+' = 43 let tokens = tok_append(tokens, "Plus", "+") let i = i + 1 } else { if c == 42 { // '*' = 42 let tokens = tok_append(tokens, "Star", "*") let i = i + 1 } else { if c == 37 { // '%' = 37 let tokens = tok_append(tokens, "Percent", "%") let i = i + 1 } else { if c == 40 { // '(' = 40 let tokens = tok_append(tokens, "LParen", "(") let i = i + 1 } else { if c == 41 { // ')' = 41 let tokens = tok_append(tokens, "RParen", ")") let i = i + 1 } else { if c == 123 { // '{' = 123 let tokens = tok_append(tokens, "LBrace", "{") let i = i + 1 } else { if c == 125 { // '}' = 125 let tokens = tok_append(tokens, "RBrace", "}") let i = i + 1 } else { if c == 91 { // '[' = 91 let tokens = tok_append(tokens, "LBracket", "[") let i = i + 1 } else { if c == 93 { // ']' = 93 let tokens = tok_append(tokens, "RBracket", "]") let i = i + 1 } else { if c == 44 { // ',' = 44 let tokens = tok_append(tokens, "Comma", ",") let i = i + 1 } else { if c == 46 { // '.' = 46: check for ..= or .. let peek2_i = i + 2 let peek2_c: Int = -1 if peek2_i < total { let peek2_c: Int = str_char_code(source, peek2_i) } if peek_c == 46 { // '..' prefix if peek2_c == 61 { // '..=' = 46 46 61 let tokens = tok_append(tokens, "DotDotEq", "..=") let i = i + 3 } else { let tokens = tok_append(tokens, "DotDot", "..") let i = i + 2 } } else { let tokens = tok_append(tokens, "Dot", ".") let i = i + 1 } } else { if c == 59 { // ';' = 59 let tokens = tok_append(tokens, "Semicolon", ";") let i = i + 1 } else { if c == 64 { // '@' = 64 let tokens = tok_append(tokens, "At", "@") let i = i + 1 } else { if c == 63 { // '?' = 63 let tokens = tok_append(tokens, "QuestionMark", "?") let i = i + 1 } else { // unknown char - skip let i = i + 1 } } } } } } } } } } } } } } } } } } } } } } } } } } } } let tokens = tok_append(tokens, "Eof", "") tokens }