Will Anderson
990ce72539
scan_string() is the right gate for this: every El source that embeds JS or CSS does so as a quoted string literal, and the lexer is the single chokepoint every backend reads. Strip there and the // line comments and /* */ block comments never reach the parser, codegen, or the served HTML. looks_like_code is intentionally narrow: - contains "<script" or "<style" (the embedded-asset case), or - contains "function" AND ";" (a JS body without an opening tag) Plain prose with stray // sequences passes through verbatim. strip_code_comments tracks JS string state (single, double, backtick) and never strips inside one. Backslash escapes inside JS strings consume the next char verbatim. URL guard: when the char before / is ':', emit the / literally and advance one — preserves https:// inside string literals. Block-comment scan walks until the matching '*/' pair. elc-cli.el is now a one-line `import "el-compiler/src/compiler.el"` shim. Top-level `let _argv = args()` was clashing with C int main()'s `char** _argv` parameter once compiler.el's fn main() body got folded into C main. compiler.el owns the CLI entry point now. Self-host fixed point reached: gen2 == gen3 byte-identical. Tagged dist/platform/elc.20260502-1104-self-host alongside dist/platform/elc.
747 lines
36 KiB
EmacsLisp
747 lines
36 KiB
EmacsLisp
// lexer.el — el self-hosting lexer
|
|
//
|
|
// Tokenises an el source string into a list of token maps.
|
|
// Each token is a Map<String, Any> with keys:
|
|
// "kind" -> String (e.g. "Int", "Ident", "Plus")
|
|
// "value" -> String (the raw text of the token)
|
|
//
|
|
// Entry point: fn lex(source: String) -> [Map<String, Any>]
|
|
//
|
|
// Uses native_string_chars to split the source into a chars list,
|
|
// then indexes it with native_list_get — avoids O(N²) string cloning.
|
|
|
|
// ── Character helpers ─────────────────────────────────────────────────────────
|
|
|
|
fn is_digit(ch: String) -> Bool {
|
|
if ch == "0" { return true }
|
|
if ch == "1" { return true }
|
|
if ch == "2" { return true }
|
|
if ch == "3" { return true }
|
|
if ch == "4" { return true }
|
|
if ch == "5" { return true }
|
|
if ch == "6" { return true }
|
|
if ch == "7" { return true }
|
|
if ch == "8" { return true }
|
|
if ch == "9" { return true }
|
|
false
|
|
}
|
|
|
|
fn is_alpha(ch: String) -> Bool {
|
|
if ch == "a" { return true }
|
|
if ch == "b" { return true }
|
|
if ch == "c" { return true }
|
|
if ch == "d" { return true }
|
|
if ch == "e" { return true }
|
|
if ch == "f" { return true }
|
|
if ch == "g" { return true }
|
|
if ch == "h" { return true }
|
|
if ch == "i" { return true }
|
|
if ch == "j" { return true }
|
|
if ch == "k" { return true }
|
|
if ch == "l" { return true }
|
|
if ch == "m" { return true }
|
|
if ch == "n" { return true }
|
|
if ch == "o" { return true }
|
|
if ch == "p" { return true }
|
|
if ch == "q" { return true }
|
|
if ch == "r" { return true }
|
|
if ch == "s" { return true }
|
|
if ch == "t" { return true }
|
|
if ch == "u" { return true }
|
|
if ch == "v" { return true }
|
|
if ch == "w" { return true }
|
|
if ch == "x" { return true }
|
|
if ch == "y" { return true }
|
|
if ch == "z" { return true }
|
|
if ch == "A" { return true }
|
|
if ch == "B" { return true }
|
|
if ch == "C" { return true }
|
|
if ch == "D" { return true }
|
|
if ch == "E" { return true }
|
|
if ch == "F" { return true }
|
|
if ch == "G" { return true }
|
|
if ch == "H" { return true }
|
|
if ch == "I" { return true }
|
|
if ch == "J" { return true }
|
|
if ch == "K" { return true }
|
|
if ch == "L" { return true }
|
|
if ch == "M" { return true }
|
|
if ch == "N" { return true }
|
|
if ch == "O" { return true }
|
|
if ch == "P" { return true }
|
|
if ch == "Q" { return true }
|
|
if ch == "R" { return true }
|
|
if ch == "S" { return true }
|
|
if ch == "T" { return true }
|
|
if ch == "U" { return true }
|
|
if ch == "V" { return true }
|
|
if ch == "W" { return true }
|
|
if ch == "X" { return true }
|
|
if ch == "Y" { return true }
|
|
if ch == "Z" { return true }
|
|
false
|
|
}
|
|
|
|
fn is_alnum_or_underscore(ch: String) -> Bool {
|
|
if is_digit(ch) { return true }
|
|
if is_alpha(ch) { return true }
|
|
if ch == "_" { return true }
|
|
false
|
|
}
|
|
|
|
fn is_whitespace(ch: String) -> Bool {
|
|
if ch == " " { return true }
|
|
if ch == "\t" { return true }
|
|
if ch == "\n" { return true }
|
|
if ch == "\r" { return true }
|
|
false
|
|
}
|
|
|
|
fn make_tok(kind: String, value: String) -> Map<String, Any> {
|
|
{ "kind": kind, "value": value }
|
|
}
|
|
|
|
// ── Keyword lookup ────────────────────────────────────────────────────────────
|
|
|
|
fn keyword_kind(word: String) -> String {
|
|
if word == "let" { return "Let" }
|
|
if word == "fn" { return "Fn" }
|
|
if word == "type" { return "Type" }
|
|
if word == "enum" { return "Enum" }
|
|
if word == "match" { return "Match" }
|
|
if word == "return" { return "Return" }
|
|
if word == "if" { return "If" }
|
|
if word == "else" { return "Else" }
|
|
if word == "for" { return "For" }
|
|
if word == "in" { return "In" }
|
|
if word == "while" { return "While" }
|
|
if word == "import" { return "Import" }
|
|
if word == "from" { return "From" }
|
|
if word == "as" { return "As" }
|
|
if word == "with" { return "With" }
|
|
if word == "sealed" { return "Sealed" }
|
|
if word == "activate" { return "Activate" }
|
|
if word == "where" { return "Where" }
|
|
if word == "test" { return "Test" }
|
|
if word == "seed" { return "Seed" }
|
|
if word == "assert" { return "Assert" }
|
|
if word == "protocol" { return "Protocol" }
|
|
if word == "impl" { return "Impl" }
|
|
if word == "retry" { return "Retry" }
|
|
if word == "times" { return "Times" }
|
|
if word == "fallback" { return "Fallback" }
|
|
if word == "reason" { return "Reason" }
|
|
if word == "parallel" { return "Parallel" }
|
|
if word == "trace" { return "Trace" }
|
|
if word == "requires" { return "Requires" }
|
|
if word == "deploy" { return "Deploy" }
|
|
if word == "to" { return "To" }
|
|
if word == "via" { return "Via" }
|
|
if word == "target" { return "Target" }
|
|
if word == "true" { return "Bool" }
|
|
if word == "false" { return "Bool" }
|
|
if word == "cgi" { return "Cgi" }
|
|
if word == "service" { return "Service" }
|
|
if word == "manager" { return "Manager" }
|
|
if word == "engine" { return "Engine" }
|
|
if word == "accessor" { return "Accessor" }
|
|
if word == "vessel" { return "Vessel" }
|
|
""
|
|
}
|
|
|
|
// ── Scan helpers ──────────────────────────────────────────────────────────────
|
|
// All scan helpers receive the chars list and total length.
|
|
|
|
// scan_digits — advance i while chars[i] is a digit
|
|
// Returns { "text": ..., "pos": i }
|
|
fn scan_digits(chars: [String], start: Int, total: Int) -> Map<String, Any> {
|
|
let i = start
|
|
let text = ""
|
|
let running = true
|
|
while running {
|
|
if i >= total {
|
|
let running = false
|
|
} else {
|
|
let ch: String = native_list_get(chars, i)
|
|
if is_digit(ch) {
|
|
let text = text + ch
|
|
let i = i + 1
|
|
} else {
|
|
let running = false
|
|
}
|
|
}
|
|
}
|
|
{ "text": text, "pos": i }
|
|
}
|
|
|
|
// scan_ident — advance i while chars[i] is alphanumeric or underscore
|
|
fn scan_ident(chars: [String], start: Int, total: Int) -> Map<String, Any> {
|
|
let i = start
|
|
let text = ""
|
|
let running = true
|
|
while running {
|
|
if i >= total {
|
|
let running = false
|
|
} else {
|
|
let ch: String = native_list_get(chars, i)
|
|
if is_alnum_or_underscore(ch) {
|
|
let text = text + ch
|
|
let i = i + 1
|
|
} else {
|
|
let running = false
|
|
}
|
|
}
|
|
}
|
|
{ "text": text, "pos": i }
|
|
}
|
|
|
|
// ── Code-bearing string detection + comment strip ────────────────────────────
|
|
// Inline JS/CSS literals embedded in El source (e.g. <script>…</script> blobs
|
|
// or stylesheet payloads inside string literals) carry their own line and
|
|
// block comments. Those comments leak into the served HTML and reveal build
|
|
// notes the visitor should never see. We strip them at the lexer so every
|
|
// downstream consumer (codegen-c, codegen-js, parser) gets the cleaned form.
|
|
//
|
|
// looks_like_code — heuristic gate so we only strip strings that actually
|
|
// embed JS or CSS. Plain prose, hex blobs, JSON, etc. pass through verbatim.
|
|
|
|
fn substr_at(chars: [String], start: Int, total: Int, needle: String) -> Bool {
|
|
let nchars: [String] = native_string_chars(needle)
|
|
let nlen: Int = native_list_len(nchars)
|
|
if start + nlen > total { return false }
|
|
let i = 0
|
|
let matched = true
|
|
while i < nlen {
|
|
let a: String = native_list_get(chars, start + i)
|
|
let b: String = native_list_get(nchars, i)
|
|
if a == b { let i = i + 1 } else { let matched = false; let i = nlen }
|
|
}
|
|
matched
|
|
}
|
|
|
|
fn str_has(s: String, needle: String) -> Bool {
|
|
let chars: [String] = native_string_chars(s)
|
|
let total: Int = native_list_len(chars)
|
|
let i = 0
|
|
let found = false
|
|
while i < total {
|
|
if substr_at(chars, i, total, needle) {
|
|
let found = true
|
|
let i = total
|
|
} else {
|
|
let i = i + 1
|
|
}
|
|
}
|
|
found
|
|
}
|
|
|
|
fn looks_like_code(s: String) -> Bool {
|
|
if str_has(s, "<script") { return true }
|
|
if str_has(s, "<style") { return true }
|
|
if str_has(s, "function") {
|
|
if str_has(s, ";") { return true }
|
|
}
|
|
false
|
|
}
|
|
|
|
// strip_code_comments — character-by-character walk. Tracks JS string state
|
|
// (single, double, backtick) and never strips inside one. Backslash escapes
|
|
// inside JS strings consume the next char verbatim. URLs like https:// are
|
|
// preserved by checking the previous char before treating // as a line
|
|
// comment opener: if the char immediately before '/' is ':', emit the '/'
|
|
// literally and advance one position.
|
|
fn strip_code_comments(s: String) -> String {
|
|
let chars: [String] = native_string_chars(s)
|
|
let total: Int = native_list_len(chars)
|
|
let out = ""
|
|
let i = 0
|
|
let in_squote = false
|
|
let in_dquote = false
|
|
let in_btick = false
|
|
let prev = ""
|
|
while i < total {
|
|
let ch: String = native_list_get(chars, i)
|
|
let in_js_string = false
|
|
if in_squote { let in_js_string = true }
|
|
if in_dquote { let in_js_string = true }
|
|
if in_btick { let in_js_string = true }
|
|
|
|
if in_js_string {
|
|
// Backslash escape: consume next char verbatim regardless of which.
|
|
if ch == "\\" {
|
|
let out = out + ch
|
|
let next_i = i + 1
|
|
if next_i < total {
|
|
let nc: String = native_list_get(chars, next_i)
|
|
let out = out + nc
|
|
let prev = nc
|
|
let i = next_i + 1
|
|
} else {
|
|
let prev = ch
|
|
let i = next_i
|
|
}
|
|
} else {
|
|
if in_squote {
|
|
if ch == "'" { let in_squote = false }
|
|
} else {
|
|
if in_dquote {
|
|
if ch == "\"" { let in_dquote = false }
|
|
} else {
|
|
if in_btick {
|
|
if ch == "`" { let in_btick = false }
|
|
}
|
|
}
|
|
}
|
|
let out = out + ch
|
|
let prev = ch
|
|
let i = i + 1
|
|
}
|
|
} else {
|
|
// Not in a JS string. Check for comment openers.
|
|
let next_i = i + 1
|
|
let next_ch = ""
|
|
if next_i < total {
|
|
let next_ch: String = native_list_get(chars, next_i)
|
|
}
|
|
|
|
if ch == "/" {
|
|
if next_ch == "/" {
|
|
// URL guard: prev char ':' means this is "://", not a comment.
|
|
if prev == ":" {
|
|
let out = out + ch
|
|
let prev = ch
|
|
let i = i + 1
|
|
} else {
|
|
// Skip until newline (newline itself is preserved so
|
|
// surrounding line counts/structure stay sane).
|
|
let i = i + 2
|
|
let scanning = true
|
|
while scanning {
|
|
if i >= total {
|
|
let scanning = false
|
|
} else {
|
|
let lc: String = native_list_get(chars, i)
|
|
if lc == "\n" {
|
|
let scanning = false
|
|
} else {
|
|
let i = i + 1
|
|
}
|
|
}
|
|
}
|
|
let prev = ""
|
|
}
|
|
} else {
|
|
if next_ch == "*" {
|
|
// Skip until matching "*/".
|
|
let i = i + 2
|
|
let scanning2 = true
|
|
while scanning2 {
|
|
if i >= total {
|
|
let scanning2 = false
|
|
} else {
|
|
let bc: String = native_list_get(chars, i)
|
|
if bc == "*" {
|
|
let after = i + 1
|
|
if after < total {
|
|
let nc2: String = native_list_get(chars, after)
|
|
if nc2 == "/" {
|
|
let i = after + 1
|
|
let scanning2 = false
|
|
} else {
|
|
let i = i + 1
|
|
}
|
|
} else {
|
|
let i = i + 1
|
|
}
|
|
} else {
|
|
let i = i + 1
|
|
}
|
|
}
|
|
}
|
|
let prev = ""
|
|
} else {
|
|
let out = out + ch
|
|
let prev = ch
|
|
let i = i + 1
|
|
}
|
|
}
|
|
} else {
|
|
// Open a JS string?
|
|
if ch == "'" {
|
|
let in_squote = true
|
|
let out = out + ch
|
|
let prev = ch
|
|
let i = i + 1
|
|
} else {
|
|
if ch == "\"" {
|
|
let in_dquote = true
|
|
let out = out + ch
|
|
let prev = ch
|
|
let i = i + 1
|
|
} else {
|
|
if ch == "`" {
|
|
let in_btick = true
|
|
let out = out + ch
|
|
let prev = ch
|
|
let i = i + 1
|
|
} else {
|
|
let out = out + ch
|
|
let prev = ch
|
|
let i = i + 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
// scan_string — scan a quoted string literal, handling \" escapes.
|
|
// Starts AFTER the opening quote. Returns { "text": content, "pos": i_after_close }
|
|
fn scan_string(chars: [String], start: Int, total: Int) -> Map<String, Any> {
|
|
let i = start
|
|
let text = ""
|
|
let running = true
|
|
while running {
|
|
if i >= total {
|
|
let running = false
|
|
} else {
|
|
let ch: String = native_list_get(chars, i)
|
|
if ch == "\\" {
|
|
// escape: peek next char
|
|
let next_i = i + 1
|
|
if next_i < total {
|
|
let next_ch: String = native_list_get(chars, next_i)
|
|
if next_ch == "\"" {
|
|
let text = text + "\""
|
|
let i = next_i + 1
|
|
} else {
|
|
if next_ch == "n" {
|
|
let text = text + "\n"
|
|
let i = next_i + 1
|
|
} else {
|
|
if next_ch == "t" {
|
|
let text = text + "\t"
|
|
let i = next_i + 1
|
|
} else {
|
|
if next_ch == "r" {
|
|
let text = text + "\r"
|
|
let i = next_i + 1
|
|
} else {
|
|
if next_ch == "\\" {
|
|
let text = text + "\\"
|
|
let i = next_i + 1
|
|
} else {
|
|
let text = text + next_ch
|
|
let i = next_i + 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
let i = i + 1
|
|
}
|
|
} else {
|
|
if ch == "\"" {
|
|
let i = i + 1
|
|
let running = false
|
|
} else {
|
|
let text = text + ch
|
|
let i = i + 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
{ "text": text, "pos": i }
|
|
}
|
|
|
|
// ── Main lexer ────────────────────────────────────────────────────────────────
|
|
|
|
fn lex(source: String) -> [Map<String, Any>] {
|
|
let chars: [String] = native_string_chars(source)
|
|
let total: Int = native_list_len(chars)
|
|
let tokens: [Map<String, Any>] = native_list_empty()
|
|
let i: Int = 0
|
|
|
|
while i < total {
|
|
let ch: String = native_list_get(chars, i)
|
|
|
|
// Skip whitespace
|
|
if is_whitespace(ch) {
|
|
let i = i + 1
|
|
} else {
|
|
// Line comments: //
|
|
if ch == "/" {
|
|
let next_i = i + 1
|
|
if next_i < total {
|
|
let next_ch: String = native_list_get(chars, next_i)
|
|
if next_ch == "/" {
|
|
// skip to end of line
|
|
let i = i + 2
|
|
let running2 = true
|
|
while running2 {
|
|
if i >= total {
|
|
let running2 = false
|
|
} else {
|
|
let lch: String = native_list_get(chars, i)
|
|
if lch == "\n" {
|
|
let running2 = false
|
|
} else {
|
|
let i = i + 1
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
let tokens = native_list_append(tokens, make_tok("Slash", "/"))
|
|
let i = i + 1
|
|
}
|
|
} else {
|
|
let tokens = native_list_append(tokens, make_tok("Slash", "/"))
|
|
let i = i + 1
|
|
}
|
|
} else {
|
|
// String literal
|
|
if ch == "\"" {
|
|
let result = scan_string(chars, i + 1, total)
|
|
let str_text: String = result["text"]
|
|
let new_pos: Int = result["pos"]
|
|
// Compile-time scrub: strings that embed JS or CSS get
|
|
// their // line comments and /* block comments stripped
|
|
// before the token reaches the parser. Plain prose passes
|
|
// through untouched.
|
|
let clean_text = str_text
|
|
if looks_like_code(str_text) {
|
|
let clean_text = strip_code_comments(str_text)
|
|
}
|
|
let tokens = native_list_append(tokens, make_tok("Str", clean_text))
|
|
let i = new_pos
|
|
} else {
|
|
// Number literal
|
|
if is_digit(ch) {
|
|
let result = scan_digits(chars, i, total)
|
|
let num_text: String = result["text"]
|
|
let new_pos: Int = result["pos"]
|
|
// check for float (dot followed by digit)
|
|
if new_pos < total {
|
|
let dot_ch: String = native_list_get(chars, new_pos)
|
|
if dot_ch == "." {
|
|
let after_dot = new_pos + 1
|
|
if after_dot < total {
|
|
let after_dot_ch: String = native_list_get(chars, after_dot)
|
|
if is_digit(after_dot_ch) {
|
|
let frac_result = scan_digits(chars, after_dot, total)
|
|
let frac_text: String = frac_result["text"]
|
|
let frac_pos: Int = frac_result["pos"]
|
|
let tokens = native_list_append(tokens, make_tok("Float", num_text + "." + frac_text))
|
|
let i = frac_pos
|
|
} else {
|
|
let tokens = native_list_append(tokens, make_tok("Int", num_text))
|
|
let i = new_pos
|
|
}
|
|
} else {
|
|
let tokens = native_list_append(tokens, make_tok("Int", num_text))
|
|
let i = new_pos
|
|
}
|
|
} else {
|
|
let tokens = native_list_append(tokens, make_tok("Int", num_text))
|
|
let i = new_pos
|
|
}
|
|
} else {
|
|
let tokens = native_list_append(tokens, make_tok("Int", num_text))
|
|
let i = new_pos
|
|
}
|
|
} else {
|
|
// Identifier or keyword
|
|
if is_alpha(ch) || ch == "_" {
|
|
let result = scan_ident(chars, i, total)
|
|
let word: String = result["text"]
|
|
let new_pos: Int = result["pos"]
|
|
let kw = keyword_kind(word)
|
|
if kw == "" {
|
|
let tokens = native_list_append(tokens, make_tok("Ident", word))
|
|
} else {
|
|
let tokens = native_list_append(tokens, make_tok(kw, word))
|
|
}
|
|
let i = new_pos
|
|
} else {
|
|
// Multi-char and single-char operators/delimiters
|
|
let peek_i = i + 1
|
|
let peek_ch = ""
|
|
if peek_i < total {
|
|
let peek_ch: String = native_list_get(chars, peek_i)
|
|
}
|
|
|
|
if ch == "=" {
|
|
if peek_ch == "=" {
|
|
let tokens = native_list_append(tokens, make_tok("EqEq", "=="))
|
|
let i = i + 2
|
|
} else {
|
|
if peek_ch == ">" {
|
|
let tokens = native_list_append(tokens, make_tok("FatArrow", "=>"))
|
|
let i = i + 2
|
|
} else {
|
|
let tokens = native_list_append(tokens, make_tok("Eq", "="))
|
|
let i = i + 1
|
|
}
|
|
}
|
|
} else {
|
|
if ch == "!" {
|
|
if peek_ch == "=" {
|
|
let tokens = native_list_append(tokens, make_tok("NotEq", "!="))
|
|
let i = i + 2
|
|
} else {
|
|
let tokens = native_list_append(tokens, make_tok("Not", "!"))
|
|
let i = i + 1
|
|
}
|
|
} else {
|
|
if ch == "<" {
|
|
if peek_ch == "=" {
|
|
let tokens = native_list_append(tokens, make_tok("LtEq", "<="))
|
|
let i = i + 2
|
|
} else {
|
|
let tokens = native_list_append(tokens, make_tok("Lt", "<"))
|
|
let i = i + 1
|
|
}
|
|
} else {
|
|
if ch == ">" {
|
|
if peek_ch == "=" {
|
|
let tokens = native_list_append(tokens, make_tok("GtEq", ">="))
|
|
let i = i + 2
|
|
} else {
|
|
let tokens = native_list_append(tokens, make_tok("Gt", ">"))
|
|
let i = i + 1
|
|
}
|
|
} else {
|
|
if ch == "&" {
|
|
if peek_ch == "&" {
|
|
let tokens = native_list_append(tokens, make_tok("And", "&&"))
|
|
let i = i + 2
|
|
} else {
|
|
let i = i + 1
|
|
}
|
|
} else {
|
|
if ch == "|" {
|
|
if peek_ch == "|" {
|
|
let tokens = native_list_append(tokens, make_tok("Or", "||"))
|
|
let i = i + 2
|
|
} else {
|
|
if peek_ch == ">" {
|
|
let tokens = native_list_append(tokens, make_tok("PipeOp", "|>"))
|
|
let i = i + 2
|
|
} else {
|
|
let tokens = native_list_append(tokens, make_tok("Pipe", "|"))
|
|
let i = i + 1
|
|
}
|
|
}
|
|
} else {
|
|
if ch == "-" {
|
|
if peek_ch == ">" {
|
|
let tokens = native_list_append(tokens, make_tok("Arrow", "->"))
|
|
let i = i + 2
|
|
} else {
|
|
let tokens = native_list_append(tokens, make_tok("Minus", "-"))
|
|
let i = i + 1
|
|
}
|
|
} else {
|
|
if ch == ":" {
|
|
if peek_ch == ":" {
|
|
let tokens = native_list_append(tokens, make_tok("ColonColon", "::"))
|
|
let i = i + 2
|
|
} else {
|
|
let tokens = native_list_append(tokens, make_tok("Colon", ":"))
|
|
let i = i + 1
|
|
}
|
|
} else {
|
|
if ch == "+" {
|
|
let tokens = native_list_append(tokens, make_tok("Plus", "+"))
|
|
let i = i + 1
|
|
} else {
|
|
if ch == "*" {
|
|
let tokens = native_list_append(tokens, make_tok("Star", "*"))
|
|
let i = i + 1
|
|
} else {
|
|
if ch == "%" {
|
|
let tokens = native_list_append(tokens, make_tok("Percent", "%"))
|
|
let i = i + 1
|
|
} else {
|
|
if ch == "(" {
|
|
let tokens = native_list_append(tokens, make_tok("LParen", "("))
|
|
let i = i + 1
|
|
} else {
|
|
if ch == ")" {
|
|
let tokens = native_list_append(tokens, make_tok("RParen", ")"))
|
|
let i = i + 1
|
|
} else {
|
|
if ch == "{" {
|
|
let tokens = native_list_append(tokens, make_tok("LBrace", "{"))
|
|
let i = i + 1
|
|
} else {
|
|
if ch == "}" {
|
|
let tokens = native_list_append(tokens, make_tok("RBrace", "}"))
|
|
let i = i + 1
|
|
} else {
|
|
if ch == "[" {
|
|
let tokens = native_list_append(tokens, make_tok("LBracket", "["))
|
|
let i = i + 1
|
|
} else {
|
|
if ch == "]" {
|
|
let tokens = native_list_append(tokens, make_tok("RBracket", "]"))
|
|
let i = i + 1
|
|
} else {
|
|
if ch == "," {
|
|
let tokens = native_list_append(tokens, make_tok("Comma", ","))
|
|
let i = i + 1
|
|
} else {
|
|
if ch == "." {
|
|
let tokens = native_list_append(tokens, make_tok("Dot", "."))
|
|
let i = i + 1
|
|
} else {
|
|
if ch == ";" {
|
|
let tokens = native_list_append(tokens, make_tok("Semicolon", ";"))
|
|
let i = i + 1
|
|
} else {
|
|
if ch == "@" {
|
|
let tokens = native_list_append(tokens, make_tok("At", "@"))
|
|
let i = i + 1
|
|
} else {
|
|
if ch == "?" {
|
|
let tokens = native_list_append(tokens, make_tok("QuestionMark", "?"))
|
|
let i = i + 1
|
|
} else {
|
|
// unknown char — skip
|
|
let i = i + 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
let tokens = native_list_append(tokens, make_tok("Eof", ""))
|
|
tokens
|
|
}
|