neuron-technologies/el
3
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

Archive Rust bootstrap — El compiler is now self-hosting

Browse Source
This commit is contained in:
Will Anderson
2026-04-29 22:21:31 -05:00
parent 9a0747aa13
commit 4f3543b068
139 changed files with 8980 additions and 1778 deletions
Download Patch File Download Diff File Expand all files Collapse all files
el-compiler/src/lexer.el
+26 -21
View File
@@ -6,6 +6,9 @@
// "value" -> String (the raw text of the token)
//
// Entry point: fn lex(source: String) -> [Map<String, Any>]
//
// Uses global char_buf via native_chars_init / native_char_at / native_char_len
// to avoid O(N²) cloning of the chars list.
// ── Character helpers ─────────────────────────────────────────────────────────
@@ -141,9 +144,11 @@ fn keyword_kind(word: String) -> String {
}
// ── Scan helpers ──────────────────────────────────────────────────────────────
// All scan helpers use the global char_buf (native_char_at / native_char_len).
// No chars parameter — avoids O(N²) cloning.
// scan_digits — advance i while chars[i] is a digit, return { "text": ..., "pos": i }
fn scan_digits(chars: [String], start: Int, total: Int) -> Map<String, Any> {
// scan_digits — advance i while char_buf[i] is a digit, return { "text": ..., "pos": i }
fn scan_digits(start: Int, total: Int) -> Map<String, Any> {
let i = start
let text = ""
let running = true
@@ -151,7 +156,7 @@ fn scan_digits(chars: [String], start: Int, total: Int) -> Map<String, Any> {
if i >= total {
let running = false
} else {
let ch = native_list_get(chars, i)
let ch = native_char_at(i)
if is_digit(ch) {
let text = text + ch
let i = i + 1
@@ -163,8 +168,8 @@ fn scan_digits(chars: [String], start: Int, total: Int) -> Map<String, Any> {
{ "text": text, "pos": i }
}
// scan_ident — advance i while chars[i] is alphanumeric or underscore
fn scan_ident(chars: [String], start: Int, total: Int) -> Map<String, Any> {
// scan_ident — advance i while char_buf[i] is alphanumeric or underscore
fn scan_ident(start: Int, total: Int) -> Map<String, Any> {
let i = start
let text = ""
let running = true
@@ -172,7 +177,7 @@ fn scan_ident(chars: [String], start: Int, total: Int) -> Map<String, Any> {
if i >= total {
let running = false
} else {
let ch = native_list_get(chars, i)
let ch = native_char_at(i)
if is_alnum_or_underscore(ch) {
let text = text + ch
let i = i + 1
@@ -186,7 +191,7 @@ fn scan_ident(chars: [String], start: Int, total: Int) -> Map<String, Any> {
// scan_string — scan a quoted string literal, handling \" escapes.
// Starts AFTER the opening quote. Returns { "text": content, "pos": i_after_close }
fn scan_string(chars: [String], start: Int, total: Int) -> Map<String, Any> {
fn scan_string(start: Int, total: Int) -> Map<String, Any> {
let i = start
let text = ""
let closed = false
@@ -195,12 +200,12 @@ fn scan_string(chars: [String], start: Int, total: Int) -> Map<String, Any> {
if i >= total {
let running = false
} else {
let ch = native_list_get(chars, i)
let ch = native_char_at(i)
if ch == "\\" {
// escape: peek next char
let next_i = i + 1
if next_i < total {
let next_ch = native_list_get(chars, next_i)
let next_ch = native_char_at(next_i)
if next_ch == "\"" {
let text = text + "\""
let i = next_i + 1
@@ -244,13 +249,13 @@ fn scan_string(chars: [String], start: Int, total: Int) -> Map<String, Any> {
// ── Main lexer ────────────────────────────────────────────────────────────────
fn lex(source: String) -> [Map<String, Any>] {
let chars: [String] = native_string_chars(source)
let total: Int = native_list_len(chars)
native_chars_init(source)
let total: Int = native_char_len()
let tokens: [Map<String, Any>] = native_list_empty()
let i: Int = 0
while i < total {
let ch: String = native_list_get(chars, i)
let ch: String = native_char_at(i)
// Skip whitespace
if is_whitespace(ch) {
@@ -260,7 +265,7 @@ fn lex(source: String) -> [Map<String, Any>] {
if ch == "/" {
let next_i = i + 1
if next_i < total {
let next_ch: String = native_list_get(chars, next_i)
let next_ch: String = native_char_at(next_i)
if next_ch == "/" {
// skip to end of line
let i = i + 2
@@ -269,7 +274,7 @@ fn lex(source: String) -> [Map<String, Any>] {
if i >= total {
let running2 = false
} else {
let lch: String = native_list_get(chars, i)
let lch: String = native_char_at(i)
if lch == "\n" {
let running2 = false
} else {
@@ -288,7 +293,7 @@ fn lex(source: String) -> [Map<String, Any>] {
} else {
// String literal
if ch == "\"" {
let result = scan_string(chars, i + 1, total)
let result = scan_string(i + 1, total)
let str_text: String = result["text"]
let new_pos: Int = result["pos"]
let tokens = native_list_append(tokens, make_tok("Str", str_text))
@@ -296,18 +301,18 @@ fn lex(source: String) -> [Map<String, Any>] {
} else {
// Number literal
if is_digit(ch) {
let result = scan_digits(chars, i, total)
let result = scan_digits(i, total)
let num_text: String = result["text"]
let new_pos: Int = result["pos"]
// check for float (dot followed by digit)
if new_pos < total {
let dot_ch: String = native_list_get(chars, new_pos)
let dot_ch: String = native_char_at(new_pos)
if dot_ch == "." {
let after_dot = new_pos + 1
if after_dot < total {
let after_dot_ch: String = native_list_get(chars, after_dot)
let after_dot_ch: String = native_char_at(after_dot)
if is_digit(after_dot_ch) {
let frac_result = scan_digits(chars, after_dot, total)
let frac_result = scan_digits(after_dot, total)
let frac_text: String = frac_result["text"]
let frac_pos: Int = frac_result["pos"]
let tokens = native_list_append(tokens, make_tok("Float", num_text + "." + frac_text))
@@ -331,7 +336,7 @@ fn lex(source: String) -> [Map<String, Any>] {
} else {
// Identifier or keyword
if is_alpha(ch) || ch == "_" {
let result = scan_ident(chars, i, total)
let result = scan_ident(i, total)
let word: String = result["text"]
let new_pos: Int = result["pos"]
let kw = keyword_kind(word)
@@ -346,7 +351,7 @@ fn lex(source: String) -> [Map<String, Any>] {
let peek_i = i + 1
let peek_ch = ""
if peek_i < total {
let peek_ch = native_list_get(chars, peek_i)
let peek_ch = native_char_at(peek_i)
}
if ch == "=" {