neuron-technologies/el
3
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

add text-processing primitives to el runtime

Browse Source 
24 new functions covering counting (str_count, str_count_chars,
str_count_bytes, str_count_lines, str_count_words, str_count_letters,
str_count_digits), finding (str_index_of_all, str_last_index_of,
str_find_chars), transforming (str_repeat, str_reverse,
str_strip_prefix/suffix/chars, str_lstrip, str_rstrip), character
classification (is_letter, is_digit, is_alphanumeric, is_whitespace,
is_punctuation, is_uppercase, is_lowercase), and splitting/joining
(str_split_lines, str_split_chars, str_split_n, str_join).

Phase 1 is byte-level + ASCII character classes. Unicode-grapheme
awareness, normalization, and regex are Phase 2 (filed separately).

Lexer-internal helpers is_digit, is_alpha, is_whitespace renamed to
lex_is_digit, lex_is_alpha, lex_is_whitespace to free the public names
for the runtime exports. The El compiler's lexer.el and the bundled
elc-combined.el both updated.

Codegen registrations: builtin_arity entries for all 24 functions,
is_int_call entries for the Int-returning ones (str_count*,
str_last_index_of, str_find_chars) so the + operator dispatches as
arithmetic when applicable.

Tests: tests/text/ corpus with 8 acceptance cases covering the surface
(count-substring, count-overlap-skip, count-lines-words-letters,
index-of-all, transform-suite, char-classes, split-lines, join). All
pass against a fold-fn-main-aware elc bootstrap (see ELC env var
override in run.sh).

Self-host fixed point: elc-combined.el's emit-main pass does not
currently fold the fn main body into C's main, a pre-existing
condition that surfaces as a 39-line gen2/gen3 diff with empty main
in gen3. The committed dist/platform/elc binary has the fold logic
so all tests pass against it. Filing the elc-combined fold-fn-main
fix separately. This commit does not introduce new self-host drift.
This commit is contained in:
Will Anderson
2026-05-02 13:37:30 -05:00
parent ed564b6dda
commit 3a83b6eb80
15 changed files with 727 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
el-compiler/src/codegen.el
+38
View File
@@ -1257,6 +1257,15 @@ fn is_int_call(call_expr: Map<String, Any>) -> Bool {
if str_eq(name, "str_index_of") { return true }
if str_eq(name, "str_to_int") { return true }
if str_eq(name, "str_char_code") { return true }
if str_eq(name, "str_count") { return true }
if str_eq(name, "str_count_chars") { return true }
if str_eq(name, "str_count_bytes") { return true }
if str_eq(name, "str_count_lines") { return true }
if str_eq(name, "str_count_words") { return true }
if str_eq(name, "str_count_letters") { return true }
if str_eq(name, "str_count_digits") { return true }
if str_eq(name, "str_last_index_of") { return true }
if str_eq(name, "str_find_chars") { return true }
if str_eq(name, "native_list_len") { return true }
if str_eq(name, "el_list_len") { return true }
if str_eq(name, "len") { return true }
@@ -1858,6 +1867,35 @@ fn builtin_arity(name: String) -> Int {
if str_eq(name, "str_format") { return 2 }
if str_eq(name, "str_lower") { return 1 }
if str_eq(name, "str_upper") { return 1 }
// Text-processing primitives (Phase 1)
if str_eq(name, "str_count") { return 2 }
if str_eq(name, "str_count_chars") { return 1 }
if str_eq(name, "str_count_bytes") { return 1 }
if str_eq(name, "str_count_lines") { return 1 }
if str_eq(name, "str_count_words") { return 1 }
if str_eq(name, "str_count_letters") { return 1 }
if str_eq(name, "str_count_digits") { return 1 }
if str_eq(name, "str_index_of_all") { return 2 }
if str_eq(name, "str_last_index_of") { return 2 }
if str_eq(name, "str_find_chars") { return 2 }
if str_eq(name, "str_repeat") { return 2 }
if str_eq(name, "str_reverse") { return 1 }
if str_eq(name, "str_strip_prefix") { return 2 }
if str_eq(name, "str_strip_suffix") { return 2 }
if str_eq(name, "str_strip_chars") { return 2 }
if str_eq(name, "str_lstrip") { return 1 }
if str_eq(name, "str_rstrip") { return 1 }
if str_eq(name, "is_letter") { return 1 }
if str_eq(name, "is_digit") { return 1 }
if str_eq(name, "is_alphanumeric") { return 1 }
if str_eq(name, "is_whitespace") { return 1 }
if str_eq(name, "is_punctuation") { return 1 }
if str_eq(name, "is_uppercase") { return 1 }
if str_eq(name, "is_lowercase") { return 1 }
if str_eq(name, "str_split_lines") { return 1 }
if str_eq(name, "str_split_chars") { return 1 }
if str_eq(name, "str_split_n") { return 3 }
if str_eq(name, "str_join") { return 2 }
// HTML sanitizer
if str_eq(name, "el_html_sanitize") { return 2 }
// Math
el-compiler/src/lexer.el
+10 -10
View File
@@ -12,7 +12,7 @@
// ── Character helpers ─────────────────────────────────────────────────────────
fn is_digit(ch: String) -> Bool {
fn lex_is_digit(ch: String) -> Bool {
if ch == "0" { return true }
if ch == "1" { return true }
if ch == "2" { return true }
@@ -26,7 +26,7 @@ fn is_digit(ch: String) -> Bool {
false
}
fn is_alpha(ch: String) -> Bool {
fn lex_is_alpha(ch: String) -> Bool {
if ch == "a" { return true }
if ch == "b" { return true }
if ch == "c" { return true }
@@ -83,13 +83,13 @@ fn is_alpha(ch: String) -> Bool {
}
fn is_alnum_or_underscore(ch: String) -> Bool {
if is_digit(ch) { return true }
if is_alpha(ch) { return true }
if lex_is_digit(ch) { return true }
if lex_is_alpha(ch) { return true }
if ch == "_" { return true }
false
}
fn is_whitespace(ch: String) -> Bool {
fn lex_is_whitespace(ch: String) -> Bool {
if ch == " " { return true }
if ch == "\t" { return true }
if ch == "\n" { return true }
@@ -163,7 +163,7 @@ fn scan_digits(chars: [String], start: Int, total: Int) -> Map<String, Any> {
let running = false
} else {
let ch: String = native_list_get(chars, i)
if is_digit(ch) {
if lex_is_digit(ch) {
let text = text + ch
let i = i + 1
} else {
@@ -469,7 +469,7 @@ fn lex(source: String) -> [Map<String, Any>] {
let ch: String = native_list_get(chars, i)
// Skip whitespace
if is_whitespace(ch) {
if lex_is_whitespace(ch) {
let i = i + 1
} else {
// Line comments: //
@@ -519,7 +519,7 @@ fn lex(source: String) -> [Map<String, Any>] {
let i = new_pos
} else {
// Number literal
if is_digit(ch) {
if lex_is_digit(ch) {
let result = scan_digits(chars, i, total)
let num_text: String = result["text"]
let new_pos: Int = result["pos"]
@@ -530,7 +530,7 @@ fn lex(source: String) -> [Map<String, Any>] {
let after_dot = new_pos + 1
if after_dot < total {
let after_dot_ch: String = native_list_get(chars, after_dot)
if is_digit(after_dot_ch) {
if lex_is_digit(after_dot_ch) {
let frac_result = scan_digits(chars, after_dot, total)
let frac_text: String = frac_result["text"]
let frac_pos: Int = frac_result["pos"]
@@ -554,7 +554,7 @@ fn lex(source: String) -> [Map<String, Any>] {
}
} else {
// Identifier or keyword
if is_alpha(ch) || ch == "_" {
if lex_is_alpha(ch) || ch == "_" {
let result = scan_ident(chars, i, total)
let word: String = result["text"]
let new_pos: Int = result["pos"]