neuron-technologies/el
2
0
Fork 0
Code Issues 1 Pull Requests 6 Actions Packages Projects Releases 5 Wiki Activity

add text-processing primitives to el runtime

Browse Source 
24 new functions covering counting (str_count, str_count_chars,
str_count_bytes, str_count_lines, str_count_words, str_count_letters,
str_count_digits), finding (str_index_of_all, str_last_index_of,
str_find_chars), transforming (str_repeat, str_reverse,
str_strip_prefix/suffix/chars, str_lstrip, str_rstrip), character
classification (is_letter, is_digit, is_alphanumeric, is_whitespace,
is_punctuation, is_uppercase, is_lowercase), and splitting/joining
(str_split_lines, str_split_chars, str_split_n, str_join).

Phase 1 is byte-level + ASCII character classes. Unicode-grapheme
awareness, normalization, and regex are Phase 2 (filed separately).

Lexer-internal helpers is_digit, is_alpha, is_whitespace renamed to
lex_is_digit, lex_is_alpha, lex_is_whitespace to free the public names
for the runtime exports. The El compiler's lexer.el and the bundled
elc-combined.el both updated.

Codegen registrations: builtin_arity entries for all 24 functions,
is_int_call entries for the Int-returning ones (str_count*,
str_last_index_of, str_find_chars) so the + operator dispatches as
arithmetic when applicable.

Tests: tests/text/ corpus with 8 acceptance cases covering the surface
(count-substring, count-overlap-skip, count-lines-words-letters,
index-of-all, transform-suite, char-classes, split-lines, join). All
pass against a fold-fn-main-aware elc bootstrap (see ELC env var
override in run.sh).

Self-host fixed point: elc-combined.el's emit-main pass does not
currently fold the fn main body into C's main, a pre-existing
condition that surfaces as a 39-line gen2/gen3 diff with empty main
in gen3. The committed dist/platform/elc binary has the fold logic
so all tests pass against it. Filing the elc-combined fold-fn-main
fix separately. This commit does not introduce new self-host drift.
This commit is contained in:
Will Anderson
2026-05-02 13:37:30 -05:00
parent ed564b6dda
commit 3a83b6eb80
15 changed files with 727 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
el-compiler/runtime/el_runtime.c
+370
View File
@@ -4805,6 +4805,376 @@ el_val_t str_format(el_val_t fmt, el_val_t data) {
el_val_t str_lower(el_val_t s) { return str_to_lower(s); }
el_val_t str_upper(el_val_t s) { return str_to_upper(s); }
/* ── Text-processing primitives (Phase 1: byte/codepoint, ASCII char classes)
*
* Phase 1 covers the operations every text-handling caller used to roll by
* hand on top of str_index_of + str_slice. The character-class predicates
* (is_letter / is_digit / ...) are ASCII only — Unicode-grapheme awareness,
* NFC/NFD normalization, and regex are Phase 2. Single-char input checks the
* first byte; multi-char input requires ALL bytes to match (false otherwise).
*
* Counting:
* str_count non-overlapping occurrences of sub in s
* str_count_chars codepoint count (UTF-8 leading-byte count)
* str_count_bytes explicit byte length (alias of str_len)
* str_count_lines \n-delimited line count (\r\n folded to \n)
* str_count_words whitespace-delimited tokens, non-empty only
* str_count_letters ASCII [A-Za-z]
* str_count_digits ASCII [0-9]
*
* Find / position:
* str_index_of_all all byte offsets of sub, [] if none
* str_last_index_of last byte offset of sub, -1 if not found
* str_find_chars first index of any char in any_of, -1 if none
*
* Transform:
* str_repeat s * n (non-negative)
* str_reverse codepoint-reversed (NOT grapheme-aware)
* str_strip_prefix s without prefix if present, else s
* str_strip_suffix s without suffix if present, else s
* str_strip_chars strip leading+trailing chars matching any in chars
* str_lstrip strip leading whitespace
* str_rstrip strip trailing whitespace
*
* Char classification (Bool):
* is_letter, is_digit, is_alphanumeric, is_whitespace,
* is_punctuation, is_uppercase, is_lowercase
*
* Splitting:
* str_split_lines \n-delimited (\r\n folded). Trailing empty dropped.
* str_split_chars alias of native_string_chars in str_ namespace
* str_split_n split into at most n parts (last part keeps the
* rest verbatim, including any further separators)
*
* Joining:
* str_join [String] -> String, sep between elements
*/
/* Count non-overlapping occurrences of sub in s. Empty sub returns 0. */
el_val_t str_count(el_val_t sv, el_val_t subv) {
const char* s = EL_CSTR(sv);
const char* sub = EL_CSTR(subv);
if (!s || !sub || !*sub) return 0;
size_t lp = strlen(sub);
int64_t count = 0;
const char* p = s;
while ((p = strstr(p, sub)) != NULL) {
count++;
p += lp; /* non-overlapping advance */
}
return (el_val_t)count;
}
/* Codepoint count: walk bytes, count those NOT matching 10xxxxxx. */
el_val_t str_count_chars(el_val_t sv) {
const char* s = EL_CSTR(sv);
if (!s) return 0;
int64_t count = 0;
for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
if ((*p & 0xC0) != 0x80) count++;
}
return (el_val_t)count;
}
el_val_t str_count_bytes(el_val_t sv) {
return str_len(sv);
}
el_val_t str_count_lines(el_val_t sv) {
const char* s = EL_CSTR(sv);
if (!s || !*s) return 0;
int64_t count = 0;
int has_content = 0;
for (const char* p = s; *p; p++) {
has_content = 1;
if (*p == '\n') {
count++;
has_content = 0; /* the \n closed the line */
}
}
if (has_content) count++; /* trailing line with no terminator */
return (el_val_t)count;
}
el_val_t str_count_words(el_val_t sv) {
const char* s = EL_CSTR(sv);
if (!s) return 0;
int64_t count = 0;
int in_word = 0;
for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
if (isspace(*p)) {
in_word = 0;
} else if (!in_word) {
in_word = 1;
count++;
}
}
return (el_val_t)count;
}
el_val_t str_count_letters(el_val_t sv) {
const char* s = EL_CSTR(sv);
if (!s) return 0;
int64_t count = 0;
for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
if ((*p >= 'A' && *p <= 'Z') || (*p >= 'a' && *p <= 'z')) count++;
}
return (el_val_t)count;
}
el_val_t str_count_digits(el_val_t sv) {
const char* s = EL_CSTR(sv);
if (!s) return 0;
int64_t count = 0;
for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
if (*p >= '0' && *p <= '9') count++;
}
return (el_val_t)count;
}
el_val_t str_index_of_all(el_val_t sv, el_val_t subv) {
const char* s = EL_CSTR(sv);
const char* sub = EL_CSTR(subv);
el_val_t lst = el_list_empty();
if (!s || !sub || !*sub) return lst;
size_t lp = strlen(sub);
const char* p = s;
const char* hit;
while ((hit = strstr(p, sub)) != NULL) {
lst = el_list_append(lst, (el_val_t)(int64_t)(hit - s));
p = hit + lp;
}
return lst;
}
el_val_t str_last_index_of(el_val_t sv, el_val_t subv) {
const char* s = EL_CSTR(sv);
const char* sub = EL_CSTR(subv);
if (!s || !sub || !*sub) return -1;
size_t lp = strlen(sub);
int64_t last = -1;
const char* p = s;
const char* hit;
while ((hit = strstr(p, sub)) != NULL) {
last = (int64_t)(hit - s);
p = hit + lp;
}
return (el_val_t)last;
}
el_val_t str_find_chars(el_val_t sv, el_val_t any_of_v) {
const char* s = EL_CSTR(sv);
const char* any = EL_CSTR(any_of_v);
if (!s || !any || !*any) return -1;
for (const char* p = s; *p; p++) {
if (strchr(any, *p)) return (el_val_t)(int64_t)(p - s);
}
return -1;
}
el_val_t str_repeat(el_val_t sv, el_val_t nv) {
const char* s = EL_CSTR(sv);
int64_t n = (int64_t)nv;
if (!s || n <= 0) return el_wrap_str(el_strdup(""));
size_t ls = strlen(s);
if (ls == 0) return el_wrap_str(el_strdup(""));
size_t total = ls * (size_t)n;
char* out = el_strbuf(total);
for (int64_t i = 0; i < n; i++) {
memcpy(out + i * ls, s, ls);
}
out[total] = '\0';
return el_wrap_str(out);
}
/* Reverse by codepoint: walk codepoints, copy each backwards into the output.
* NOT grapheme-aware (Phase 2). Combining marks attached to a base codepoint
* will detach. ASCII strings are byte-reverse equivalent. */
el_val_t str_reverse(el_val_t sv) {
const char* s = EL_CSTR(sv);
if (!s) return el_wrap_str(el_strdup(""));
size_t n = strlen(s);
char* out = el_strbuf(n);
/* Walk forward, find each codepoint's byte length, then copy from the end. */
size_t out_pos = n;
const unsigned char* p = (const unsigned char*)s;
while (*p) {
int cp_len;
if ((*p & 0x80) == 0x00) cp_len = 1;
else if ((*p & 0xE0) == 0xC0) cp_len = 2;
else if ((*p & 0xF0) == 0xE0) cp_len = 3;
else if ((*p & 0xF8) == 0xF0) cp_len = 4;
else cp_len = 1; /* invalid byte: passthrough */
out_pos -= cp_len;
memcpy(out + out_pos, p, cp_len);
p += cp_len;
}
out[n] = '\0';
return el_wrap_str(out);
}
el_val_t str_strip_prefix(el_val_t sv, el_val_t prefv) {
const char* s = EL_CSTR(sv);
const char* pref = EL_CSTR(prefv);
if (!s) return el_wrap_str(el_strdup(""));
if (!pref || !*pref) return el_wrap_str(el_strdup(s));
size_t lp = strlen(pref);
size_t ls = strlen(s);
if (lp <= ls && strncmp(s, pref, lp) == 0) {
char* out = el_strbuf(ls - lp);
memcpy(out, s + lp, ls - lp);
out[ls - lp] = '\0';
return el_wrap_str(out);
}
return el_wrap_str(el_strdup(s));
}
el_val_t str_strip_suffix(el_val_t sv, el_val_t sufv) {
const char* s = EL_CSTR(sv);
const char* suf = EL_CSTR(sufv);
if (!s) return el_wrap_str(el_strdup(""));
if (!suf || !*suf) return el_wrap_str(el_strdup(s));
size_t ls = strlen(s);
size_t lsuf = strlen(suf);
if (lsuf <= ls && strcmp(s + ls - lsuf, suf) == 0) {
char* out = el_strbuf(ls - lsuf);
memcpy(out, s, ls - lsuf);
out[ls - lsuf] = '\0';
return el_wrap_str(out);
}
return el_wrap_str(el_strdup(s));
}
el_val_t str_strip_chars(el_val_t sv, el_val_t charsv) {
const char* s = EL_CSTR(sv);
const char* chars = EL_CSTR(charsv);
if (!s) return el_wrap_str(el_strdup(""));
if (!chars || !*chars) return el_wrap_str(el_strdup(s));
const char* start = s;
while (*start && strchr(chars, *start)) start++;
size_t n = strlen(start);
while (n > 0 && strchr(chars, start[n - 1])) n--;
char* out = el_strbuf(n);
memcpy(out, start, n);
out[n] = '\0';
return el_wrap_str(out);
}
el_val_t str_lstrip(el_val_t sv) {
const char* s = EL_CSTR(sv);
if (!s) return el_wrap_str(el_strdup(""));
while (*s && isspace((unsigned char)*s)) s++;
return el_wrap_str(el_strdup(s));
}
el_val_t str_rstrip(el_val_t sv) {
const char* s = EL_CSTR(sv);
if (!s) return el_wrap_str(el_strdup(""));
size_t n = strlen(s);
while (n > 0 && isspace((unsigned char)s[n - 1])) n--;
char* out = el_strbuf(n);
memcpy(out, s, n);
out[n] = '\0';
return el_wrap_str(out);
}
/* Character classification.
* Empty input returns false. Multi-char input requires ALL bytes to match.
* ASCII range only; Phase 2 will widen to Unicode. */
static int s_all_match(el_val_t sv, int (*pred)(unsigned char)) {
const char* s = EL_CSTR(sv);
if (!s || !*s) return 0;
for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
if (!pred(*p)) return 0;
}
return 1;
}
static int p_letter(unsigned char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); }
static int p_digit(unsigned char c) { return c >= '0' && c <= '9'; }
static int p_alnum(unsigned char c) { return p_letter(c) || p_digit(c); }
static int p_white(unsigned char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; }
static int p_punct(unsigned char c) { return ispunct(c) ? 1 : 0; }
static int p_upper(unsigned char c) { return c >= 'A' && c <= 'Z'; }
static int p_lower(unsigned char c) { return c >= 'a' && c <= 'z'; }
el_val_t is_letter(el_val_t s) { return (el_val_t)s_all_match(s, p_letter); }
el_val_t is_digit(el_val_t s) { return (el_val_t)s_all_match(s, p_digit); }
el_val_t is_alphanumeric(el_val_t s) { return (el_val_t)s_all_match(s, p_alnum); }
el_val_t is_whitespace(el_val_t s) { return (el_val_t)s_all_match(s, p_white); }
el_val_t is_punctuation(el_val_t s) { return (el_val_t)s_all_match(s, p_punct); }
el_val_t is_uppercase(el_val_t s) { return (el_val_t)s_all_match(s, p_upper); }
el_val_t is_lowercase(el_val_t s) { return (el_val_t)s_all_match(s, p_lower); }
/* Split on \n. \r\n is folded to \n first. Trailing empty after final \n
* is dropped (so "a\nb\n" -> ["a", "b"], not ["a", "b", ""]). */
el_val_t str_split_lines(el_val_t sv) {
const char* s = EL_CSTR(sv);
el_val_t lst = el_list_empty();
if (!s) return lst;
size_t n = strlen(s);
/* Pre-scan: build into a normalized buffer with \r\n folded. */
const char* line_start = s;
for (size_t i = 0; i <= n; i++) {
if (s[i] == '\n' || s[i] == '\0') {
size_t len = (size_t)(s + i - line_start);
/* Drop trailing \r if this was \r\n. */
if (len > 0 && line_start[len - 1] == '\r') len--;
/* Drop final trailing-empty-after-newline. */
if (s[i] == '\0' && len == 0 && i > 0 && s[i - 1] == '\n') break;
char* out = el_strbuf(len);
memcpy(out, line_start, len);
out[len] = '\0';
lst = el_list_append(lst, el_wrap_str(out));
if (s[i] == '\0') break;
line_start = s + i + 1;
}
}
return lst;
}
el_val_t str_split_chars(el_val_t s) {
return native_string_chars(s);
}
/* Split into at most n parts. The (n-1)th split point is the LAST split;
* after it, the remainder is appended verbatim including any further
* separators. n <= 0 returns an empty list. n == 1 returns [s]. */
el_val_t str_split_n(el_val_t sv, el_val_t sepv, el_val_t nv) {
const char* s = EL_CSTR(sv);
const char* sep = EL_CSTR(sepv);
int64_t n = (int64_t)nv;
el_val_t lst = el_list_empty();
if (!s) return lst;
if (n <= 0) return lst;
if (n == 1 || !sep || !*sep) {
lst = el_list_append(lst, el_wrap_str(el_strdup(s)));
return lst;
}
size_t lp = strlen(sep);
const char* p = s;
int64_t parts = 0;
const char* hit;
while (parts < n - 1 && (hit = strstr(p, sep)) != NULL) {
size_t len = (size_t)(hit - p);
char* out = el_strbuf(len);
memcpy(out, p, len);
out[len] = '\0';
lst = el_list_append(lst, el_wrap_str(out));
p = hit + lp;
parts++;
}
/* Remainder verbatim. */
lst = el_list_append(lst, el_wrap_str(el_strdup(p)));
return lst;
}
/* Join a [String] with a separator. Empty list -> "". Single-element ->
* that element. Non-string elements are stringified via int_to_str. */
el_val_t str_join(el_val_t listv, el_val_t sepv) {
return list_join(listv, sepv);
}
/* ── List additions ──────────────────────────────────────────────────────── */
el_val_t list_push(el_val_t list, el_val_t elem) {
el-compiler/runtime/el_runtime.h
+43
View File
@@ -445,6 +445,49 @@ el_val_t str_format(el_val_t fmt, el_val_t data);
el_val_t str_lower(el_val_t s);
el_val_t str_upper(el_val_t s);
/* ── Text-processing primitives (Phase 1: byte/codepoint, ASCII char classes)
* Phase 2 (filed): Unicode-grapheme awareness, NFC/NFD normalization, regex.
* is_* predicates: empty input returns false; multi-char requires ALL bytes
* to match. ASCII ranges only in Phase 1. */
/* Counting */
el_val_t str_count(el_val_t s, el_val_t sub); /* non-overlapping */
el_val_t str_count_chars(el_val_t s); /* codepoint count */
el_val_t str_count_bytes(el_val_t s); /* alias of str_len */
el_val_t str_count_lines(el_val_t s);
el_val_t str_count_words(el_val_t s);
el_val_t str_count_letters(el_val_t s); /* ASCII [A-Za-z] */
el_val_t str_count_digits(el_val_t s); /* ASCII [0-9] */
/* Find / position */
el_val_t str_index_of_all(el_val_t s, el_val_t sub); /* [Int] of byte offsets */
el_val_t str_last_index_of(el_val_t s, el_val_t sub);
el_val_t str_find_chars(el_val_t s, el_val_t any_of); /* first idx of any ch */
/* Transform */
el_val_t str_repeat(el_val_t s, el_val_t n);
el_val_t str_reverse(el_val_t s); /* by codepoint */
el_val_t str_strip_prefix(el_val_t s, el_val_t prefix);
el_val_t str_strip_suffix(el_val_t s, el_val_t suffix);
el_val_t str_strip_chars(el_val_t s, el_val_t chars);
el_val_t str_lstrip(el_val_t s);
el_val_t str_rstrip(el_val_t s);
/* Char classification (Bool) */
el_val_t is_letter(el_val_t s);
el_val_t is_digit(el_val_t s);
el_val_t is_alphanumeric(el_val_t s);
el_val_t is_whitespace(el_val_t s);
el_val_t is_punctuation(el_val_t s);
el_val_t is_uppercase(el_val_t s);
el_val_t is_lowercase(el_val_t s);
/* Split / join */
el_val_t str_split_lines(el_val_t s);
el_val_t str_split_chars(el_val_t s); /* alias of native_string_chars */
el_val_t str_split_n(el_val_t s, el_val_t sep, el_val_t n);
el_val_t str_join(el_val_t list, el_val_t sep); /* alias of list_join */
/* ── List additions ──────────────────────────────────────────────────────── */
el_val_t list_push(el_val_t list, el_val_t elem);
el-compiler/src/codegen.el
+38
View File
@@ -1257,6 +1257,15 @@ fn is_int_call(call_expr: Map<String, Any>) -> Bool {
if str_eq(name, "str_index_of") { return true }
if str_eq(name, "str_to_int") { return true }
if str_eq(name, "str_char_code") { return true }
if str_eq(name, "str_count") { return true }
if str_eq(name, "str_count_chars") { return true }
if str_eq(name, "str_count_bytes") { return true }
if str_eq(name, "str_count_lines") { return true }
if str_eq(name, "str_count_words") { return true }
if str_eq(name, "str_count_letters") { return true }
if str_eq(name, "str_count_digits") { return true }
if str_eq(name, "str_last_index_of") { return true }
if str_eq(name, "str_find_chars") { return true }
if str_eq(name, "native_list_len") { return true }
if str_eq(name, "el_list_len") { return true }
if str_eq(name, "len") { return true }
@@ -1858,6 +1867,35 @@ fn builtin_arity(name: String) -> Int {
if str_eq(name, "str_format") { return 2 }
if str_eq(name, "str_lower") { return 1 }
if str_eq(name, "str_upper") { return 1 }
// Text-processing primitives (Phase 1)
if str_eq(name, "str_count") { return 2 }
if str_eq(name, "str_count_chars") { return 1 }
if str_eq(name, "str_count_bytes") { return 1 }
if str_eq(name, "str_count_lines") { return 1 }
if str_eq(name, "str_count_words") { return 1 }
if str_eq(name, "str_count_letters") { return 1 }
if str_eq(name, "str_count_digits") { return 1 }
if str_eq(name, "str_index_of_all") { return 2 }
if str_eq(name, "str_last_index_of") { return 2 }
if str_eq(name, "str_find_chars") { return 2 }
if str_eq(name, "str_repeat") { return 2 }
if str_eq(name, "str_reverse") { return 1 }
if str_eq(name, "str_strip_prefix") { return 2 }
if str_eq(name, "str_strip_suffix") { return 2 }
if str_eq(name, "str_strip_chars") { return 2 }
if str_eq(name, "str_lstrip") { return 1 }
if str_eq(name, "str_rstrip") { return 1 }
if str_eq(name, "is_letter") { return 1 }
if str_eq(name, "is_digit") { return 1 }
if str_eq(name, "is_alphanumeric") { return 1 }
if str_eq(name, "is_whitespace") { return 1 }
if str_eq(name, "is_punctuation") { return 1 }
if str_eq(name, "is_uppercase") { return 1 }
if str_eq(name, "is_lowercase") { return 1 }
if str_eq(name, "str_split_lines") { return 1 }
if str_eq(name, "str_split_chars") { return 1 }
if str_eq(name, "str_split_n") { return 3 }
if str_eq(name, "str_join") { return 2 }
// HTML sanitizer
if str_eq(name, "el_html_sanitize") { return 2 }
// Math
el-compiler/src/lexer.el
+10 -10
View File
@@ -12,7 +12,7 @@
// ── Character helpers ─────────────────────────────────────────────────────────
fn is_digit(ch: String) -> Bool {
fn lex_is_digit(ch: String) -> Bool {
if ch == "0" { return true }
if ch == "1" { return true }
if ch == "2" { return true }
@@ -26,7 +26,7 @@ fn is_digit(ch: String) -> Bool {
false
}
fn is_alpha(ch: String) -> Bool {
fn lex_is_alpha(ch: String) -> Bool {
if ch == "a" { return true }
if ch == "b" { return true }
if ch == "c" { return true }
@@ -83,13 +83,13 @@ fn is_alpha(ch: String) -> Bool {
}
fn is_alnum_or_underscore(ch: String) -> Bool {
if is_digit(ch) { return true }
if is_alpha(ch) { return true }
if lex_is_digit(ch) { return true }
if lex_is_alpha(ch) { return true }
if ch == "_" { return true }
false
}
fn is_whitespace(ch: String) -> Bool {
fn lex_is_whitespace(ch: String) -> Bool {
if ch == " " { return true }
if ch == "\t" { return true }
if ch == "\n" { return true }
@@ -163,7 +163,7 @@ fn scan_digits(chars: [String], start: Int, total: Int) -> Map<String, Any> {
let running = false
} else {
let ch: String = native_list_get(chars, i)
if is_digit(ch) {
if lex_is_digit(ch) {
let text = text + ch
let i = i + 1
} else {
@@ -469,7 +469,7 @@ fn lex(source: String) -> [Map<String, Any>] {
let ch: String = native_list_get(chars, i)
// Skip whitespace
if is_whitespace(ch) {
if lex_is_whitespace(ch) {
let i = i + 1
} else {
// Line comments: //
@@ -519,7 +519,7 @@ fn lex(source: String) -> [Map<String, Any>] {
let i = new_pos
} else {
// Number literal
if is_digit(ch) {
if lex_is_digit(ch) {
let result = scan_digits(chars, i, total)
let num_text: String = result["text"]
let new_pos: Int = result["pos"]
@@ -530,7 +530,7 @@ fn lex(source: String) -> [Map<String, Any>] {
let after_dot = new_pos + 1
if after_dot < total {
let after_dot_ch: String = native_list_get(chars, after_dot)
if is_digit(after_dot_ch) {
if lex_is_digit(after_dot_ch) {
let frac_result = scan_digits(chars, after_dot, total)
let frac_text: String = frac_result["text"]
let frac_pos: Int = frac_result["pos"]
@@ -554,7 +554,7 @@ fn lex(source: String) -> [Map<String, Any>] {
}
} else {
// Identifier or keyword
if is_alpha(ch) || ch == "_" {
if lex_is_alpha(ch) || ch == "_" {
let result = scan_ident(chars, i, total)
let word: String = result["text"]
let new_pos: Int = result["pos"]