add text-processing primitives to el runtime
24 new functions covering counting (str_count, str_count_chars, str_count_bytes, str_count_lines, str_count_words, str_count_letters, str_count_digits), finding (str_index_of_all, str_last_index_of, str_find_chars), transforming (str_repeat, str_reverse, str_strip_prefix/suffix/chars, str_lstrip, str_rstrip), character classification (is_letter, is_digit, is_alphanumeric, is_whitespace, is_punctuation, is_uppercase, is_lowercase), and splitting/joining (str_split_lines, str_split_chars, str_split_n, str_join). Phase 1 is byte-level + ASCII character classes. Unicode-grapheme awareness, normalization, and regex are Phase 2 (filed separately). Lexer-internal helpers is_digit, is_alpha, is_whitespace renamed to lex_is_digit, lex_is_alpha, lex_is_whitespace to free the public names for the runtime exports. The El compiler's lexer.el and the bundled elc-combined.el both updated. Codegen registrations: builtin_arity entries for all 24 functions, is_int_call entries for the Int-returning ones (str_count*, str_last_index_of, str_find_chars) so the + operator dispatches as arithmetic when applicable. Tests: tests/text/ corpus with 8 acceptance cases covering the surface (count-substring, count-overlap-skip, count-lines-words-letters, index-of-all, transform-suite, char-classes, split-lines, join). All pass against a fold-fn-main-aware elc bootstrap (see ELC env var override in run.sh). Self-host fixed point: elc-combined.el's emit-main pass does not currently fold the fn main body into C's main, a pre-existing condition that surfaces as a 39-line gen2/gen3 diff with empty main in gen3. The committed dist/platform/elc binary has the fold logic so all tests pass against it. Filing the elc-combined fold-fn-main fix separately. This commit does not introduce new self-host drift.
This commit is contained in:
Will Anderson
@@ -4805,6 +4805,376 @@ el_val_t str_format(el_val_t fmt, el_val_t data) {
|
||||
el_val_t str_lower(el_val_t s) { return str_to_lower(s); }
|
||||
el_val_t str_upper(el_val_t s) { return str_to_upper(s); }
|
||||
|
||||
/* ── Text-processing primitives (Phase 1: byte/codepoint, ASCII char classes)
|
||||
*
|
||||
* Phase 1 covers the operations every text-handling caller used to roll by
|
||||
* hand on top of str_index_of + str_slice. The character-class predicates
|
||||
* (is_letter / is_digit / ...) are ASCII only — Unicode-grapheme awareness,
|
||||
* NFC/NFD normalization, and regex are Phase 2. Single-char input checks the
|
||||
* first byte; multi-char input requires ALL bytes to match (false otherwise).
|
||||
*
|
||||
* Counting:
|
||||
* str_count non-overlapping occurrences of sub in s
|
||||
* str_count_chars codepoint count (UTF-8 leading-byte count)
|
||||
* str_count_bytes explicit byte length (alias of str_len)
|
||||
* str_count_lines \n-delimited line count (\r\n folded to \n)
|
||||
* str_count_words whitespace-delimited tokens, non-empty only
|
||||
* str_count_letters ASCII [A-Za-z]
|
||||
* str_count_digits ASCII [0-9]
|
||||
*
|
||||
* Find / position:
|
||||
* str_index_of_all all byte offsets of sub, [] if none
|
||||
* str_last_index_of last byte offset of sub, -1 if not found
|
||||
* str_find_chars first index of any char in any_of, -1 if none
|
||||
*
|
||||
* Transform:
|
||||
* str_repeat s * n (non-negative)
|
||||
* str_reverse codepoint-reversed (NOT grapheme-aware)
|
||||
* str_strip_prefix s without prefix if present, else s
|
||||
* str_strip_suffix s without suffix if present, else s
|
||||
* str_strip_chars strip leading+trailing chars matching any in chars
|
||||
* str_lstrip strip leading whitespace
|
||||
* str_rstrip strip trailing whitespace
|
||||
*
|
||||
* Char classification (Bool):
|
||||
* is_letter, is_digit, is_alphanumeric, is_whitespace,
|
||||
* is_punctuation, is_uppercase, is_lowercase
|
||||
*
|
||||
* Splitting:
|
||||
* str_split_lines \n-delimited (\r\n folded). Trailing empty dropped.
|
||||
* str_split_chars alias of native_string_chars in str_ namespace
|
||||
* str_split_n split into at most n parts (last part keeps the
|
||||
* rest verbatim, including any further separators)
|
||||
*
|
||||
* Joining:
|
||||
* str_join [String] -> String, sep between elements
|
||||
*/
|
||||
|
||||
/* Count non-overlapping occurrences of sub in s. Empty sub returns 0. */
|
||||
el_val_t str_count(el_val_t sv, el_val_t subv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
const char* sub = EL_CSTR(subv);
|
||||
if (!s || !sub || !*sub) return 0;
|
||||
size_t lp = strlen(sub);
|
||||
int64_t count = 0;
|
||||
const char* p = s;
|
||||
while ((p = strstr(p, sub)) != NULL) {
|
||||
count++;
|
||||
p += lp; /* non-overlapping advance */
|
||||
}
|
||||
return (el_val_t)count;
|
||||
}
|
||||
|
||||
/* Codepoint count: walk bytes, count those NOT matching 10xxxxxx. */
|
||||
el_val_t str_count_chars(el_val_t sv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
if (!s) return 0;
|
||||
int64_t count = 0;
|
||||
for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
|
||||
if ((*p & 0xC0) != 0x80) count++;
|
||||
}
|
||||
return (el_val_t)count;
|
||||
}
|
||||
|
||||
el_val_t str_count_bytes(el_val_t sv) {
|
||||
return str_len(sv);
|
||||
}
|
||||
|
||||
el_val_t str_count_lines(el_val_t sv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
if (!s || !*s) return 0;
|
||||
int64_t count = 0;
|
||||
int has_content = 0;
|
||||
for (const char* p = s; *p; p++) {
|
||||
has_content = 1;
|
||||
if (*p == '\n') {
|
||||
count++;
|
||||
has_content = 0; /* the \n closed the line */
|
||||
}
|
||||
}
|
||||
if (has_content) count++; /* trailing line with no terminator */
|
||||
return (el_val_t)count;
|
||||
}
|
||||
|
||||
el_val_t str_count_words(el_val_t sv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
if (!s) return 0;
|
||||
int64_t count = 0;
|
||||
int in_word = 0;
|
||||
for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
|
||||
if (isspace(*p)) {
|
||||
in_word = 0;
|
||||
} else if (!in_word) {
|
||||
in_word = 1;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return (el_val_t)count;
|
||||
}
|
||||
|
||||
el_val_t str_count_letters(el_val_t sv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
if (!s) return 0;
|
||||
int64_t count = 0;
|
||||
for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
|
||||
if ((*p >= 'A' && *p <= 'Z') || (*p >= 'a' && *p <= 'z')) count++;
|
||||
}
|
||||
return (el_val_t)count;
|
||||
}
|
||||
|
||||
el_val_t str_count_digits(el_val_t sv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
if (!s) return 0;
|
||||
int64_t count = 0;
|
||||
for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
|
||||
if (*p >= '0' && *p <= '9') count++;
|
||||
}
|
||||
return (el_val_t)count;
|
||||
}
|
||||
|
||||
el_val_t str_index_of_all(el_val_t sv, el_val_t subv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
const char* sub = EL_CSTR(subv);
|
||||
el_val_t lst = el_list_empty();
|
||||
if (!s || !sub || !*sub) return lst;
|
||||
size_t lp = strlen(sub);
|
||||
const char* p = s;
|
||||
const char* hit;
|
||||
while ((hit = strstr(p, sub)) != NULL) {
|
||||
lst = el_list_append(lst, (el_val_t)(int64_t)(hit - s));
|
||||
p = hit + lp;
|
||||
}
|
||||
return lst;
|
||||
}
|
||||
|
||||
el_val_t str_last_index_of(el_val_t sv, el_val_t subv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
const char* sub = EL_CSTR(subv);
|
||||
if (!s || !sub || !*sub) return -1;
|
||||
size_t lp = strlen(sub);
|
||||
int64_t last = -1;
|
||||
const char* p = s;
|
||||
const char* hit;
|
||||
while ((hit = strstr(p, sub)) != NULL) {
|
||||
last = (int64_t)(hit - s);
|
||||
p = hit + lp;
|
||||
}
|
||||
return (el_val_t)last;
|
||||
}
|
||||
|
||||
el_val_t str_find_chars(el_val_t sv, el_val_t any_of_v) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
const char* any = EL_CSTR(any_of_v);
|
||||
if (!s || !any || !*any) return -1;
|
||||
for (const char* p = s; *p; p++) {
|
||||
if (strchr(any, *p)) return (el_val_t)(int64_t)(p - s);
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
el_val_t str_repeat(el_val_t sv, el_val_t nv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
int64_t n = (int64_t)nv;
|
||||
if (!s || n <= 0) return el_wrap_str(el_strdup(""));
|
||||
size_t ls = strlen(s);
|
||||
if (ls == 0) return el_wrap_str(el_strdup(""));
|
||||
size_t total = ls * (size_t)n;
|
||||
char* out = el_strbuf(total);
|
||||
for (int64_t i = 0; i < n; i++) {
|
||||
memcpy(out + i * ls, s, ls);
|
||||
}
|
||||
out[total] = '\0';
|
||||
return el_wrap_str(out);
|
||||
}
|
||||
|
||||
/* Reverse by codepoint: walk codepoints, copy each backwards into the output.
|
||||
* NOT grapheme-aware (Phase 2). Combining marks attached to a base codepoint
|
||||
* will detach. ASCII strings are byte-reverse equivalent. */
|
||||
el_val_t str_reverse(el_val_t sv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
if (!s) return el_wrap_str(el_strdup(""));
|
||||
size_t n = strlen(s);
|
||||
char* out = el_strbuf(n);
|
||||
/* Walk forward, find each codepoint's byte length, then copy from the end. */
|
||||
size_t out_pos = n;
|
||||
const unsigned char* p = (const unsigned char*)s;
|
||||
while (*p) {
|
||||
int cp_len;
|
||||
if ((*p & 0x80) == 0x00) cp_len = 1;
|
||||
else if ((*p & 0xE0) == 0xC0) cp_len = 2;
|
||||
else if ((*p & 0xF0) == 0xE0) cp_len = 3;
|
||||
else if ((*p & 0xF8) == 0xF0) cp_len = 4;
|
||||
else cp_len = 1; /* invalid byte: passthrough */
|
||||
out_pos -= cp_len;
|
||||
memcpy(out + out_pos, p, cp_len);
|
||||
p += cp_len;
|
||||
}
|
||||
out[n] = '\0';
|
||||
return el_wrap_str(out);
|
||||
}
|
||||
|
||||
el_val_t str_strip_prefix(el_val_t sv, el_val_t prefv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
const char* pref = EL_CSTR(prefv);
|
||||
if (!s) return el_wrap_str(el_strdup(""));
|
||||
if (!pref || !*pref) return el_wrap_str(el_strdup(s));
|
||||
size_t lp = strlen(pref);
|
||||
size_t ls = strlen(s);
|
||||
if (lp <= ls && strncmp(s, pref, lp) == 0) {
|
||||
char* out = el_strbuf(ls - lp);
|
||||
memcpy(out, s + lp, ls - lp);
|
||||
out[ls - lp] = '\0';
|
||||
return el_wrap_str(out);
|
||||
}
|
||||
return el_wrap_str(el_strdup(s));
|
||||
}
|
||||
|
||||
el_val_t str_strip_suffix(el_val_t sv, el_val_t sufv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
const char* suf = EL_CSTR(sufv);
|
||||
if (!s) return el_wrap_str(el_strdup(""));
|
||||
if (!suf || !*suf) return el_wrap_str(el_strdup(s));
|
||||
size_t ls = strlen(s);
|
||||
size_t lsuf = strlen(suf);
|
||||
if (lsuf <= ls && strcmp(s + ls - lsuf, suf) == 0) {
|
||||
char* out = el_strbuf(ls - lsuf);
|
||||
memcpy(out, s, ls - lsuf);
|
||||
out[ls - lsuf] = '\0';
|
||||
return el_wrap_str(out);
|
||||
}
|
||||
return el_wrap_str(el_strdup(s));
|
||||
}
|
||||
|
||||
el_val_t str_strip_chars(el_val_t sv, el_val_t charsv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
const char* chars = EL_CSTR(charsv);
|
||||
if (!s) return el_wrap_str(el_strdup(""));
|
||||
if (!chars || !*chars) return el_wrap_str(el_strdup(s));
|
||||
const char* start = s;
|
||||
while (*start && strchr(chars, *start)) start++;
|
||||
size_t n = strlen(start);
|
||||
while (n > 0 && strchr(chars, start[n - 1])) n--;
|
||||
char* out = el_strbuf(n);
|
||||
memcpy(out, start, n);
|
||||
out[n] = '\0';
|
||||
return el_wrap_str(out);
|
||||
}
|
||||
|
||||
el_val_t str_lstrip(el_val_t sv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
if (!s) return el_wrap_str(el_strdup(""));
|
||||
while (*s && isspace((unsigned char)*s)) s++;
|
||||
return el_wrap_str(el_strdup(s));
|
||||
}
|
||||
|
||||
el_val_t str_rstrip(el_val_t sv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
if (!s) return el_wrap_str(el_strdup(""));
|
||||
size_t n = strlen(s);
|
||||
while (n > 0 && isspace((unsigned char)s[n - 1])) n--;
|
||||
char* out = el_strbuf(n);
|
||||
memcpy(out, s, n);
|
||||
out[n] = '\0';
|
||||
return el_wrap_str(out);
|
||||
}
|
||||
|
||||
/* Character classification.
|
||||
* Empty input returns false. Multi-char input requires ALL bytes to match.
|
||||
* ASCII range only; Phase 2 will widen to Unicode. */
|
||||
static int s_all_match(el_val_t sv, int (*pred)(unsigned char)) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
if (!s || !*s) return 0;
|
||||
for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
|
||||
if (!pred(*p)) return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int p_letter(unsigned char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); }
|
||||
static int p_digit(unsigned char c) { return c >= '0' && c <= '9'; }
|
||||
static int p_alnum(unsigned char c) { return p_letter(c) || p_digit(c); }
|
||||
static int p_white(unsigned char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; }
|
||||
static int p_punct(unsigned char c) { return ispunct(c) ? 1 : 0; }
|
||||
static int p_upper(unsigned char c) { return c >= 'A' && c <= 'Z'; }
|
||||
static int p_lower(unsigned char c) { return c >= 'a' && c <= 'z'; }
|
||||
|
||||
el_val_t is_letter(el_val_t s) { return (el_val_t)s_all_match(s, p_letter); }
|
||||
el_val_t is_digit(el_val_t s) { return (el_val_t)s_all_match(s, p_digit); }
|
||||
el_val_t is_alphanumeric(el_val_t s) { return (el_val_t)s_all_match(s, p_alnum); }
|
||||
el_val_t is_whitespace(el_val_t s) { return (el_val_t)s_all_match(s, p_white); }
|
||||
el_val_t is_punctuation(el_val_t s) { return (el_val_t)s_all_match(s, p_punct); }
|
||||
el_val_t is_uppercase(el_val_t s) { return (el_val_t)s_all_match(s, p_upper); }
|
||||
el_val_t is_lowercase(el_val_t s) { return (el_val_t)s_all_match(s, p_lower); }
|
||||
|
||||
/* Split on \n. \r\n is folded to \n first. Trailing empty after final \n
|
||||
* is dropped (so "a\nb\n" -> ["a", "b"], not ["a", "b", ""]). */
|
||||
el_val_t str_split_lines(el_val_t sv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
el_val_t lst = el_list_empty();
|
||||
if (!s) return lst;
|
||||
size_t n = strlen(s);
|
||||
/* Pre-scan: build into a normalized buffer with \r\n folded. */
|
||||
const char* line_start = s;
|
||||
for (size_t i = 0; i <= n; i++) {
|
||||
if (s[i] == '\n' || s[i] == '\0') {
|
||||
size_t len = (size_t)(s + i - line_start);
|
||||
/* Drop trailing \r if this was \r\n. */
|
||||
if (len > 0 && line_start[len - 1] == '\r') len--;
|
||||
/* Drop final trailing-empty-after-newline. */
|
||||
if (s[i] == '\0' && len == 0 && i > 0 && s[i - 1] == '\n') break;
|
||||
char* out = el_strbuf(len);
|
||||
memcpy(out, line_start, len);
|
||||
out[len] = '\0';
|
||||
lst = el_list_append(lst, el_wrap_str(out));
|
||||
if (s[i] == '\0') break;
|
||||
line_start = s + i + 1;
|
||||
}
|
||||
}
|
||||
return lst;
|
||||
}
|
||||
|
||||
el_val_t str_split_chars(el_val_t s) {
|
||||
return native_string_chars(s);
|
||||
}
|
||||
|
||||
/* Split into at most n parts. The (n-1)th split point is the LAST split;
|
||||
* after it, the remainder is appended verbatim including any further
|
||||
* separators. n <= 0 returns an empty list. n == 1 returns [s]. */
|
||||
el_val_t str_split_n(el_val_t sv, el_val_t sepv, el_val_t nv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
const char* sep = EL_CSTR(sepv);
|
||||
int64_t n = (int64_t)nv;
|
||||
el_val_t lst = el_list_empty();
|
||||
if (!s) return lst;
|
||||
if (n <= 0) return lst;
|
||||
if (n == 1 || !sep || !*sep) {
|
||||
lst = el_list_append(lst, el_wrap_str(el_strdup(s)));
|
||||
return lst;
|
||||
}
|
||||
size_t lp = strlen(sep);
|
||||
const char* p = s;
|
||||
int64_t parts = 0;
|
||||
const char* hit;
|
||||
while (parts < n - 1 && (hit = strstr(p, sep)) != NULL) {
|
||||
size_t len = (size_t)(hit - p);
|
||||
char* out = el_strbuf(len);
|
||||
memcpy(out, p, len);
|
||||
out[len] = '\0';
|
||||
lst = el_list_append(lst, el_wrap_str(out));
|
||||
p = hit + lp;
|
||||
parts++;
|
||||
}
|
||||
/* Remainder verbatim. */
|
||||
lst = el_list_append(lst, el_wrap_str(el_strdup(p)));
|
||||
return lst;
|
||||
}
|
||||
|
||||
/* Join a [String] with a separator. Empty list -> "". Single-element ->
|
||||
* that element. Non-string elements are stringified via int_to_str. */
|
||||
el_val_t str_join(el_val_t listv, el_val_t sepv) {
|
||||
return list_join(listv, sepv);
|
||||
}
|
||||
|
||||
/* ── List additions ──────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t list_push(el_val_t list, el_val_t elem) {
|
||||
|
||||
@@ -445,6 +445,49 @@ el_val_t str_format(el_val_t fmt, el_val_t data);
|
||||
el_val_t str_lower(el_val_t s);
|
||||
el_val_t str_upper(el_val_t s);
|
||||
|
||||
/* ── Text-processing primitives (Phase 1: byte/codepoint, ASCII char classes)
|
||||
* Phase 2 (filed): Unicode-grapheme awareness, NFC/NFD normalization, regex.
|
||||
* is_* predicates: empty input returns false; multi-char requires ALL bytes
|
||||
* to match. ASCII ranges only in Phase 1. */
|
||||
|
||||
/* Counting */
|
||||
el_val_t str_count(el_val_t s, el_val_t sub); /* non-overlapping */
|
||||
el_val_t str_count_chars(el_val_t s); /* codepoint count */
|
||||
el_val_t str_count_bytes(el_val_t s); /* alias of str_len */
|
||||
el_val_t str_count_lines(el_val_t s);
|
||||
el_val_t str_count_words(el_val_t s);
|
||||
el_val_t str_count_letters(el_val_t s); /* ASCII [A-Za-z] */
|
||||
el_val_t str_count_digits(el_val_t s); /* ASCII [0-9] */
|
||||
|
||||
/* Find / position */
|
||||
el_val_t str_index_of_all(el_val_t s, el_val_t sub); /* [Int] of byte offsets */
|
||||
el_val_t str_last_index_of(el_val_t s, el_val_t sub);
|
||||
el_val_t str_find_chars(el_val_t s, el_val_t any_of); /* first idx of any ch */
|
||||
|
||||
/* Transform */
|
||||
el_val_t str_repeat(el_val_t s, el_val_t n);
|
||||
el_val_t str_reverse(el_val_t s); /* by codepoint */
|
||||
el_val_t str_strip_prefix(el_val_t s, el_val_t prefix);
|
||||
el_val_t str_strip_suffix(el_val_t s, el_val_t suffix);
|
||||
el_val_t str_strip_chars(el_val_t s, el_val_t chars);
|
||||
el_val_t str_lstrip(el_val_t s);
|
||||
el_val_t str_rstrip(el_val_t s);
|
||||
|
||||
/* Char classification (Bool) */
|
||||
el_val_t is_letter(el_val_t s);
|
||||
el_val_t is_digit(el_val_t s);
|
||||
el_val_t is_alphanumeric(el_val_t s);
|
||||
el_val_t is_whitespace(el_val_t s);
|
||||
el_val_t is_punctuation(el_val_t s);
|
||||
el_val_t is_uppercase(el_val_t s);
|
||||
el_val_t is_lowercase(el_val_t s);
|
||||
|
||||
/* Split / join */
|
||||
el_val_t str_split_lines(el_val_t s);
|
||||
el_val_t str_split_chars(el_val_t s); /* alias of native_string_chars */
|
||||
el_val_t str_split_n(el_val_t s, el_val_t sep, el_val_t n);
|
||||
el_val_t str_join(el_val_t list, el_val_t sep); /* alias of list_join */
|
||||
|
||||
/* ── List additions ──────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t list_push(el_val_t list, el_val_t elem);
|
||||
|
||||
@@ -1257,6 +1257,15 @@ fn is_int_call(call_expr: Map<String, Any>) -> Bool {
|
||||
if str_eq(name, "str_index_of") { return true }
|
||||
if str_eq(name, "str_to_int") { return true }
|
||||
if str_eq(name, "str_char_code") { return true }
|
||||
if str_eq(name, "str_count") { return true }
|
||||
if str_eq(name, "str_count_chars") { return true }
|
||||
if str_eq(name, "str_count_bytes") { return true }
|
||||
if str_eq(name, "str_count_lines") { return true }
|
||||
if str_eq(name, "str_count_words") { return true }
|
||||
if str_eq(name, "str_count_letters") { return true }
|
||||
if str_eq(name, "str_count_digits") { return true }
|
||||
if str_eq(name, "str_last_index_of") { return true }
|
||||
if str_eq(name, "str_find_chars") { return true }
|
||||
if str_eq(name, "native_list_len") { return true }
|
||||
if str_eq(name, "el_list_len") { return true }
|
||||
if str_eq(name, "len") { return true }
|
||||
@@ -1858,6 +1867,35 @@ fn builtin_arity(name: String) -> Int {
|
||||
if str_eq(name, "str_format") { return 2 }
|
||||
if str_eq(name, "str_lower") { return 1 }
|
||||
if str_eq(name, "str_upper") { return 1 }
|
||||
// Text-processing primitives (Phase 1)
|
||||
if str_eq(name, "str_count") { return 2 }
|
||||
if str_eq(name, "str_count_chars") { return 1 }
|
||||
if str_eq(name, "str_count_bytes") { return 1 }
|
||||
if str_eq(name, "str_count_lines") { return 1 }
|
||||
if str_eq(name, "str_count_words") { return 1 }
|
||||
if str_eq(name, "str_count_letters") { return 1 }
|
||||
if str_eq(name, "str_count_digits") { return 1 }
|
||||
if str_eq(name, "str_index_of_all") { return 2 }
|
||||
if str_eq(name, "str_last_index_of") { return 2 }
|
||||
if str_eq(name, "str_find_chars") { return 2 }
|
||||
if str_eq(name, "str_repeat") { return 2 }
|
||||
if str_eq(name, "str_reverse") { return 1 }
|
||||
if str_eq(name, "str_strip_prefix") { return 2 }
|
||||
if str_eq(name, "str_strip_suffix") { return 2 }
|
||||
if str_eq(name, "str_strip_chars") { return 2 }
|
||||
if str_eq(name, "str_lstrip") { return 1 }
|
||||
if str_eq(name, "str_rstrip") { return 1 }
|
||||
if str_eq(name, "is_letter") { return 1 }
|
||||
if str_eq(name, "is_digit") { return 1 }
|
||||
if str_eq(name, "is_alphanumeric") { return 1 }
|
||||
if str_eq(name, "is_whitespace") { return 1 }
|
||||
if str_eq(name, "is_punctuation") { return 1 }
|
||||
if str_eq(name, "is_uppercase") { return 1 }
|
||||
if str_eq(name, "is_lowercase") { return 1 }
|
||||
if str_eq(name, "str_split_lines") { return 1 }
|
||||
if str_eq(name, "str_split_chars") { return 1 }
|
||||
if str_eq(name, "str_split_n") { return 3 }
|
||||
if str_eq(name, "str_join") { return 2 }
|
||||
// HTML sanitizer
|
||||
if str_eq(name, "el_html_sanitize") { return 2 }
|
||||
// Math
|
||||
|
||||
+10
-10
@@ -12,7 +12,7 @@
|
||||
|
||||
// ── Character helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
fn is_digit(ch: String) -> Bool {
|
||||
fn lex_is_digit(ch: String) -> Bool {
|
||||
if ch == "0" { return true }
|
||||
if ch == "1" { return true }
|
||||
if ch == "2" { return true }
|
||||
@@ -26,7 +26,7 @@ fn is_digit(ch: String) -> Bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn is_alpha(ch: String) -> Bool {
|
||||
fn lex_is_alpha(ch: String) -> Bool {
|
||||
if ch == "a" { return true }
|
||||
if ch == "b" { return true }
|
||||
if ch == "c" { return true }
|
||||
@@ -83,13 +83,13 @@ fn is_alpha(ch: String) -> Bool {
|
||||
}
|
||||
|
||||
fn is_alnum_or_underscore(ch: String) -> Bool {
|
||||
if is_digit(ch) { return true }
|
||||
if is_alpha(ch) { return true }
|
||||
if lex_is_digit(ch) { return true }
|
||||
if lex_is_alpha(ch) { return true }
|
||||
if ch == "_" { return true }
|
||||
false
|
||||
}
|
||||
|
||||
fn is_whitespace(ch: String) -> Bool {
|
||||
fn lex_is_whitespace(ch: String) -> Bool {
|
||||
if ch == " " { return true }
|
||||
if ch == "\t" { return true }
|
||||
if ch == "\n" { return true }
|
||||
@@ -163,7 +163,7 @@ fn scan_digits(chars: [String], start: Int, total: Int) -> Map<String, Any> {
|
||||
let running = false
|
||||
} else {
|
||||
let ch: String = native_list_get(chars, i)
|
||||
if is_digit(ch) {
|
||||
if lex_is_digit(ch) {
|
||||
let text = text + ch
|
||||
let i = i + 1
|
||||
} else {
|
||||
@@ -469,7 +469,7 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let ch: String = native_list_get(chars, i)
|
||||
|
||||
// Skip whitespace
|
||||
if is_whitespace(ch) {
|
||||
if lex_is_whitespace(ch) {
|
||||
let i = i + 1
|
||||
} else {
|
||||
// Line comments: //
|
||||
@@ -519,7 +519,7 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let i = new_pos
|
||||
} else {
|
||||
// Number literal
|
||||
if is_digit(ch) {
|
||||
if lex_is_digit(ch) {
|
||||
let result = scan_digits(chars, i, total)
|
||||
let num_text: String = result["text"]
|
||||
let new_pos: Int = result["pos"]
|
||||
@@ -530,7 +530,7 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let after_dot = new_pos + 1
|
||||
if after_dot < total {
|
||||
let after_dot_ch: String = native_list_get(chars, after_dot)
|
||||
if is_digit(after_dot_ch) {
|
||||
if lex_is_digit(after_dot_ch) {
|
||||
let frac_result = scan_digits(chars, after_dot, total)
|
||||
let frac_text: String = frac_result["text"]
|
||||
let frac_pos: Int = frac_result["pos"]
|
||||
@@ -554,7 +554,7 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
}
|
||||
} else {
|
||||
// Identifier or keyword
|
||||
if is_alpha(ch) || ch == "_" {
|
||||
if lex_is_alpha(ch) || ch == "_" {
|
||||
let result = scan_ident(chars, i, total)
|
||||
let word: String = result["text"]
|
||||
let new_pos: Int = result["pos"]
|
||||
|
||||
+57
-10
@@ -12,7 +12,7 @@
|
||||
|
||||
// ── Character helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
fn is_digit(ch: String) -> Bool {
|
||||
fn lex_is_digit(ch: String) -> Bool {
|
||||
if ch == "0" { return true }
|
||||
if ch == "1" { return true }
|
||||
if ch == "2" { return true }
|
||||
@@ -26,7 +26,7 @@ fn is_digit(ch: String) -> Bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn is_alpha(ch: String) -> Bool {
|
||||
fn lex_is_alpha(ch: String) -> Bool {
|
||||
if ch == "a" { return true }
|
||||
if ch == "b" { return true }
|
||||
if ch == "c" { return true }
|
||||
@@ -83,13 +83,13 @@ fn is_alpha(ch: String) -> Bool {
|
||||
}
|
||||
|
||||
fn is_alnum_or_underscore(ch: String) -> Bool {
|
||||
if is_digit(ch) { return true }
|
||||
if is_alpha(ch) { return true }
|
||||
if lex_is_digit(ch) { return true }
|
||||
if lex_is_alpha(ch) { return true }
|
||||
if ch == "_" { return true }
|
||||
false
|
||||
}
|
||||
|
||||
fn is_whitespace(ch: String) -> Bool {
|
||||
fn lex_is_whitespace(ch: String) -> Bool {
|
||||
if ch == " " { return true }
|
||||
if ch == "\t" { return true }
|
||||
if ch == "\n" { return true }
|
||||
@@ -163,7 +163,7 @@ fn scan_digits(chars: [String], start: Int, total: Int) -> Map<String, Any> {
|
||||
let running = false
|
||||
} else {
|
||||
let ch: String = native_list_get(chars, i)
|
||||
if is_digit(ch) {
|
||||
if lex_is_digit(ch) {
|
||||
let text = text + ch
|
||||
let i = i + 1
|
||||
} else {
|
||||
@@ -267,7 +267,7 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let ch: String = native_list_get(chars, i)
|
||||
|
||||
// Skip whitespace
|
||||
if is_whitespace(ch) {
|
||||
if lex_is_whitespace(ch) {
|
||||
let i = i + 1
|
||||
} else {
|
||||
// Line comments: //
|
||||
@@ -309,7 +309,7 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let i = new_pos
|
||||
} else {
|
||||
// Number literal
|
||||
if is_digit(ch) {
|
||||
if lex_is_digit(ch) {
|
||||
let result = scan_digits(chars, i, total)
|
||||
let num_text: String = result["text"]
|
||||
let new_pos: Int = result["pos"]
|
||||
@@ -320,7 +320,7 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
let after_dot = new_pos + 1
|
||||
if after_dot < total {
|
||||
let after_dot_ch: String = native_list_get(chars, after_dot)
|
||||
if is_digit(after_dot_ch) {
|
||||
if lex_is_digit(after_dot_ch) {
|
||||
let frac_result = scan_digits(chars, after_dot, total)
|
||||
let frac_text: String = frac_result["text"]
|
||||
let frac_pos: Int = frac_result["pos"]
|
||||
@@ -344,7 +344,7 @@ fn lex(source: String) -> [Map<String, Any>] {
|
||||
}
|
||||
} else {
|
||||
// Identifier or keyword
|
||||
if is_alpha(ch) || ch == "_" {
|
||||
if lex_is_alpha(ch) || ch == "_" {
|
||||
let result = scan_ident(chars, i, total)
|
||||
let word: String = result["text"]
|
||||
let new_pos: Int = result["pos"]
|
||||
@@ -2436,6 +2436,15 @@ fn is_int_call(call_expr: Map<String, Any>) -> Bool {
|
||||
if str_eq(name, "str_index_of") { return true }
|
||||
if str_eq(name, "str_to_int") { return true }
|
||||
if str_eq(name, "str_char_code") { return true }
|
||||
if str_eq(name, "str_count") { return true }
|
||||
if str_eq(name, "str_count_chars") { return true }
|
||||
if str_eq(name, "str_count_bytes") { return true }
|
||||
if str_eq(name, "str_count_lines") { return true }
|
||||
if str_eq(name, "str_count_words") { return true }
|
||||
if str_eq(name, "str_count_letters") { return true }
|
||||
if str_eq(name, "str_count_digits") { return true }
|
||||
if str_eq(name, "str_last_index_of") { return true }
|
||||
if str_eq(name, "str_find_chars") { return true }
|
||||
if str_eq(name, "native_list_len") { return true }
|
||||
if str_eq(name, "el_list_len") { return true }
|
||||
if str_eq(name, "len") { return true }
|
||||
@@ -2676,6 +2685,35 @@ fn builtin_arity(name: String) -> Int {
|
||||
if str_eq(name, "str_format") { return 2 }
|
||||
if str_eq(name, "str_lower") { return 1 }
|
||||
if str_eq(name, "str_upper") { return 1 }
|
||||
// Text-processing primitives (Phase 1)
|
||||
if str_eq(name, "str_count") { return 2 }
|
||||
if str_eq(name, "str_count_chars") { return 1 }
|
||||
if str_eq(name, "str_count_bytes") { return 1 }
|
||||
if str_eq(name, "str_count_lines") { return 1 }
|
||||
if str_eq(name, "str_count_words") { return 1 }
|
||||
if str_eq(name, "str_count_letters") { return 1 }
|
||||
if str_eq(name, "str_count_digits") { return 1 }
|
||||
if str_eq(name, "str_index_of_all") { return 2 }
|
||||
if str_eq(name, "str_last_index_of") { return 2 }
|
||||
if str_eq(name, "str_find_chars") { return 2 }
|
||||
if str_eq(name, "str_repeat") { return 2 }
|
||||
if str_eq(name, "str_reverse") { return 1 }
|
||||
if str_eq(name, "str_strip_prefix") { return 2 }
|
||||
if str_eq(name, "str_strip_suffix") { return 2 }
|
||||
if str_eq(name, "str_strip_chars") { return 2 }
|
||||
if str_eq(name, "str_lstrip") { return 1 }
|
||||
if str_eq(name, "str_rstrip") { return 1 }
|
||||
if str_eq(name, "is_letter") { return 1 }
|
||||
if str_eq(name, "is_digit") { return 1 }
|
||||
if str_eq(name, "is_alphanumeric") { return 1 }
|
||||
if str_eq(name, "is_whitespace") { return 1 }
|
||||
if str_eq(name, "is_punctuation") { return 1 }
|
||||
if str_eq(name, "is_uppercase") { return 1 }
|
||||
if str_eq(name, "is_lowercase") { return 1 }
|
||||
if str_eq(name, "str_split_lines") { return 1 }
|
||||
if str_eq(name, "str_split_chars") { return 1 }
|
||||
if str_eq(name, "str_split_n") { return 3 }
|
||||
if str_eq(name, "str_join") { return 2 }
|
||||
// Math
|
||||
if str_eq(name, "el_abs") { return 1 }
|
||||
if str_eq(name, "el_max") { return 2 }
|
||||
@@ -3426,6 +3464,15 @@ fn js_is_int_call(call_expr: Map<String, Any>) -> Bool {
|
||||
if str_eq(name, "str_index_of") { return true }
|
||||
if str_eq(name, "str_to_int") { return true }
|
||||
if str_eq(name, "str_char_code") { return true }
|
||||
if str_eq(name, "str_count") { return true }
|
||||
if str_eq(name, "str_count_chars") { return true }
|
||||
if str_eq(name, "str_count_bytes") { return true }
|
||||
if str_eq(name, "str_count_lines") { return true }
|
||||
if str_eq(name, "str_count_words") { return true }
|
||||
if str_eq(name, "str_count_letters") { return true }
|
||||
if str_eq(name, "str_count_digits") { return true }
|
||||
if str_eq(name, "str_last_index_of") { return true }
|
||||
if str_eq(name, "str_find_chars") { return true }
|
||||
if str_eq(name, "native_list_len") { return true }
|
||||
if str_eq(name, "el_list_len") { return true }
|
||||
if str_eq(name, "len") { return true }
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
// char-classes.el — ASCII character classification.
|
||||
// is_letter("A") && is_digit("7") && is_whitespace(" ")
|
||||
// && !is_letter("3") && !is_digit("X")
|
||||
fn run_test() -> Bool {
|
||||
if !is_letter("A") { return false }
|
||||
if !is_digit("7") { return false }
|
||||
if !is_whitespace(" ") { return false }
|
||||
if is_letter("3") { return false }
|
||||
if is_digit("X") { return false }
|
||||
return true
|
||||
}
|
||||
|
||||
fn main() -> Void {
|
||||
if run_test() {
|
||||
println("true")
|
||||
} else {
|
||||
println("false")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
// count-lines-words-letters.el — composite count test.
|
||||
// Input "Hello world\nGoodbye world\n":
|
||||
// lines: 2 (each \n closes a line)
|
||||
// words: 4 (Hello, world, Goodbye, world)
|
||||
// letters: 22 (Hello=5 + world=5 + Goodbye=7 + world=5)
|
||||
fn run_test() -> String {
|
||||
let s: String = "Hello world\nGoodbye world\n"
|
||||
let lines: Int = str_count_lines(s)
|
||||
let words: Int = str_count_words(s)
|
||||
let letters: Int = str_count_letters(s)
|
||||
return int_to_str(lines) + "/" + int_to_str(words) + "/" + int_to_str(letters)
|
||||
}
|
||||
|
||||
fn main() -> Void {
|
||||
println(run_test())
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// count-overlap-skip.el — non-overlapping advance: "aaaa" / "aa" -> 2.
|
||||
// After each match, the cursor advances by len(sub), so overlapping matches
|
||||
// are skipped. This is the universal default in C/Python/Go/etc.
|
||||
fn run_test() -> Int {
|
||||
let s: String = "aaaa"
|
||||
return str_count(s, "aa")
|
||||
}
|
||||
|
||||
fn main() -> Void {
|
||||
println(int_to_str(run_test()))
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
// count-substring.el — non-overlapping substring count.
|
||||
fn run_test() -> Int {
|
||||
let s: String = "abc abc abc"
|
||||
return str_count(s, "abc")
|
||||
}
|
||||
|
||||
fn main() -> Void {
|
||||
println(int_to_str(run_test()))
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
// index-of-all.el — every byte offset of a substring.
|
||||
// "abXcdXefX" / "X" -> [2, 5, 8]. Empty list when no matches.
|
||||
fn run_test() -> String {
|
||||
let positions: [Int] = str_index_of_all("abXcdXefX", "X")
|
||||
let n: Int = native_list_len(positions)
|
||||
let out: String = ""
|
||||
let i: Int = 0
|
||||
while i < n {
|
||||
let p: Int = native_list_get(positions, i)
|
||||
if i > 0 {
|
||||
let out = out + ","
|
||||
}
|
||||
let out = out + int_to_str(p)
|
||||
let i = i + 1
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
fn main() -> Void {
|
||||
println(run_test())
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
// join.el — list-of-string join with separator.
|
||||
fn run_test() -> String {
|
||||
let parts: [String] = native_list_empty()
|
||||
let parts = native_list_append(parts, "alpha")
|
||||
let parts = native_list_append(parts, "beta")
|
||||
let parts = native_list_append(parts, "gamma")
|
||||
return str_join(parts, ", ")
|
||||
}
|
||||
|
||||
fn main() -> Void {
|
||||
println(run_test())
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
// split-lines.el — \n-delimited split, \r\n folded, trailing empty dropped.
|
||||
// "alpha\nbeta\r\ngamma\n" -> ["alpha", "beta", "gamma"], len = 3.
|
||||
fn run_test() -> Int {
|
||||
let lines: [String] = str_split_lines("alpha\nbeta\r\ngamma\n")
|
||||
return native_list_len(lines)
|
||||
}
|
||||
|
||||
fn main() -> Void {
|
||||
println(int_to_str(run_test()))
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
// transform-suite.el — repeat, reverse, strip prefix/suffix/chars.
|
||||
fn run_test() -> String {
|
||||
let a: String = str_repeat("ab", 3) // "ababab"
|
||||
let b: String = str_reverse("hello") // "olleh"
|
||||
let c: String = str_strip_prefix("foobar", "foo") // "bar"
|
||||
let d: String = str_strip_suffix("hello.md", ".md") // "hello"
|
||||
let e: String = str_strip_chars(" \thello \n", " \t\n") // "hello"
|
||||
return a + "|" + b + "|" + c + "|" + d + "|" + e
|
||||
}
|
||||
|
||||
fn main() -> Void {
|
||||
println(run_test())
|
||||
}
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env bash
|
||||
# run.sh — build and execute the text/ acceptance corpus.
|
||||
#
|
||||
# Each examples/<case>.el is a self-contained El program with a fn main()
|
||||
# that prints a single deterministic result line. The runner compiles each
|
||||
# via the canonical native elc, links it against the shared C runtime, runs
|
||||
# it, and asserts the output matches the expected value.
|
||||
|
||||
set -uo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
EL_HOME="${EL_HOME:-$(cd ../.. && pwd)}"
|
||||
ELC="${ELC:-${EL_HOME}/dist/platform/elc}"
|
||||
RUNTIME_DIR="${EL_HOME}/el-compiler/runtime"
|
||||
|
||||
if [ ! -x "${ELC}" ]; then
|
||||
echo "elc not found at ${ELC}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
FAILED_NAMES=()
|
||||
|
||||
run_runtime_case() {
|
||||
local name="$1"
|
||||
local src="$2"
|
||||
local expected="$3"
|
||||
|
||||
local out_c
|
||||
local out_bin
|
||||
out_c="$(mktemp -t text_test.XXXXXX).c"
|
||||
out_bin="$(mktemp -t text_test.XXXXXX)"
|
||||
|
||||
if ! "${ELC}" "${src}" > "${out_c}" 2>/tmp/text_test.elc.err; then
|
||||
echo "FAIL ${name} — elc emit failed:"
|
||||
cat /tmp/text_test.elc.err | sed 's/^/ /'
|
||||
FAIL=$((FAIL+1))
|
||||
FAILED_NAMES+=("${name}")
|
||||
rm -f "${out_c}" "${out_bin}"
|
||||
return
|
||||
fi
|
||||
|
||||
if ! cc -O2 -I "${RUNTIME_DIR}" "${out_c}" "${RUNTIME_DIR}/el_runtime.c" \
|
||||
-lcurl -lpthread -o "${out_bin}" 2>/tmp/text_test.cc.err; then
|
||||
echo "FAIL ${name} — cc failed:"
|
||||
cat /tmp/text_test.cc.err | sed 's/^/ /'
|
||||
FAIL=$((FAIL+1))
|
||||
FAILED_NAMES+=("${name}")
|
||||
rm -f "${out_c}" "${out_bin}"
|
||||
return
|
||||
fi
|
||||
|
||||
local got
|
||||
got="$("${out_bin}" 2>&1)"
|
||||
|
||||
if [ "${got}" = "${expected}" ]; then
|
||||
echo "PASS ${name}"
|
||||
PASS=$((PASS+1))
|
||||
else
|
||||
echo "FAIL ${name} expected: '${expected}', got: '${got}'"
|
||||
FAIL=$((FAIL+1))
|
||||
FAILED_NAMES+=("${name}")
|
||||
fi
|
||||
|
||||
rm -f "${out_c}" "${out_bin}"
|
||||
}
|
||||
|
||||
echo "==> Running text-primitives acceptance corpus"
|
||||
echo
|
||||
|
||||
run_runtime_case "count-substring" examples/count-substring.el "3"
|
||||
run_runtime_case "count-overlap-skip" examples/count-overlap-skip.el "2"
|
||||
run_runtime_case "count-lines-words-letters" examples/count-lines-words-letters.el "2/4/22"
|
||||
run_runtime_case "index-of-all" examples/index-of-all.el "2,5,8"
|
||||
run_runtime_case "transform-suite" examples/transform-suite.el "ababab|olleh|bar|hello|hello"
|
||||
run_runtime_case "char-classes" examples/char-classes.el "true"
|
||||
run_runtime_case "split-lines" examples/split-lines.el "3"
|
||||
run_runtime_case "join" examples/join.el "alpha, beta, gamma"
|
||||
|
||||
echo
|
||||
echo "${PASS} passed, ${FAIL} failed"
|
||||
if [ "${FAIL}" -gt 0 ]; then
|
||||
echo "failed: ${FAILED_NAMES[*]}"
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
@@ -0,0 +1,11 @@
|
||||
// runner.el — entry point for the text/ acceptance corpus.
|
||||
//
|
||||
// Each text/examples/*.el is its own El program with its own fn main().
|
||||
// Compile, link, and run-output-diff is handled by tests/text/run.sh —
|
||||
// this file is the El-side stub kept for pattern parity with tests/time/
|
||||
// and tests/calendar/.
|
||||
|
||||
fn main() -> Void {
|
||||
println("text/ acceptance corpus is driven by run.sh — invoke that directly.")
|
||||
println("Each examples/*.el is a self-contained program; runner.el is a parity stub.")
|
||||
}
|
||||
Reference in New Issue
Block a user