From 3a83b6eb802e93a802a2646e0b482c01b3b44b06 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Sat, 2 May 2026 13:37:30 -0500 Subject: [PATCH] add text-processing primitives to el runtime 24 new functions covering counting (str_count, str_count_chars, str_count_bytes, str_count_lines, str_count_words, str_count_letters, str_count_digits), finding (str_index_of_all, str_last_index_of, str_find_chars), transforming (str_repeat, str_reverse, str_strip_prefix/suffix/chars, str_lstrip, str_rstrip), character classification (is_letter, is_digit, is_alphanumeric, is_whitespace, is_punctuation, is_uppercase, is_lowercase), and splitting/joining (str_split_lines, str_split_chars, str_split_n, str_join). Phase 1 is byte-level + ASCII character classes. Unicode-grapheme awareness, normalization, and regex are Phase 2 (filed separately). Lexer-internal helpers is_digit, is_alpha, is_whitespace renamed to lex_is_digit, lex_is_alpha, lex_is_whitespace to free the public names for the runtime exports. The El compiler's lexer.el and the bundled elc-combined.el both updated. Codegen registrations: builtin_arity entries for all 24 functions, is_int_call entries for the Int-returning ones (str_count*, str_last_index_of, str_find_chars) so the + operator dispatches as arithmetic when applicable. Tests: tests/text/ corpus with 8 acceptance cases covering the surface (count-substring, count-overlap-skip, count-lines-words-letters, index-of-all, transform-suite, char-classes, split-lines, join). All pass against a fold-fn-main-aware elc bootstrap (see ELC env var override in run.sh). Self-host fixed point: elc-combined.el's emit-main pass does not currently fold the fn main body into C's main, a pre-existing condition that surfaces as a 39-line gen2/gen3 diff with empty main in gen3. The committed dist/platform/elc binary has the fold logic so all tests pass against it. Filing the elc-combined fold-fn-main fix separately. This commit does not introduce new self-host drift. --- el-compiler/runtime/el_runtime.c | 370 ++++++++++++++++++ el-compiler/runtime/el_runtime.h | 43 ++ el-compiler/src/codegen.el | 38 ++ el-compiler/src/lexer.el | 20 +- elc-combined.el | 67 +++- tests/text/examples/char-classes.el | 19 + .../examples/count-lines-words-letters.el | 16 + tests/text/examples/count-overlap-skip.el | 11 + tests/text/examples/count-substring.el | 9 + tests/text/examples/index-of-all.el | 21 + tests/text/examples/join.el | 12 + tests/text/examples/split-lines.el | 10 + tests/text/examples/transform-suite.el | 13 + tests/text/run.sh | 87 ++++ tests/text/runner.el | 11 + 15 files changed, 727 insertions(+), 20 deletions(-) create mode 100644 tests/text/examples/char-classes.el create mode 100644 tests/text/examples/count-lines-words-letters.el create mode 100644 tests/text/examples/count-overlap-skip.el create mode 100644 tests/text/examples/count-substring.el create mode 100644 tests/text/examples/index-of-all.el create mode 100644 tests/text/examples/join.el create mode 100644 tests/text/examples/split-lines.el create mode 100644 tests/text/examples/transform-suite.el create mode 100755 tests/text/run.sh create mode 100644 tests/text/runner.el diff --git a/el-compiler/runtime/el_runtime.c b/el-compiler/runtime/el_runtime.c index da8ab3f..2a3beaf 100644 --- a/el-compiler/runtime/el_runtime.c +++ b/el-compiler/runtime/el_runtime.c @@ -4805,6 +4805,376 @@ el_val_t str_format(el_val_t fmt, el_val_t data) { el_val_t str_lower(el_val_t s) { return str_to_lower(s); } el_val_t str_upper(el_val_t s) { return str_to_upper(s); } +/* ── Text-processing primitives (Phase 1: byte/codepoint, ASCII char classes) + * + * Phase 1 covers the operations every text-handling caller used to roll by + * hand on top of str_index_of + str_slice. The character-class predicates + * (is_letter / is_digit / ...) are ASCII only — Unicode-grapheme awareness, + * NFC/NFD normalization, and regex are Phase 2. Single-char input checks the + * first byte; multi-char input requires ALL bytes to match (false otherwise). + * + * Counting: + * str_count non-overlapping occurrences of sub in s + * str_count_chars codepoint count (UTF-8 leading-byte count) + * str_count_bytes explicit byte length (alias of str_len) + * str_count_lines \n-delimited line count (\r\n folded to \n) + * str_count_words whitespace-delimited tokens, non-empty only + * str_count_letters ASCII [A-Za-z] + * str_count_digits ASCII [0-9] + * + * Find / position: + * str_index_of_all all byte offsets of sub, [] if none + * str_last_index_of last byte offset of sub, -1 if not found + * str_find_chars first index of any char in any_of, -1 if none + * + * Transform: + * str_repeat s * n (non-negative) + * str_reverse codepoint-reversed (NOT grapheme-aware) + * str_strip_prefix s without prefix if present, else s + * str_strip_suffix s without suffix if present, else s + * str_strip_chars strip leading+trailing chars matching any in chars + * str_lstrip strip leading whitespace + * str_rstrip strip trailing whitespace + * + * Char classification (Bool): + * is_letter, is_digit, is_alphanumeric, is_whitespace, + * is_punctuation, is_uppercase, is_lowercase + * + * Splitting: + * str_split_lines \n-delimited (\r\n folded). Trailing empty dropped. + * str_split_chars alias of native_string_chars in str_ namespace + * str_split_n split into at most n parts (last part keeps the + * rest verbatim, including any further separators) + * + * Joining: + * str_join [String] -> String, sep between elements + */ + +/* Count non-overlapping occurrences of sub in s. Empty sub returns 0. */ +el_val_t str_count(el_val_t sv, el_val_t subv) { + const char* s = EL_CSTR(sv); + const char* sub = EL_CSTR(subv); + if (!s || !sub || !*sub) return 0; + size_t lp = strlen(sub); + int64_t count = 0; + const char* p = s; + while ((p = strstr(p, sub)) != NULL) { + count++; + p += lp; /* non-overlapping advance */ + } + return (el_val_t)count; +} + +/* Codepoint count: walk bytes, count those NOT matching 10xxxxxx. */ +el_val_t str_count_chars(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return 0; + int64_t count = 0; + for (const unsigned char* p = (const unsigned char*)s; *p; p++) { + if ((*p & 0xC0) != 0x80) count++; + } + return (el_val_t)count; +} + +el_val_t str_count_bytes(el_val_t sv) { + return str_len(sv); +} + +el_val_t str_count_lines(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s || !*s) return 0; + int64_t count = 0; + int has_content = 0; + for (const char* p = s; *p; p++) { + has_content = 1; + if (*p == '\n') { + count++; + has_content = 0; /* the \n closed the line */ + } + } + if (has_content) count++; /* trailing line with no terminator */ + return (el_val_t)count; +} + +el_val_t str_count_words(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return 0; + int64_t count = 0; + int in_word = 0; + for (const unsigned char* p = (const unsigned char*)s; *p; p++) { + if (isspace(*p)) { + in_word = 0; + } else if (!in_word) { + in_word = 1; + count++; + } + } + return (el_val_t)count; +} + +el_val_t str_count_letters(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return 0; + int64_t count = 0; + for (const unsigned char* p = (const unsigned char*)s; *p; p++) { + if ((*p >= 'A' && *p <= 'Z') || (*p >= 'a' && *p <= 'z')) count++; + } + return (el_val_t)count; +} + +el_val_t str_count_digits(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return 0; + int64_t count = 0; + for (const unsigned char* p = (const unsigned char*)s; *p; p++) { + if (*p >= '0' && *p <= '9') count++; + } + return (el_val_t)count; +} + +el_val_t str_index_of_all(el_val_t sv, el_val_t subv) { + const char* s = EL_CSTR(sv); + const char* sub = EL_CSTR(subv); + el_val_t lst = el_list_empty(); + if (!s || !sub || !*sub) return lst; + size_t lp = strlen(sub); + const char* p = s; + const char* hit; + while ((hit = strstr(p, sub)) != NULL) { + lst = el_list_append(lst, (el_val_t)(int64_t)(hit - s)); + p = hit + lp; + } + return lst; +} + +el_val_t str_last_index_of(el_val_t sv, el_val_t subv) { + const char* s = EL_CSTR(sv); + const char* sub = EL_CSTR(subv); + if (!s || !sub || !*sub) return -1; + size_t lp = strlen(sub); + int64_t last = -1; + const char* p = s; + const char* hit; + while ((hit = strstr(p, sub)) != NULL) { + last = (int64_t)(hit - s); + p = hit + lp; + } + return (el_val_t)last; +} + +el_val_t str_find_chars(el_val_t sv, el_val_t any_of_v) { + const char* s = EL_CSTR(sv); + const char* any = EL_CSTR(any_of_v); + if (!s || !any || !*any) return -1; + for (const char* p = s; *p; p++) { + if (strchr(any, *p)) return (el_val_t)(int64_t)(p - s); + } + return -1; +} + +el_val_t str_repeat(el_val_t sv, el_val_t nv) { + const char* s = EL_CSTR(sv); + int64_t n = (int64_t)nv; + if (!s || n <= 0) return el_wrap_str(el_strdup("")); + size_t ls = strlen(s); + if (ls == 0) return el_wrap_str(el_strdup("")); + size_t total = ls * (size_t)n; + char* out = el_strbuf(total); + for (int64_t i = 0; i < n; i++) { + memcpy(out + i * ls, s, ls); + } + out[total] = '\0'; + return el_wrap_str(out); +} + +/* Reverse by codepoint: walk codepoints, copy each backwards into the output. + * NOT grapheme-aware (Phase 2). Combining marks attached to a base codepoint + * will detach. ASCII strings are byte-reverse equivalent. */ +el_val_t str_reverse(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return el_wrap_str(el_strdup("")); + size_t n = strlen(s); + char* out = el_strbuf(n); + /* Walk forward, find each codepoint's byte length, then copy from the end. */ + size_t out_pos = n; + const unsigned char* p = (const unsigned char*)s; + while (*p) { + int cp_len; + if ((*p & 0x80) == 0x00) cp_len = 1; + else if ((*p & 0xE0) == 0xC0) cp_len = 2; + else if ((*p & 0xF0) == 0xE0) cp_len = 3; + else if ((*p & 0xF8) == 0xF0) cp_len = 4; + else cp_len = 1; /* invalid byte: passthrough */ + out_pos -= cp_len; + memcpy(out + out_pos, p, cp_len); + p += cp_len; + } + out[n] = '\0'; + return el_wrap_str(out); +} + +el_val_t str_strip_prefix(el_val_t sv, el_val_t prefv) { + const char* s = EL_CSTR(sv); + const char* pref = EL_CSTR(prefv); + if (!s) return el_wrap_str(el_strdup("")); + if (!pref || !*pref) return el_wrap_str(el_strdup(s)); + size_t lp = strlen(pref); + size_t ls = strlen(s); + if (lp <= ls && strncmp(s, pref, lp) == 0) { + char* out = el_strbuf(ls - lp); + memcpy(out, s + lp, ls - lp); + out[ls - lp] = '\0'; + return el_wrap_str(out); + } + return el_wrap_str(el_strdup(s)); +} + +el_val_t str_strip_suffix(el_val_t sv, el_val_t sufv) { + const char* s = EL_CSTR(sv); + const char* suf = EL_CSTR(sufv); + if (!s) return el_wrap_str(el_strdup("")); + if (!suf || !*suf) return el_wrap_str(el_strdup(s)); + size_t ls = strlen(s); + size_t lsuf = strlen(suf); + if (lsuf <= ls && strcmp(s + ls - lsuf, suf) == 0) { + char* out = el_strbuf(ls - lsuf); + memcpy(out, s, ls - lsuf); + out[ls - lsuf] = '\0'; + return el_wrap_str(out); + } + return el_wrap_str(el_strdup(s)); +} + +el_val_t str_strip_chars(el_val_t sv, el_val_t charsv) { + const char* s = EL_CSTR(sv); + const char* chars = EL_CSTR(charsv); + if (!s) return el_wrap_str(el_strdup("")); + if (!chars || !*chars) return el_wrap_str(el_strdup(s)); + const char* start = s; + while (*start && strchr(chars, *start)) start++; + size_t n = strlen(start); + while (n > 0 && strchr(chars, start[n - 1])) n--; + char* out = el_strbuf(n); + memcpy(out, start, n); + out[n] = '\0'; + return el_wrap_str(out); +} + +el_val_t str_lstrip(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return el_wrap_str(el_strdup("")); + while (*s && isspace((unsigned char)*s)) s++; + return el_wrap_str(el_strdup(s)); +} + +el_val_t str_rstrip(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return el_wrap_str(el_strdup("")); + size_t n = strlen(s); + while (n > 0 && isspace((unsigned char)s[n - 1])) n--; + char* out = el_strbuf(n); + memcpy(out, s, n); + out[n] = '\0'; + return el_wrap_str(out); +} + +/* Character classification. + * Empty input returns false. Multi-char input requires ALL bytes to match. + * ASCII range only; Phase 2 will widen to Unicode. */ +static int s_all_match(el_val_t sv, int (*pred)(unsigned char)) { + const char* s = EL_CSTR(sv); + if (!s || !*s) return 0; + for (const unsigned char* p = (const unsigned char*)s; *p; p++) { + if (!pred(*p)) return 0; + } + return 1; +} + +static int p_letter(unsigned char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); } +static int p_digit(unsigned char c) { return c >= '0' && c <= '9'; } +static int p_alnum(unsigned char c) { return p_letter(c) || p_digit(c); } +static int p_white(unsigned char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; } +static int p_punct(unsigned char c) { return ispunct(c) ? 1 : 0; } +static int p_upper(unsigned char c) { return c >= 'A' && c <= 'Z'; } +static int p_lower(unsigned char c) { return c >= 'a' && c <= 'z'; } + +el_val_t is_letter(el_val_t s) { return (el_val_t)s_all_match(s, p_letter); } +el_val_t is_digit(el_val_t s) { return (el_val_t)s_all_match(s, p_digit); } +el_val_t is_alphanumeric(el_val_t s) { return (el_val_t)s_all_match(s, p_alnum); } +el_val_t is_whitespace(el_val_t s) { return (el_val_t)s_all_match(s, p_white); } +el_val_t is_punctuation(el_val_t s) { return (el_val_t)s_all_match(s, p_punct); } +el_val_t is_uppercase(el_val_t s) { return (el_val_t)s_all_match(s, p_upper); } +el_val_t is_lowercase(el_val_t s) { return (el_val_t)s_all_match(s, p_lower); } + +/* Split on \n. \r\n is folded to \n first. Trailing empty after final \n + * is dropped (so "a\nb\n" -> ["a", "b"], not ["a", "b", ""]). */ +el_val_t str_split_lines(el_val_t sv) { + const char* s = EL_CSTR(sv); + el_val_t lst = el_list_empty(); + if (!s) return lst; + size_t n = strlen(s); + /* Pre-scan: build into a normalized buffer with \r\n folded. */ + const char* line_start = s; + for (size_t i = 0; i <= n; i++) { + if (s[i] == '\n' || s[i] == '\0') { + size_t len = (size_t)(s + i - line_start); + /* Drop trailing \r if this was \r\n. */ + if (len > 0 && line_start[len - 1] == '\r') len--; + /* Drop final trailing-empty-after-newline. */ + if (s[i] == '\0' && len == 0 && i > 0 && s[i - 1] == '\n') break; + char* out = el_strbuf(len); + memcpy(out, line_start, len); + out[len] = '\0'; + lst = el_list_append(lst, el_wrap_str(out)); + if (s[i] == '\0') break; + line_start = s + i + 1; + } + } + return lst; +} + +el_val_t str_split_chars(el_val_t s) { + return native_string_chars(s); +} + +/* Split into at most n parts. The (n-1)th split point is the LAST split; + * after it, the remainder is appended verbatim including any further + * separators. n <= 0 returns an empty list. n == 1 returns [s]. */ +el_val_t str_split_n(el_val_t sv, el_val_t sepv, el_val_t nv) { + const char* s = EL_CSTR(sv); + const char* sep = EL_CSTR(sepv); + int64_t n = (int64_t)nv; + el_val_t lst = el_list_empty(); + if (!s) return lst; + if (n <= 0) return lst; + if (n == 1 || !sep || !*sep) { + lst = el_list_append(lst, el_wrap_str(el_strdup(s))); + return lst; + } + size_t lp = strlen(sep); + const char* p = s; + int64_t parts = 0; + const char* hit; + while (parts < n - 1 && (hit = strstr(p, sep)) != NULL) { + size_t len = (size_t)(hit - p); + char* out = el_strbuf(len); + memcpy(out, p, len); + out[len] = '\0'; + lst = el_list_append(lst, el_wrap_str(out)); + p = hit + lp; + parts++; + } + /* Remainder verbatim. */ + lst = el_list_append(lst, el_wrap_str(el_strdup(p))); + return lst; +} + +/* Join a [String] with a separator. Empty list -> "". Single-element -> + * that element. Non-string elements are stringified via int_to_str. */ +el_val_t str_join(el_val_t listv, el_val_t sepv) { + return list_join(listv, sepv); +} + /* ── List additions ──────────────────────────────────────────────────────── */ el_val_t list_push(el_val_t list, el_val_t elem) { diff --git a/el-compiler/runtime/el_runtime.h b/el-compiler/runtime/el_runtime.h index a5c2733..68bae8b 100644 --- a/el-compiler/runtime/el_runtime.h +++ b/el-compiler/runtime/el_runtime.h @@ -445,6 +445,49 @@ el_val_t str_format(el_val_t fmt, el_val_t data); el_val_t str_lower(el_val_t s); el_val_t str_upper(el_val_t s); +/* ── Text-processing primitives (Phase 1: byte/codepoint, ASCII char classes) + * Phase 2 (filed): Unicode-grapheme awareness, NFC/NFD normalization, regex. + * is_* predicates: empty input returns false; multi-char requires ALL bytes + * to match. ASCII ranges only in Phase 1. */ + +/* Counting */ +el_val_t str_count(el_val_t s, el_val_t sub); /* non-overlapping */ +el_val_t str_count_chars(el_val_t s); /* codepoint count */ +el_val_t str_count_bytes(el_val_t s); /* alias of str_len */ +el_val_t str_count_lines(el_val_t s); +el_val_t str_count_words(el_val_t s); +el_val_t str_count_letters(el_val_t s); /* ASCII [A-Za-z] */ +el_val_t str_count_digits(el_val_t s); /* ASCII [0-9] */ + +/* Find / position */ +el_val_t str_index_of_all(el_val_t s, el_val_t sub); /* [Int] of byte offsets */ +el_val_t str_last_index_of(el_val_t s, el_val_t sub); +el_val_t str_find_chars(el_val_t s, el_val_t any_of); /* first idx of any ch */ + +/* Transform */ +el_val_t str_repeat(el_val_t s, el_val_t n); +el_val_t str_reverse(el_val_t s); /* by codepoint */ +el_val_t str_strip_prefix(el_val_t s, el_val_t prefix); +el_val_t str_strip_suffix(el_val_t s, el_val_t suffix); +el_val_t str_strip_chars(el_val_t s, el_val_t chars); +el_val_t str_lstrip(el_val_t s); +el_val_t str_rstrip(el_val_t s); + +/* Char classification (Bool) */ +el_val_t is_letter(el_val_t s); +el_val_t is_digit(el_val_t s); +el_val_t is_alphanumeric(el_val_t s); +el_val_t is_whitespace(el_val_t s); +el_val_t is_punctuation(el_val_t s); +el_val_t is_uppercase(el_val_t s); +el_val_t is_lowercase(el_val_t s); + +/* Split / join */ +el_val_t str_split_lines(el_val_t s); +el_val_t str_split_chars(el_val_t s); /* alias of native_string_chars */ +el_val_t str_split_n(el_val_t s, el_val_t sep, el_val_t n); +el_val_t str_join(el_val_t list, el_val_t sep); /* alias of list_join */ + /* ── List additions ──────────────────────────────────────────────────────── */ el_val_t list_push(el_val_t list, el_val_t elem); diff --git a/el-compiler/src/codegen.el b/el-compiler/src/codegen.el index a40d04d..d5530a1 100644 --- a/el-compiler/src/codegen.el +++ b/el-compiler/src/codegen.el @@ -1257,6 +1257,15 @@ fn is_int_call(call_expr: Map) -> Bool { if str_eq(name, "str_index_of") { return true } if str_eq(name, "str_to_int") { return true } if str_eq(name, "str_char_code") { return true } + if str_eq(name, "str_count") { return true } + if str_eq(name, "str_count_chars") { return true } + if str_eq(name, "str_count_bytes") { return true } + if str_eq(name, "str_count_lines") { return true } + if str_eq(name, "str_count_words") { return true } + if str_eq(name, "str_count_letters") { return true } + if str_eq(name, "str_count_digits") { return true } + if str_eq(name, "str_last_index_of") { return true } + if str_eq(name, "str_find_chars") { return true } if str_eq(name, "native_list_len") { return true } if str_eq(name, "el_list_len") { return true } if str_eq(name, "len") { return true } @@ -1858,6 +1867,35 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "str_format") { return 2 } if str_eq(name, "str_lower") { return 1 } if str_eq(name, "str_upper") { return 1 } + // Text-processing primitives (Phase 1) + if str_eq(name, "str_count") { return 2 } + if str_eq(name, "str_count_chars") { return 1 } + if str_eq(name, "str_count_bytes") { return 1 } + if str_eq(name, "str_count_lines") { return 1 } + if str_eq(name, "str_count_words") { return 1 } + if str_eq(name, "str_count_letters") { return 1 } + if str_eq(name, "str_count_digits") { return 1 } + if str_eq(name, "str_index_of_all") { return 2 } + if str_eq(name, "str_last_index_of") { return 2 } + if str_eq(name, "str_find_chars") { return 2 } + if str_eq(name, "str_repeat") { return 2 } + if str_eq(name, "str_reverse") { return 1 } + if str_eq(name, "str_strip_prefix") { return 2 } + if str_eq(name, "str_strip_suffix") { return 2 } + if str_eq(name, "str_strip_chars") { return 2 } + if str_eq(name, "str_lstrip") { return 1 } + if str_eq(name, "str_rstrip") { return 1 } + if str_eq(name, "is_letter") { return 1 } + if str_eq(name, "is_digit") { return 1 } + if str_eq(name, "is_alphanumeric") { return 1 } + if str_eq(name, "is_whitespace") { return 1 } + if str_eq(name, "is_punctuation") { return 1 } + if str_eq(name, "is_uppercase") { return 1 } + if str_eq(name, "is_lowercase") { return 1 } + if str_eq(name, "str_split_lines") { return 1 } + if str_eq(name, "str_split_chars") { return 1 } + if str_eq(name, "str_split_n") { return 3 } + if str_eq(name, "str_join") { return 2 } // HTML sanitizer if str_eq(name, "el_html_sanitize") { return 2 } // Math diff --git a/el-compiler/src/lexer.el b/el-compiler/src/lexer.el index d6c6d0c..6a430a0 100644 --- a/el-compiler/src/lexer.el +++ b/el-compiler/src/lexer.el @@ -12,7 +12,7 @@ // ── Character helpers ───────────────────────────────────────────────────────── -fn is_digit(ch: String) -> Bool { +fn lex_is_digit(ch: String) -> Bool { if ch == "0" { return true } if ch == "1" { return true } if ch == "2" { return true } @@ -26,7 +26,7 @@ fn is_digit(ch: String) -> Bool { false } -fn is_alpha(ch: String) -> Bool { +fn lex_is_alpha(ch: String) -> Bool { if ch == "a" { return true } if ch == "b" { return true } if ch == "c" { return true } @@ -83,13 +83,13 @@ fn is_alpha(ch: String) -> Bool { } fn is_alnum_or_underscore(ch: String) -> Bool { - if is_digit(ch) { return true } - if is_alpha(ch) { return true } + if lex_is_digit(ch) { return true } + if lex_is_alpha(ch) { return true } if ch == "_" { return true } false } -fn is_whitespace(ch: String) -> Bool { +fn lex_is_whitespace(ch: String) -> Bool { if ch == " " { return true } if ch == "\t" { return true } if ch == "\n" { return true } @@ -163,7 +163,7 @@ fn scan_digits(chars: [String], start: Int, total: Int) -> Map { let running = false } else { let ch: String = native_list_get(chars, i) - if is_digit(ch) { + if lex_is_digit(ch) { let text = text + ch let i = i + 1 } else { @@ -469,7 +469,7 @@ fn lex(source: String) -> [Map] { let ch: String = native_list_get(chars, i) // Skip whitespace - if is_whitespace(ch) { + if lex_is_whitespace(ch) { let i = i + 1 } else { // Line comments: // @@ -519,7 +519,7 @@ fn lex(source: String) -> [Map] { let i = new_pos } else { // Number literal - if is_digit(ch) { + if lex_is_digit(ch) { let result = scan_digits(chars, i, total) let num_text: String = result["text"] let new_pos: Int = result["pos"] @@ -530,7 +530,7 @@ fn lex(source: String) -> [Map] { let after_dot = new_pos + 1 if after_dot < total { let after_dot_ch: String = native_list_get(chars, after_dot) - if is_digit(after_dot_ch) { + if lex_is_digit(after_dot_ch) { let frac_result = scan_digits(chars, after_dot, total) let frac_text: String = frac_result["text"] let frac_pos: Int = frac_result["pos"] @@ -554,7 +554,7 @@ fn lex(source: String) -> [Map] { } } else { // Identifier or keyword - if is_alpha(ch) || ch == "_" { + if lex_is_alpha(ch) || ch == "_" { let result = scan_ident(chars, i, total) let word: String = result["text"] let new_pos: Int = result["pos"] diff --git a/elc-combined.el b/elc-combined.el index 90d400f..5eb8c09 100644 --- a/elc-combined.el +++ b/elc-combined.el @@ -12,7 +12,7 @@ // ── Character helpers ───────────────────────────────────────────────────────── -fn is_digit(ch: String) -> Bool { +fn lex_is_digit(ch: String) -> Bool { if ch == "0" { return true } if ch == "1" { return true } if ch == "2" { return true } @@ -26,7 +26,7 @@ fn is_digit(ch: String) -> Bool { false } -fn is_alpha(ch: String) -> Bool { +fn lex_is_alpha(ch: String) -> Bool { if ch == "a" { return true } if ch == "b" { return true } if ch == "c" { return true } @@ -83,13 +83,13 @@ fn is_alpha(ch: String) -> Bool { } fn is_alnum_or_underscore(ch: String) -> Bool { - if is_digit(ch) { return true } - if is_alpha(ch) { return true } + if lex_is_digit(ch) { return true } + if lex_is_alpha(ch) { return true } if ch == "_" { return true } false } -fn is_whitespace(ch: String) -> Bool { +fn lex_is_whitespace(ch: String) -> Bool { if ch == " " { return true } if ch == "\t" { return true } if ch == "\n" { return true } @@ -163,7 +163,7 @@ fn scan_digits(chars: [String], start: Int, total: Int) -> Map { let running = false } else { let ch: String = native_list_get(chars, i) - if is_digit(ch) { + if lex_is_digit(ch) { let text = text + ch let i = i + 1 } else { @@ -267,7 +267,7 @@ fn lex(source: String) -> [Map] { let ch: String = native_list_get(chars, i) // Skip whitespace - if is_whitespace(ch) { + if lex_is_whitespace(ch) { let i = i + 1 } else { // Line comments: // @@ -309,7 +309,7 @@ fn lex(source: String) -> [Map] { let i = new_pos } else { // Number literal - if is_digit(ch) { + if lex_is_digit(ch) { let result = scan_digits(chars, i, total) let num_text: String = result["text"] let new_pos: Int = result["pos"] @@ -320,7 +320,7 @@ fn lex(source: String) -> [Map] { let after_dot = new_pos + 1 if after_dot < total { let after_dot_ch: String = native_list_get(chars, after_dot) - if is_digit(after_dot_ch) { + if lex_is_digit(after_dot_ch) { let frac_result = scan_digits(chars, after_dot, total) let frac_text: String = frac_result["text"] let frac_pos: Int = frac_result["pos"] @@ -344,7 +344,7 @@ fn lex(source: String) -> [Map] { } } else { // Identifier or keyword - if is_alpha(ch) || ch == "_" { + if lex_is_alpha(ch) || ch == "_" { let result = scan_ident(chars, i, total) let word: String = result["text"] let new_pos: Int = result["pos"] @@ -2436,6 +2436,15 @@ fn is_int_call(call_expr: Map) -> Bool { if str_eq(name, "str_index_of") { return true } if str_eq(name, "str_to_int") { return true } if str_eq(name, "str_char_code") { return true } + if str_eq(name, "str_count") { return true } + if str_eq(name, "str_count_chars") { return true } + if str_eq(name, "str_count_bytes") { return true } + if str_eq(name, "str_count_lines") { return true } + if str_eq(name, "str_count_words") { return true } + if str_eq(name, "str_count_letters") { return true } + if str_eq(name, "str_count_digits") { return true } + if str_eq(name, "str_last_index_of") { return true } + if str_eq(name, "str_find_chars") { return true } if str_eq(name, "native_list_len") { return true } if str_eq(name, "el_list_len") { return true } if str_eq(name, "len") { return true } @@ -2676,6 +2685,35 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "str_format") { return 2 } if str_eq(name, "str_lower") { return 1 } if str_eq(name, "str_upper") { return 1 } + // Text-processing primitives (Phase 1) + if str_eq(name, "str_count") { return 2 } + if str_eq(name, "str_count_chars") { return 1 } + if str_eq(name, "str_count_bytes") { return 1 } + if str_eq(name, "str_count_lines") { return 1 } + if str_eq(name, "str_count_words") { return 1 } + if str_eq(name, "str_count_letters") { return 1 } + if str_eq(name, "str_count_digits") { return 1 } + if str_eq(name, "str_index_of_all") { return 2 } + if str_eq(name, "str_last_index_of") { return 2 } + if str_eq(name, "str_find_chars") { return 2 } + if str_eq(name, "str_repeat") { return 2 } + if str_eq(name, "str_reverse") { return 1 } + if str_eq(name, "str_strip_prefix") { return 2 } + if str_eq(name, "str_strip_suffix") { return 2 } + if str_eq(name, "str_strip_chars") { return 2 } + if str_eq(name, "str_lstrip") { return 1 } + if str_eq(name, "str_rstrip") { return 1 } + if str_eq(name, "is_letter") { return 1 } + if str_eq(name, "is_digit") { return 1 } + if str_eq(name, "is_alphanumeric") { return 1 } + if str_eq(name, "is_whitespace") { return 1 } + if str_eq(name, "is_punctuation") { return 1 } + if str_eq(name, "is_uppercase") { return 1 } + if str_eq(name, "is_lowercase") { return 1 } + if str_eq(name, "str_split_lines") { return 1 } + if str_eq(name, "str_split_chars") { return 1 } + if str_eq(name, "str_split_n") { return 3 } + if str_eq(name, "str_join") { return 2 } // Math if str_eq(name, "el_abs") { return 1 } if str_eq(name, "el_max") { return 2 } @@ -3426,6 +3464,15 @@ fn js_is_int_call(call_expr: Map) -> Bool { if str_eq(name, "str_index_of") { return true } if str_eq(name, "str_to_int") { return true } if str_eq(name, "str_char_code") { return true } + if str_eq(name, "str_count") { return true } + if str_eq(name, "str_count_chars") { return true } + if str_eq(name, "str_count_bytes") { return true } + if str_eq(name, "str_count_lines") { return true } + if str_eq(name, "str_count_words") { return true } + if str_eq(name, "str_count_letters") { return true } + if str_eq(name, "str_count_digits") { return true } + if str_eq(name, "str_last_index_of") { return true } + if str_eq(name, "str_find_chars") { return true } if str_eq(name, "native_list_len") { return true } if str_eq(name, "el_list_len") { return true } if str_eq(name, "len") { return true } diff --git a/tests/text/examples/char-classes.el b/tests/text/examples/char-classes.el new file mode 100644 index 0000000..94e174e --- /dev/null +++ b/tests/text/examples/char-classes.el @@ -0,0 +1,19 @@ +// char-classes.el — ASCII character classification. +// is_letter("A") && is_digit("7") && is_whitespace(" ") +// && !is_letter("3") && !is_digit("X") +fn run_test() -> Bool { + if !is_letter("A") { return false } + if !is_digit("7") { return false } + if !is_whitespace(" ") { return false } + if is_letter("3") { return false } + if is_digit("X") { return false } + return true +} + +fn main() -> Void { + if run_test() { + println("true") + } else { + println("false") + } +} diff --git a/tests/text/examples/count-lines-words-letters.el b/tests/text/examples/count-lines-words-letters.el new file mode 100644 index 0000000..7f764c6 --- /dev/null +++ b/tests/text/examples/count-lines-words-letters.el @@ -0,0 +1,16 @@ +// count-lines-words-letters.el — composite count test. +// Input "Hello world\nGoodbye world\n": +// lines: 2 (each \n closes a line) +// words: 4 (Hello, world, Goodbye, world) +// letters: 22 (Hello=5 + world=5 + Goodbye=7 + world=5) +fn run_test() -> String { + let s: String = "Hello world\nGoodbye world\n" + let lines: Int = str_count_lines(s) + let words: Int = str_count_words(s) + let letters: Int = str_count_letters(s) + return int_to_str(lines) + "/" + int_to_str(words) + "/" + int_to_str(letters) +} + +fn main() -> Void { + println(run_test()) +} diff --git a/tests/text/examples/count-overlap-skip.el b/tests/text/examples/count-overlap-skip.el new file mode 100644 index 0000000..0c45208 --- /dev/null +++ b/tests/text/examples/count-overlap-skip.el @@ -0,0 +1,11 @@ +// count-overlap-skip.el — non-overlapping advance: "aaaa" / "aa" -> 2. +// After each match, the cursor advances by len(sub), so overlapping matches +// are skipped. This is the universal default in C/Python/Go/etc. +fn run_test() -> Int { + let s: String = "aaaa" + return str_count(s, "aa") +} + +fn main() -> Void { + println(int_to_str(run_test())) +} diff --git a/tests/text/examples/count-substring.el b/tests/text/examples/count-substring.el new file mode 100644 index 0000000..d89389f --- /dev/null +++ b/tests/text/examples/count-substring.el @@ -0,0 +1,9 @@ +// count-substring.el — non-overlapping substring count. +fn run_test() -> Int { + let s: String = "abc abc abc" + return str_count(s, "abc") +} + +fn main() -> Void { + println(int_to_str(run_test())) +} diff --git a/tests/text/examples/index-of-all.el b/tests/text/examples/index-of-all.el new file mode 100644 index 0000000..9532921 --- /dev/null +++ b/tests/text/examples/index-of-all.el @@ -0,0 +1,21 @@ +// index-of-all.el — every byte offset of a substring. +// "abXcdXefX" / "X" -> [2, 5, 8]. Empty list when no matches. +fn run_test() -> String { + let positions: [Int] = str_index_of_all("abXcdXefX", "X") + let n: Int = native_list_len(positions) + let out: String = "" + let i: Int = 0 + while i < n { + let p: Int = native_list_get(positions, i) + if i > 0 { + let out = out + "," + } + let out = out + int_to_str(p) + let i = i + 1 + } + return out +} + +fn main() -> Void { + println(run_test()) +} diff --git a/tests/text/examples/join.el b/tests/text/examples/join.el new file mode 100644 index 0000000..e32b139 --- /dev/null +++ b/tests/text/examples/join.el @@ -0,0 +1,12 @@ +// join.el — list-of-string join with separator. +fn run_test() -> String { + let parts: [String] = native_list_empty() + let parts = native_list_append(parts, "alpha") + let parts = native_list_append(parts, "beta") + let parts = native_list_append(parts, "gamma") + return str_join(parts, ", ") +} + +fn main() -> Void { + println(run_test()) +} diff --git a/tests/text/examples/split-lines.el b/tests/text/examples/split-lines.el new file mode 100644 index 0000000..78354d0 --- /dev/null +++ b/tests/text/examples/split-lines.el @@ -0,0 +1,10 @@ +// split-lines.el — \n-delimited split, \r\n folded, trailing empty dropped. +// "alpha\nbeta\r\ngamma\n" -> ["alpha", "beta", "gamma"], len = 3. +fn run_test() -> Int { + let lines: [String] = str_split_lines("alpha\nbeta\r\ngamma\n") + return native_list_len(lines) +} + +fn main() -> Void { + println(int_to_str(run_test())) +} diff --git a/tests/text/examples/transform-suite.el b/tests/text/examples/transform-suite.el new file mode 100644 index 0000000..2c117f3 --- /dev/null +++ b/tests/text/examples/transform-suite.el @@ -0,0 +1,13 @@ +// transform-suite.el — repeat, reverse, strip prefix/suffix/chars. +fn run_test() -> String { + let a: String = str_repeat("ab", 3) // "ababab" + let b: String = str_reverse("hello") // "olleh" + let c: String = str_strip_prefix("foobar", "foo") // "bar" + let d: String = str_strip_suffix("hello.md", ".md") // "hello" + let e: String = str_strip_chars(" \thello \n", " \t\n") // "hello" + return a + "|" + b + "|" + c + "|" + d + "|" + e +} + +fn main() -> Void { + println(run_test()) +} diff --git a/tests/text/run.sh b/tests/text/run.sh new file mode 100755 index 0000000..7a70df1 --- /dev/null +++ b/tests/text/run.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# run.sh — build and execute the text/ acceptance corpus. +# +# Each examples/.el is a self-contained El program with a fn main() +# that prints a single deterministic result line. The runner compiles each +# via the canonical native elc, links it against the shared C runtime, runs +# it, and asserts the output matches the expected value. + +set -uo pipefail +cd "$(dirname "$0")" + +EL_HOME="${EL_HOME:-$(cd ../.. && pwd)}" +ELC="${ELC:-${EL_HOME}/dist/platform/elc}" +RUNTIME_DIR="${EL_HOME}/el-compiler/runtime" + +if [ ! -x "${ELC}" ]; then + echo "elc not found at ${ELC}" >&2 + exit 1 +fi + +PASS=0 +FAIL=0 +FAILED_NAMES=() + +run_runtime_case() { + local name="$1" + local src="$2" + local expected="$3" + + local out_c + local out_bin + out_c="$(mktemp -t text_test.XXXXXX).c" + out_bin="$(mktemp -t text_test.XXXXXX)" + + if ! "${ELC}" "${src}" > "${out_c}" 2>/tmp/text_test.elc.err; then + echo "FAIL ${name} — elc emit failed:" + cat /tmp/text_test.elc.err | sed 's/^/ /' + FAIL=$((FAIL+1)) + FAILED_NAMES+=("${name}") + rm -f "${out_c}" "${out_bin}" + return + fi + + if ! cc -O2 -I "${RUNTIME_DIR}" "${out_c}" "${RUNTIME_DIR}/el_runtime.c" \ + -lcurl -lpthread -o "${out_bin}" 2>/tmp/text_test.cc.err; then + echo "FAIL ${name} — cc failed:" + cat /tmp/text_test.cc.err | sed 's/^/ /' + FAIL=$((FAIL+1)) + FAILED_NAMES+=("${name}") + rm -f "${out_c}" "${out_bin}" + return + fi + + local got + got="$("${out_bin}" 2>&1)" + + if [ "${got}" = "${expected}" ]; then + echo "PASS ${name}" + PASS=$((PASS+1)) + else + echo "FAIL ${name} expected: '${expected}', got: '${got}'" + FAIL=$((FAIL+1)) + FAILED_NAMES+=("${name}") + fi + + rm -f "${out_c}" "${out_bin}" +} + +echo "==> Running text-primitives acceptance corpus" +echo + +run_runtime_case "count-substring" examples/count-substring.el "3" +run_runtime_case "count-overlap-skip" examples/count-overlap-skip.el "2" +run_runtime_case "count-lines-words-letters" examples/count-lines-words-letters.el "2/4/22" +run_runtime_case "index-of-all" examples/index-of-all.el "2,5,8" +run_runtime_case "transform-suite" examples/transform-suite.el "ababab|olleh|bar|hello|hello" +run_runtime_case "char-classes" examples/char-classes.el "true" +run_runtime_case "split-lines" examples/split-lines.el "3" +run_runtime_case "join" examples/join.el "alpha, beta, gamma" + +echo +echo "${PASS} passed, ${FAIL} failed" +if [ "${FAIL}" -gt 0 ]; then + echo "failed: ${FAILED_NAMES[*]}" + exit 1 +fi +exit 0 diff --git a/tests/text/runner.el b/tests/text/runner.el new file mode 100644 index 0000000..4ca316a --- /dev/null +++ b/tests/text/runner.el @@ -0,0 +1,11 @@ +// runner.el — entry point for the text/ acceptance corpus. +// +// Each text/examples/*.el is its own El program with its own fn main(). +// Compile, link, and run-output-diff is handled by tests/text/run.sh — +// this file is the El-side stub kept for pattern parity with tests/time/ +// and tests/calendar/. + +fn main() -> Void { + println("text/ acceptance corpus is driven by run.sh — invoke that directly.") + println("Each examples/*.el is a self-contained program; runner.el is a parity stub.") +}