Merge pull request 'test framework phase 1: compile-time registry + El-side runner with per-test timing' (#133) from wt/soul-runtime-reconcile into dev

lang/el-compiler/src/codegen.el

@@ -1705,9 +1705,13 @@ fn cg_stmt(stmt: Map<String, Any>, indent: String, declared: [String]) -> [Strin
         } else {
             let c_msg = "EL_STR_PTR(" + cg_expr(msg_node) + ")"
         }
+        // Assertions record into PER-TEST state, not global counters. The test
+        // is the unit of result; a global pass/fail tally cannot say which test
+        // failed or whether a test ran at all. Reporting is the runner's job —
+        // nothing is printed here.
         emit_line(indent + "if (!(" + c_cond + ")) {")
-        emit_line(indent + "  __el_test_fail(__el_cur_test, " + c_msg + "); __el_fail++;")
-        emit_line(indent + "} else { __el_pass++; }")
+        emit_line(indent + "  __el_test_fail(" + c_msg + ");")
+        emit_line(indent + "} else { __el_cur_asserts++; }")
         return declared
     }
 
@@ -2602,6 +2606,17 @@ fn builtin_arity(name: String) -> Int {
     // LSP seed primitives
     if str_eq(name, "__read_n") { return 1 }
     if str_eq(name, "__print_raw") { return 1 }
+    // Test-registry accessors. These are not runtime builtins — they are
+    // GENERATED into the same translation unit by the --test path below, one
+    // set per test binary. They are declared here so the El-side runner in
+    // runtime/eltest.el can call them with a known arity.
+    if str_eq(name, "__el_reg_count") { return 0 }
+    if str_eq(name, "__el_reg_name") { return 1 }
+    if str_eq(name, "__el_reg_invoke") { return 1 }
+    if str_eq(name, "__el_reg_last_ns") { return 0 }
+    if str_eq(name, "__el_reg_msg") { return 0 }
+    if str_eq(name, "__el_reg_asserts") { return 0 }
+    if str_eq(name, "__el_opt_json") { return 0 }
     // String
     if str_eq(name, "el_str_concat") { return 2 }
     if str_eq(name, "str_eq") { return 2 }
@@ -4116,13 +4131,36 @@ fn codegen_streaming(tokens: [Any], sigs: [Map<String, Any>], source: String) ->
     // Emit test harness preamble (counters, fail printer) when in test mode.
     if test_is_mode {
         emit_line("#include <stdio.h>")
+        emit_line("#include <string.h>")
+        emit_line("#include <time.h>")
         emit_blank()
-        emit_line("static int __el_pass = 0, __el_fail = 0;")
+        // Per-test result state. Reset by __el_reg_invoke before each test, so
+        // every test gets its own record rather than contributing to a global
+        // tally. The first failure message is retained; later ones only bump
+        // the count, which keeps the common case allocation-free.
+        emit_line("static int __el_cur_fails = 0;")
+        emit_line("static int __el_cur_asserts = 0;")
+        emit_line("static char __el_cur_msg[512] = \"\";")
         emit_line("static const char *__el_cur_test = \"(none)\";")
-        emit_line("static void __el_test_fail(const char *test, const char *msg) {")
-        emit_line("  fprintf(stderr, \"FAIL  %-40s %s\\n\", test, msg);")
+        emit_line("static void __el_test_fail(const char *msg) {")
+        emit_line("  if (__el_cur_fails == 0 && msg) {")
+        emit_line("    snprintf(__el_cur_msg, sizeof __el_cur_msg, \"%s\", msg);")
+        emit_line("  }")
+        emit_line("  __el_cur_fails++; __el_cur_asserts++;")
         emit_line("}")
         emit_blank()
+        // Forward declarations for the registry accessors. The definitions are
+        // emitted at the END of the unit (they reference the test functions,
+        // which do not exist yet at this point), but the El-side runner is
+        // compiled in between and calls them — so it needs the prototypes here.
+        emit_line("el_val_t __el_reg_count(void);")
+        emit_line("el_val_t __el_reg_name(el_val_t i);")
+        emit_line("el_val_t __el_reg_invoke(el_val_t i);")
+        emit_line("el_val_t __el_reg_last_ns(void);")
+        emit_line("el_val_t __el_reg_msg(void);")
+        emit_line("el_val_t __el_reg_asserts(void);")
+        emit_line("el_val_t __el_opt_json(void);")
+        emit_blank()
     }
 
     // Streaming parse-emit loop.
@@ -4318,17 +4356,72 @@ fn codegen_streaming(tokens: [Any], sigs: [Map<String, Any>], source: String) ->
         el_release(sigs)
 
         let test_arena_mark: Any = el_arena_push()
+        let tn: Int = native_list_len(test_c_names)
+
+        // ── Generated test registry ──────────────────────────────────────────
+        // Discovery happens HERE, at compile time. The runner never searches
+        // for tests; it walks this table. That ordering — discovery strictly
+        // before execution — is what makes --list, filtering, sharding and
+        // per-test reporting possible later, and it is why the old harness
+        // (which inlined direct calls into main) could not have any of them.
+        emit_line("typedef void (*__el_test_fp)(void);")
+        emit_line("typedef struct { const char *name; __el_test_fp fn; } __el_test_entry;")
+        emit_line("static const __el_test_entry __el_registry[] = {")
+        let ri: Int = 0
+        while ri < tn {
+            let r_name: String = native_list_get(test_names, ri)
+            let r_cfn: String = native_list_get(test_c_names, ri)
+            emit_line("  { \"" + c_escape(r_name) + "\", " + r_cfn + " },")
+            let ri = ri + 1
+        }
+        // Trailing sentinel keeps the array non-empty when a file declares no
+        // tests (a zero-length array is not valid C).
+        emit_line("  { 0, 0 }")
+        emit_line("};")
+        emit_line("static const int __el_registry_n = " + int_to_str(tn) + ";")
+        emit_blank()
+        emit_line("static long long __el_last_ns = 0;")
+        emit_line("static int __el_opt_json_v = 0;")
+        emit_blank()
+
+        // ── Index-based accessors ────────────────────────────────────────────
+        // El has no function pointers, so the runner works purely in indices.
+        // This is the whole seam between generated C and the El-side runner.
+        emit_line("el_val_t __el_reg_count(void) { return (el_val_t)(int64_t)__el_registry_n; }")
+        emit_line("el_val_t __el_reg_name(el_val_t i) {")
+        emit_line("  int64_t k = (int64_t)i;")
+        emit_line("  if (k < 0 || k >= __el_registry_n) return EL_STR(\"\");")
+        emit_line("  return EL_STR(__el_registry[k].name);")
+        emit_line("}")
+        // Timing is taken immediately around the call, in C, on the MONOTONIC
+        // clock — never the wall clock, which can step backwards under NTP.
+        emit_line("el_val_t __el_reg_invoke(el_val_t i) {")
+        emit_line("  int64_t k = (int64_t)i;")
+        emit_line("  if (k < 0 || k >= __el_registry_n) return 0;")
+        emit_line("  __el_cur_fails = 0; __el_cur_asserts = 0; __el_cur_msg[0] = '\\0';")
+        emit_line("  __el_cur_test = __el_registry[k].name;")
+        emit_line("  struct timespec _t0, _t1;")
+        emit_line("  clock_gettime(CLOCK_MONOTONIC, &_t0);")
+        emit_line("  __el_registry[k].fn();")
+        emit_line("  clock_gettime(CLOCK_MONOTONIC, &_t1);")
+        emit_line("  __el_last_ns = (long long)(_t1.tv_sec - _t0.tv_sec) * 1000000000LL")
+        emit_line("              + (long long)(_t1.tv_nsec - _t0.tv_nsec);")
+        emit_line("  return (el_val_t)(int64_t)__el_cur_fails;")
+        emit_line("}")
+        emit_line("el_val_t __el_reg_last_ns(void) { return (el_val_t)(int64_t)__el_last_ns; }")
+        emit_line("el_val_t __el_reg_msg(void) { return EL_STR(__el_cur_msg); }")
+        emit_line("el_val_t __el_reg_asserts(void) { return (el_val_t)(int64_t)__el_cur_asserts; }")
+        emit_line("el_val_t __el_opt_json(void) { return (el_val_t)(int64_t)__el_opt_json_v; }")
+        emit_blank()
+
+        // main() delegates to the El-side runner. Everything above this line is
+        // generated glue; all reporting logic lives in runtime/eltest.el.
         emit_line("int main(int _argc, char **_argv) {")
         emit_line("  el_runtime_init_args(_argc, _argv);")
-        let ti: Int = 0
-        let tn: Int = native_list_len(test_c_names)
-        while ti < tn {
-            let tc_name: String = native_list_get(test_c_names, ti)
-            emit_line("  " + tc_name + "();")
-            let ti = ti + 1
-        }
-        emit_line("  printf(\"%d passed, %d failed\\n\", __el_pass, __el_fail);")
-        emit_line("  return __el_fail;")
+        emit_line("  for (int _i = 1; _i < _argc; _i++) {")
+        emit_line("    if (strcmp(_argv[_i], \"--json\") == 0) __el_opt_json_v = 1;")
+        emit_line("  }")
+        emit_line("  return (int)(int64_t)el_test_main();")
         emit_line("}")
         el_arena_pop(test_arena_mark)
         el_release(test_names)

lang/runtime/eltest.el

@@ -0,0 +1,194 @@
+// runtime/eltest.el — El test framework runner (Phase 1).
+//
+// This is the RUNNER. It is written in El and consumes a registry that the
+// compiler generates into the same translation unit when invoked as
+// `elc --test`. Nothing here discovers tests; discovery already happened at
+// compile time, which is what makes `--list` and filtering possible later.
+//
+// ── Architecture ─────────────────────────────────────────────────────────────
+//
+// The compiler lowers each `test "name" { ... }` block into a static C
+// function and emits a static table of (name, fn) pairs plus a small set of
+// index-based accessors. El has no function pointers, so the runner never
+// sees one — it works entirely in indices:
+//
+//   __el_reg_count()      -> Int     number of registered tests
+//   __el_reg_name(i)      -> String  test name at index i
+//   __el_reg_invoke(i)    -> Int     run test i, return its failure count
+//   __el_reg_last_ns()    -> Int     wall-clock ns of the last invoke
+//   __el_reg_msg()        -> String  first failure message of the last invoke
+//   __el_reg_asserts()    -> Int     assertions executed in the last invoke
+//   __el_opt_json()       -> Int     1 if --json was passed
+//
+// Timing is taken in the generated C, immediately around the call, so no El
+// call overhead lands inside the measurement.
+//
+// ── Output ───────────────────────────────────────────────────────────────────
+//
+// Structured events are the source of truth. The human renderer is written
+// FROM the same fields the NDJSON renderer emits — never the reverse. Parsing
+// human output back into structure is the one clear architectural mistake in
+// Go's test tooling and we do not repeat it.
+//
+// Every result carries a duration. Always. A framework that cannot report how
+// long its tests took cannot surface a performance regression, and a
+// regression nobody can see is one nobody fixes.
+
+// ── Small helpers (no imports — this file must stay self-contained) ──────────
+
+// _elt_json_escape — minimal JSON string escaping for the NDJSON renderer.
+fn _elt_json_escape(s: String) -> String {
+    let out: String = ""
+    let n: Int = str_len(s)
+    let i: Int = 0
+    while i < n {
+        let ch: String = str_slice(s, i, i + 1)
+        if str_eq(ch, "\"") {
+            let out = out + "\\\""
+        } else {
+            if str_eq(ch, "\\") {
+                let out = out + "\\\\"
+            } else {
+                if str_eq(ch, "\n") {
+                    let out = out + "\\n"
+                } else {
+                    if str_eq(ch, "\t") {
+                        let out = out + "\\t"
+                    } else {
+                        if str_eq(ch, "\r") {
+                            let out = out + "\\r"
+                        } else {
+                            let out = out + ch
+                        }
+                    }
+                }
+            }
+        }
+        let i = i + 1
+    }
+    return out
+}
+
+// _elt_pad3 — left-pad an integer to three digits (for the ms.fraction form).
+fn _elt_pad3(v: Int) -> String {
+    if v < 10 { return "00" + int_to_str(v) }
+    if v < 100 { return "0" + int_to_str(v) }
+    return int_to_str(v)
+}
+
+// _elt_ms — render a nanosecond duration as "M.mmm" milliseconds.
+//
+// Deliberately avoids the modulo operator: the remainder is derived by
+// subtraction so this stays portable across El backends.
+fn _elt_ms(ns: Int) -> String {
+    let total_us: Int = ns / 1000
+    let ms_whole: Int = total_us / 1000
+    let us_rem: Int = total_us - (ms_whole * 1000)
+    return int_to_str(ms_whole) + "." + _elt_pad3(us_rem)
+}
+
+// _elt_secs — render a nanosecond duration as fractional seconds, for the
+// NDJSON `elapsed` field. JUnit XML and test2json both use seconds-as-decimal.
+fn _elt_secs(ns: Int) -> String {
+    let total_ms: Int = ns / 1000000
+    let s_whole: Int = total_ms / 1000
+    let ms_rem: Int = total_ms - (s_whole * 1000)
+    return int_to_str(s_whole) + "." + _elt_pad3(ms_rem)
+}
+
+// ── Event emission ───────────────────────────────────────────────────────────
+//
+// One function per event shape. Both renderers read the same fields; the
+// human renderer is a projection of the event, not a separate code path.
+
+fn _elt_emit_run(json_mode: Bool, name: String) {
+    if json_mode {
+        println("{\"action\":\"run\",\"test\":\"" + _elt_json_escape(name) + "\"}")
+    }
+}
+
+fn _elt_emit_result(json_mode: Bool, name: String, fails: Int, ns: Int, asserts: Int, msg: String) {
+    if json_mode {
+        let action: String = "pass"
+        if fails > 0 { let action = "fail" }
+        let line: String = "{\"action\":\"" + action + "\""
+        let line = line + ",\"test\":\"" + _elt_json_escape(name) + "\""
+        let line = line + ",\"elapsed\":" + _elt_secs(ns)
+        let line = line + ",\"assertions\":" + int_to_str(asserts)
+        if fails > 0 {
+            let line = line + ",\"failures\":" + int_to_str(fails)
+            let line = line + ",\"message\":\"" + _elt_json_escape(msg) + "\""
+        }
+        let line = line + "}"
+        println(line)
+        return
+    }
+    // Human renderer — duration is never optional.
+    if fails > 0 {
+        println("FAIL  " + name + "  (" + _elt_ms(ns) + "ms)")
+        println("      " + msg)
+        return
+    }
+    println("ok    " + name + "  (" + _elt_ms(ns) + "ms)")
+    return
+}
+
+fn _elt_emit_summary(json_mode: Bool, total: Int, failed: Int, ns: Int, asserts: Int) {
+    let passed: Int = total - failed
+    if json_mode {
+        let line: String = "{\"action\":\"summary\""
+        let line = line + ",\"tests\":" + int_to_str(total)
+        let line = line + ",\"passed\":" + int_to_str(passed)
+        let line = line + ",\"failed\":" + int_to_str(failed)
+        let line = line + ",\"assertions\":" + int_to_str(asserts)
+        let line = line + ",\"elapsed\":" + _elt_secs(ns)
+        let line = line + "}"
+        println(line)
+        return
+    }
+    println("")
+    println(int_to_str(total) + " tests, " + int_to_str(passed) + " passed, "
+        + int_to_str(failed) + " failed, " + int_to_str(asserts) + " assertions in "
+        + _elt_ms(ns) + "ms")
+    return
+}
+
+// ── The runner ───────────────────────────────────────────────────────────────
+
+// el_test_main — drive the compile-time registry.
+//
+// Called from the generated main(). Returns the number of FAILING TESTS, which
+// becomes the process exit code. Note that this counts tests, not assertions:
+// a test is the unit of result. The old harness counted assertions globally and
+// therefore could not say which test failed, how long any of them took, or
+// whether a test had run at all.
+fn el_test_main() -> Int {
+    let json_mode: Bool = false
+    if __el_opt_json() == 1 { let json_mode = true }
+
+    let n: Int = __el_reg_count()
+    let i: Int = 0
+    let failed: Int = 0
+    let total_ns: Int = 0
+    let total_asserts: Int = 0
+
+    while i < n {
+        let name: String = __el_reg_name(i)
+        _elt_emit_run(json_mode, name)
+
+        let fails: Int = __el_reg_invoke(i)
+        let ns: Int = __el_reg_last_ns()
+        let asserts: Int = __el_reg_asserts()
+        let msg: String = __el_reg_msg()
+
+        let total_ns = total_ns + ns
+        let total_asserts = total_asserts + asserts
+        if fails > 0 { let failed = failed + 1 }
+
+        _elt_emit_result(json_mode, name, fails, ns, asserts, msg)
+        let i = i + 1
+    }
+
+    _elt_emit_summary(json_mode, n, failed, total_ns, total_asserts)
+    return failed
+}

lang/tests/native/test_compiler.el

@@ -1,3 +1,4 @@
+import "../../runtime/eltest.el"
 // tests/native/test_compiler.el — comprehensive tests for the El compiler pipeline.
 //
 // Tests the lexer (lexer.el), parser (parser.el), and codegen (codegen.el)

lang/tests/native/test_core.el

@@ -1,3 +1,4 @@
+import "../../runtime/eltest.el"
 // test_codegen_js.el - basic tests for JS codegen features.
 //
 // These tests verify that core El language features produce correct values

lang/tests/native/test_env.el

@@ -1,3 +1,4 @@
+import "../../runtime/eltest.el"
 // test_env.el - native test suite for runtime/env.el
 //
 // Covers: env() for reading environment variables, args() returning a list,

lang/tests/native/test_fs.el

@@ -1,3 +1,4 @@
+import "../../runtime/eltest.el"
 // test_fs.el - native test suite for runtime/fs.el
 //
 // Covers: fs_write/read round-trip, fs_exists, fs_mkdir, fs_list,

lang/tests/native/test_json.el

@@ -1,3 +1,4 @@
+import "../../runtime/eltest.el"
 // test_json.el - native test suite for runtime/json.el
 //
 // Covers: json_get (dot-path), typed extractors (int, bool, float),

lang/tests/native/test_math.el

@@ -1,3 +1,4 @@
+import "../../runtime/eltest.el"
 // test_math.el - native test suite for runtime/math.el
 //
 // Covers: integer math (abs, max, min), float math (sqrt, log, sin, cos, pi),

lang/tests/native/test_state.el

@@ -1,3 +1,4 @@
+import "../../runtime/eltest.el"
 // test_state.el - native test suite for runtime/state.el
 //
 // Covers: state_set/get/del, state_has, state_get_or, state_keys,

lang/tests/native/test_string.el

@@ -1,3 +1,4 @@
+import "../../runtime/eltest.el"
 // test_string.el - native test suite for runtime/string.el
 //
 // Covers: type conversions, core primitives, comparison and search,

lang/tests/native/test_text.el

@@ -1,3 +1,4 @@
+import "../../runtime/eltest.el"
 // test_text.el - native test suite for text primitives.
 //
 // Mirrors the acceptance corpus in tests/text/examples/ using the

lang/tests/native/test_time.el

@@ -1,3 +1,4 @@
+import "../../runtime/eltest.el"
 // test_time.el - native test suite for runtime/time.el
 //
 // Covers: time_now (positive timestamp), time_to_parts (UTC decomposition),

lang/tests/runtime/string_test.el

@@ -1,3 +1,4 @@
+import "../../runtime/eltest.el"
 // tests/runtime/string_test.el — Test suite for runtime/string.el
 //
 // Exercises every public function exported by runtime/string.el using the