thread provenance end to end: a diagnostic can finally name a place

lang/el-compiler/src/codegen.el

@@ -2802,7 +2802,7 @@ fn cg_fn(stmt: Map<String, Any>) -> Void {
     // functions into one translation unit. cc catches it, but names the
     // generated helpers first and the user's fn fourth. Recording it lets the
     // collision be reported at El level, in El terms.
-    record_call(fn_name, "defines")
+    record_call(fn_name, "defines_at:" + stmt["line"])
     // Emit which constructs this fn carries, so the prohibition check can be a
     // query over relations instead of a walk inside the emitter.
     let cdl = stmt["decorators"]
@@ -3630,7 +3630,7 @@ fn route_sort_desc(recs: [Map<String, Any>]) -> [Map<String, Any>] {
 // be synthesized in the streaming backend, which discards per-fn ASTs. Handles
 // decorator STACKING: `@route(...) @manager fn` still records the route.
 fn scan_routes(tokens: [Any]) -> [Map<String, Any>] {
-    let total: Int = native_list_len(tokens) / 2
+    let total: Int = native_list_len(tokens) / 3
     let recs: [Map<String, Any>] = native_list_empty()
     let has_pending: Bool = false
     let pending_args: [String] = native_list_empty()
@@ -3804,7 +3804,7 @@ fn declare_prohibition(construct: String, names_csv: String) -> Void {
 // to walk.
 fn scan_declared_decorators(tokens: [Any]) -> Void {
     declare_prohibition("manager", "dharma_emit,dharma_field")
-    let total: Int = native_list_len(tokens) / 2
+    let total: Int = native_list_len(tokens) / 3
     let has_pending_p: Bool = false
     let pending_prohibit: String = ""
     let pos: Int = 0
@@ -4015,7 +4015,7 @@ fn emit_streaming_preamble(sigs: [Map<String, Any>], source: String) -> Void {
 // sigs:   pre-scanned signature list from scan_fn_sigs(tokens)
 // source: original source string (for string literal lookup)
 fn codegen_streaming(tokens: [Any], sigs: [Map<String, Any>], source: String) -> String {
-    let total_tokens: Int = native_list_len(tokens) / 2
+    let total_tokens: Int = native_list_len(tokens) / 3
 
     // Emit preamble (forward decls, file-scope lets, #includes)
     // Arena scope: free intermediate strings built during preamble emission.

lang/el-compiler/src/compiler.el

@@ -414,6 +414,12 @@ fn parse_import_line(trimmed: String, dir: String) -> String {
 // Accumulates chunks into lists and joins once at the end to avoid the O(n²)
 // memory growth caused by repeated `prefix = prefix + chunk` concatenation.
 fn resolve_imports(src_path: String) -> String {
+    // Only the OUTERMOST call publishes provenance. Nested calls number their
+    // lines from 1 within themselves, so their spans are meaningless once the
+    // text is spliced into the parent.
+    let depth: String = state_get("__elc_prov_depth")
+    if str_eq(depth, "") { state_set("__elc_prov_depth", "1") }
+    let is_top: Bool = str_eq(depth, "")
     let seen_key: String = "__elc_imp__:" + src_path
     let already: String = state_get(seen_key)
     if !str_eq(already, "") { return "" }
@@ -443,6 +449,7 @@ fn resolve_imports(src_path: String) -> String {
     // Collect chunks into lists — O(1) amortized per append.
     // Join once at the end — O(n) single pass.
     let prefix_chunks: [String] = native_list_empty()
+    let prefix_paths: [String] = native_list_empty()
     let body_chunks: [String] = native_list_empty()
     let i: Int = 0
     while i < n {
@@ -454,21 +461,54 @@ fn resolve_imports(src_path: String) -> String {
             // Only check .elh for imported files — never for the entry file itself.
             let imp_elh_path: String = str_slice(imp_path, 0, str_len(imp_path) - 3) + ".elh"
             let imp_elh: String = fs_read(imp_elh_path)
+            // Provenance: record which line range of the combined source came
+            // from which file, so a diagnostic can name the FILE and not just a
+            // line in a string that no longer exists on disk.
             if !str_eq(imp_elh, "") {
                 // Header exists: mark the .el as seen (so it won't be re-inlined
                 // if something else also imports it) and use the header text.
                 let seen_imp_key: String = "__elc_imp__:" + imp_path
                 state_set(seen_imp_key, "1")
                 let prefix_chunks = native_list_append(prefix_chunks, imp_elh)
+                let prefix_paths = native_list_append(prefix_paths, imp_path)
             } else {
                 let imp_body: String = resolve_imports(imp_path)
                 let prefix_chunks = native_list_append(prefix_chunks, imp_body)
+                let prefix_paths = native_list_append(prefix_paths, imp_path)
             }
         } else {
             let body_chunks = native_list_append(body_chunks, line + "\n")
         }
         let i = i + 1
     }
+    // Walk the assembled chunks once and publish <file> spans <start> <end>.
+    // LIMIT: nested imports return a single string, so their internal
+    // boundaries are already lost by the time we see them -- a definition
+    // inside a transitively imported file is attributed to the direct import.
+    // Local, not accumulated in state: a nested call numbers its lines from 1
+    // within itself, so letting it append to a shared buffer republishes
+    // meaningless spans under the parent's name.
+    let prov: String = ""
+    let line_at: Int = 1
+    let ci: Int = 0
+    let nchunks: Int = native_list_len(prefix_chunks)
+    while ci < nchunks {
+        let chunk: String = native_list_get(prefix_chunks, ci)
+        let nlines: Int = str_count_lines(chunk)
+        let src: String = native_list_get(prefix_paths, ci)
+        let prov = prov + src + " spans " + native_int_to_str(line_at) + " " + native_int_to_str(line_at + nlines - 1) + "\n"
+        let line_at = line_at + nlines
+        let ci = ci + 1
+    }
+    let prov = prov + src_path + " spans " + native_int_to_str(line_at) + " 999999\n"
+    if is_top {
+        let prov_out: String = env("EL_RELATIONS_OUT")
+        if !str_eq(prov_out, "") {
+            let existing: String = ""
+            if fs_exists(prov_out) { let existing = fs_read(prov_out) }
+            fs_write(prov_out, existing + prov)
+        }
+    }
     return str_join(prefix_chunks, "") + str_join(body_chunks, "")
 }
 

lang/el-compiler/src/lexer.el

@@ -138,9 +138,18 @@ fn lex_is_whitespace(ch: String) -> Bool {
 
 // tok_append — append a (kind, value) pair to a flat token list.
 // Returns the updated list. Gamma combines flat-list + char-code for max savings.
+// A token is (kind, value, line). The line comes from state rather than a
+// parameter so the ~200 existing tok_append call sites are untouched -- the
+// lexer advances __lex_line as it walks, and every token minted takes the line
+// it was minted on.
+//
+// WHY AT ALL: before this a token carried no position, so no diagnostic in El
+// could name a place. Every error named a symbol and never a line, and after
+// textual inlining there was no way to say which FILE a definition came from.
 fn tok_append(tokens: [Any], kind: String, value: String) -> [Any] {
     let tokens = native_list_append(tokens, kind)
-    native_list_append(tokens, value)
+    let tokens = native_list_append(tokens, value)
+    native_list_append(tokens, state_get("__lex_line"))
 }
 
 // -- Keyword lookup ------------------------------------------------------------
@@ -532,6 +541,12 @@ fn scan_interp_brace(src: String, start: Int, total: Int) -> Map<String, Any> {
 
 // interp_tokens_append_all - copy every (kind, value) pair from flat src list
 // into flat dst list, skipping the trailing Eof pair that lex() always appends.
+// Splices re-lexed interpolation tokens into the stream. This walks the token
+// list DIRECTLY rather than through tok_append, so it carries its own copy of
+// the stride -- which is why giving tokens a line broke the compiler's second
+// generation and not its first: the compiler's own source uses string
+// interpolation, so gen1 (built by the old compiler) was fine and gen2 emitted
+// a corrupted stream.
 fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] {
     let src_len: Int = native_list_len(src)
     let j = 0
@@ -542,9 +557,11 @@ fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] {
             let j = src_len
         } else {
             let val: String = native_list_get(src, j + 1)
+            let ln: String = native_list_get(src, j + 2)
             let result = native_list_append(result, kind)
             let result = native_list_append(result, val)
-            let j = j + 2
+            let result = native_list_append(result, ln)
+            let j = j + 3
         }
     }
     result
@@ -775,8 +792,14 @@ fn lex(source: String) -> [Any] {
     let total: Int = str_len(source)
     let tokens: [Any] = native_list_empty()
     let i: Int = 0
+    state_set("__lex_line", "1")
+    let line_no: Int = 1
 
     while i < total {
+        if str_eq(str_slice(source, i, i + 1), "\n") {
+            let line_no = line_no + 1
+            state_set("__lex_line", native_int_to_str(line_no))
+        }
         let c: Int = str_char_code(source, i)
 
         // Skip whitespace (space=32, tab=9, newline=10, CR=13)

lang/el-compiler/src/parser.el

@@ -17,8 +17,8 @@
 // programs. All callers use these helpers -- only these three need updating.
 
 fn tok_at(tokens: [Any], pos: Int) -> Map<String, Any> {
-    let kind: String = native_list_get(tokens, pos * 2)
-    let value: String = native_list_get(tokens, pos * 2 + 1)
+    let kind: String = native_list_get(tokens, pos * 3)
+    let value: String = native_list_get(tokens, pos * 3 + 1)
     { "kind": kind, "value": value }
 }
 
@@ -28,25 +28,32 @@ fn tok_kind(tokens: [Any], pos: Int) -> String {
     // single trailing Eof token returns runtime null (el_list_get OOB -> 0),
     // which matches no delimiter, letting inner parse loops append AST nodes
     // forever on malformed input -> unbounded allocation -> OOM.
-    let n: Int = native_list_len(tokens) / 2
+    let n: Int = native_list_len(tokens) / 3
     if pos < 0 {
         return "Eof"
     }
     if pos >= n {
         return "Eof"
     }
-    native_list_get(tokens, pos * 2)
+    native_list_get(tokens, pos * 3)
+}
+
+fn tok_line(tokens: [Any], pos: Int) -> String {
+    let n: Int = native_list_len(tokens) / 3
+    if pos < 0 { return "0" }
+    if pos >= n { return "0" }
+    native_list_get(tokens, pos * 3 + 2)
 }
 
 fn tok_value(tokens: [Any], pos: Int) -> String {
-    let n: Int = native_list_len(tokens) / 2
+    let n: Int = native_list_len(tokens) / 3
     if pos < 0 {
         return ""
     }
     if pos >= n {
         return ""
     }
-    native_list_get(tokens, pos * 2 + 1)
+    native_list_get(tokens, pos * 3 + 1)
 }
 
 // parse_progress_fatal — robustness backstop. Called by the token-consuming
@@ -1230,7 +1237,7 @@ fn parse_block(tokens: [Any], pos: Int) -> Map<String, Any> {
     // Runaway backstop: a block can hold at most (token count) statements, since
     // every iteration consumes >= 1 token. If we exceed that, the cursor has run
     // off the end without terminating (malformed input) -> fail fast, don't hang.
-    let blk_total: Int = native_list_len(tokens) / 2
+    let blk_total: Int = native_list_len(tokens) / 3
     let blk_iters: Int = 0
     while running {
         let blk_iters = blk_iters + 1
@@ -1550,7 +1557,10 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map<String, Any> {
         let p = r2["pos"]
         // r2 result map fully consumed — release to free peak heap.
         el_release(r2)
-        return make_result({ "stmt": "FnDef", "name": name, "params": params, "body": body, "ret_type": ret_type }, p)
+        // The definition carries the line it was written on. Without it no
+        // diagnostic can name a place, and after textual inlining there is no
+        // way to say which FILE a definition came from.
+        return make_result({ "stmt": "FnDef", "name": name, "params": params, "body": body, "ret_type": ret_type, "line": tok_line(tokens, pos) }, p)
     }
 
     // type definition: `type Name = { field: Type, ... }`
@@ -1842,6 +1852,7 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map<String, Any> {
                 "params": inner["params"],
                 "body": inner["body"],
                 "ret_type": inner["ret_type"],
+                "line": inner["line"],
                 "decorator": dec_name,
                 "decorators": dlist
             }
@@ -2158,7 +2169,7 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map<String, Any> {
 
 fn parse(tokens: [Any]) -> [Map<String, Any>] {
     // Flat list: 2 entries per token, so divide by 2 for token count.
-    let total: Int = native_list_len(tokens) / 2
+    let total: Int = native_list_len(tokens) / 3
     let stmts: [Map<String, Any>] = native_list_empty()
     let pos: Int = 0
     let running = true
@@ -2201,7 +2212,7 @@ fn parse_one(tokens: [Any], pos: Int) -> Map<String, Any> {
 // On entry, pos must point at the LBrace token.
 // Returns the position of the token AFTER the matching RBrace.
 fn skip_to_rbrace(tokens: [Any], pos: Int) -> Int {
-    let total: Int = native_list_len(tokens) / 2
+    let total: Int = native_list_len(tokens) / 3
     let p: Int = pos + 1
     let depth: Int = 1
     let going: Bool = true
@@ -2253,7 +2264,7 @@ fn is_stmt_start_kind(k: String) -> Bool {
 // token that could start a new top-level statement, staying depth-aware
 // so that braces inside expressions don't fool us.
 fn skip_expr_to_stmt_boundary(tokens: [Any], pos: Int) -> Int {
-    let total: Int = native_list_len(tokens) / 2
+    let total: Int = native_list_len(tokens) / 3
     let p: Int = pos
     let depth: Int = 0
     let going: Bool = true
@@ -2419,7 +2430,7 @@ fn scan_params_el(tokens: [Any], pos: Int) -> Map<String, Any> {
 //
 // Peak memory: O(tokens) with no expression AST allocation.
 fn scan_fn_sigs_el(tokens: [Any]) -> [Map<String, Any>] {
-    let total: Int = native_list_len(tokens) / 2
+    let total: Int = native_list_len(tokens) / 3
     let sigs: [Map<String, Any>] = native_list_empty()
     let pos: Int = 0
     let going: Bool = true
@@ -2561,7 +2572,7 @@ fn scan_params_c(tokens: [Any], pos: Int) -> Map<String, Any> {
 //
 // The scan allocates only small string values per entry, keeping peak RSS low.
 fn scan_fn_sigs(tokens: [Any]) -> [Map<String, Any>] {
-    let total: Int = native_list_len(tokens) / 2
+    let total: Int = native_list_len(tokens) / 3
     let sigs: [Map<String, Any>] = native_list_empty()
     let pos: Int = 0
     let going: Bool = true

lang/tests/integration/definitions_query.sh

@@ -17,9 +17,11 @@ out=$("$LANG_DIR/tools/check/definitions.sh" "$W/r.txt" 2>&1); rc=$?
 chk "a collision across modules is caught at El level" "1" "$rc"
 chk "the colliding name is reported" "1" "$(echo "$out" | grep -c "'helper' is defined 2 times")"
 chk "and the reason is given" "1" "$(echo "$out" | grep -c 'no namespacing')"
+chk "both source FILES are named" "1" "$(echo "$out" | grep -c 'a.el:1')"
+chk "with file-local line numbers, not combined ones" "1" "$(echo "$out" | grep -c 'b.el:1')"
 
 printf 'fn only_once() -> Int { return 1 }\nfn main() { println(int_to_str(only_once())) }\n' > "$W/c.el"
 EL_RELATIONS_OUT="$W/r2.txt" "$ELC" "$W/c.el" >/dev/null 2>&1
 "$LANG_DIR/tools/check/definitions.sh" "$W/r2.txt" >/dev/null 2>&1
 chk "a clean program exits 0" "0" "$?"
-echo; echo "  4 assertions, $((4-F)) passed, $F failed"; exit $F
+echo; echo "  6 assertions, $((6-F)) passed, $F failed"; exit $F

lang/tests/native/test_compiler.el

@@ -18,7 +18,9 @@ import "../../el-compiler/src/compiler.el"
 // ── Lexer helpers ─────────────────────────────────────────────────────────────
 
 fn tok_count(tokens: [Any]) -> Int {
-    native_list_len(tokens) / 2
+    // A token is (kind, value, line). This helper carried its own copy of the
+    // stride, so it escaped a search scoped to the compiler sources.
+    native_list_len(tokens) / 3
 }
 
 // ── Codegen helper: capture compile() stdout to a string ─────────────────────
@@ -259,22 +261,28 @@ test "lex-multiline-source" {
     assert tok_kind(tokens, 0) == "Let", "first token is Let"
 }
 
-test "lex-flat-stride-2-layout" {
-    // Verify that the flat stride-2 layout: token i has kind at index 2*i, value at 2*i+1
+test "lex-flat-stride-3-layout" {
+    // A token is (kind, value, line): token i has kind at 3*i, value at 3*i+1,
+    // line at 3*i+2. Before 2026-08-17 a token carried no position at all, so
+    // no diagnostic in El could name a place.
     let tokens: [Any] = lex("fn foo")
-    // tokens[0] = "Fn", tokens[1] = "fn", tokens[2] = "Ident", tokens[3] = "foo", ...
     let raw_len: Int = native_list_len(tokens)
-    assert raw_len == 6, "fn + foo + Eof = 3 tokens = 6 raw entries"
-    let kind0: String = native_list_get(tokens, 0)
-    let val0: String = native_list_get(tokens, 1)
-    let kind1: String = native_list_get(tokens, 2)
-    let val1: String = native_list_get(tokens, 3)
-    assert kind0 == "Fn", "raw[0] is Fn kind"
-    assert val0 == "fn", "raw[1] is fn value"
-    assert kind1 == "Ident", "raw[2] is Ident kind"
-    assert val1 == "foo", "raw[3] is foo value"
+    assert raw_len == 9, "fn + foo + Eof = 3 tokens = 9 raw entries"
+    assert native_list_get(tokens, 0) == "Fn", "raw[0] is the kind"
+    assert native_list_get(tokens, 1) == "fn", "raw[1] is the value"
+    assert native_list_get(tokens, 2) == "1", "raw[2] is the line"
+    assert native_list_get(tokens, 3) == "Ident", "raw[3] is the next kind"
+    assert native_list_get(tokens, 5) == "1", "still line 1"
 }
 
+test "lexer-tracks-line-numbers" {
+    let tokens: [Any] = lex("fn a\nfn b\nfn c")
+    assert tok_line(tokens, 0) == "1", "first fn is on line 1"
+    assert tok_line(tokens, 2) == "2", "second fn is on line 2"
+    assert tok_line(tokens, 4) == "3", "third fn is on line 3"
+}
+
+
 // ── Parser tests ──────────────────────────────────────────────────────────────
 
 fn get_first_stmt_kind(src: String) -> String {

lang/tools/check/definitions.sh

@@ -1,24 +1,42 @@
 #!/usr/bin/env bash
-# definitions.sh — catch duplicate top-level definitions at El level.
+# definitions.sh — catch duplicate top-level definitions, and name the files.
 #
 # El has no namespacing. `import` is textual inlining, so two modules defining
-# the same name emit two C functions into one translation unit. cc does catch
-# it, but reports the generated helpers (__el_body_f, __env_f, __thunk_f) before
-# the user's own function, so the first three errors name symbols the user never
-# wrote.
+# the same name emit two C functions into one translation unit. cc catches it,
+# but reports the generated helpers (__el_body_f, __env_f, __thunk_f) before the
+# user's own function, so the first three errors name symbols nobody wrote.
 #
-# LIMIT, stated rather than hidden: textual inlining destroys file provenance.
-# By the time codegen runs there is one source string and no record of which
-# file a definition came from, so this can say WHICH name collides but not which
-# files. Naming the files needs provenance threaded through resolve_imports.
+# Naming the FILES needed provenance threaded end to end: tokens had no line
+# numbers at all, so no diagnostic in El could name a place. Now a token is
+# (kind, value, line), FnDef carries its line, and resolve_imports publishes
+# which line range of the combined source came from which file.
+#
+# LIMIT: a nested import returns one string, so a definition inside a
+# transitively imported file is attributed to the direct import.
 set -uo pipefail
 REL="${1:?usage: definitions.sh <relations-file>}"
 [ -f "$REL" ] || exit 0
+
+# line in the COMBINED source -> "file:line-within-that-file". Reporting the
+# combined line against a filename would point at a line that file does not
+# have, which is worse than reporting no line at all.
+locate() {
+    awk -v L="$1" '$2=="spans" && $3<=L && $4>=L {printf "%s:%d", $1, L-$3+1; found=1; exit}
+                   END{ if(!found) printf "" }' "$REL"
+}
+
 V=0
-while read -r name count; do
-    [ "$count" -gt 1 ] || continue
-    printf "duplicate definition: '%s' is defined %s times — El has no namespacing, so imported modules share one global scope\n" "$name" "$count"
+while read -r name; do
+    lines=$(grep -E "^$name calls defines_at:" "$REL" | sed 's/.*defines_at://' | sort -un)
+    n=$(echo "$lines" | wc -l | tr -d ' ')
+    [ "$n" -gt 1 ] || continue
+    printf "duplicate definition: '%s' is defined %s times — El has no namespacing, so imported modules share one global scope\n" "$name" "$n"
+    for l in $lines; do
+        loc=$(locate "$l")
+        [ -n "$loc" ] && printf "    %s\n" "$loc" || printf "    combined line %s\n" "$l"
+    done
     V=$((V+1))
-done < <(grep ' calls defines$' "$REL" | awk '{print $1}' | sort | uniq -c | awk '{print $2, $1}')
+done < <(grep ' calls defines_at:' "$REL" | awk '{print $1}' | sort -u)
+
 [ "$V" -eq 0 ] && echo "definitions: clean"
 exit "$V"