From 6c975b1d5019446c29d84d99b9b1c0ec512be8a6 Mon Sep 17 00:00:00 2001 From: bigmerge Date: Mon, 17 Aug 2026 10:07:27 -0500 Subject: [PATCH] thread provenance through resolve_imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The module question ended with a limit: textual inlining destroys file provenance, so a duplicate-definition message could name the symbol but not the files. Threading it exposed a bigger absence first. TOKENS HAD NO POSITION AT ALL. A token was a flat (kind, value) pair, so NO diagnostic in El could name a place -- every error named a symbol and never a line. That is the prerequisite the module question was resting on. THE CHAIN, end to end lexer counts newlines; tok_append mints (kind, value, line) parser stride 2 -> 3; tok_line added; FnDef carries its line codegen records defines_at: resolve_imports publishes spans for the combined source checker maps a combined line back to file:line-within-that-file duplicate definition: 'helper' is defined 2 times — El has no namespacing, so imported modules share one global scope /tmp/modtest/a.el:1 /tmp/modtest/b.el:1 PREDICTIONS AND RESULTS P1 15 stride sites, encapsulated in tok_kind/tok_value TRUE, but see below P2 adding a line field is mechanical TRUE P3 the lexer must count newlines TRUE P4 resolve_imports can record per-file line ranges TRUE P5 the message can then name both files TRUE P6 token memory grows TRUE, 25.0 -> 33.9 MB (+36%) FOUR DEFECTS, EACH FOUND BY RUNNING AND NOT BY READING 1. interp_tokens_append_all walks the token list DIRECTLY with its own copy of the stride. Gen1 built fine and gen2 emitted corrupt C, because the compiler's own source uses string interpolation. My search missed it because I grepped for the variable name `tokens`; it is called `dst`/`result`. Searching by name instead of by shape -- third time today. 2. tok_count in test_compiler.el carried the stride too. I had scoped the search to compiler sources and it had escaped into the tests. 3. Nested resolve_imports calls accumulated spans into shared state, so each republished meaningless line ranges under the parent's name. Making the buffer local fixed it; guarding the WRITE did not, which is what I tried first. 4. The first working version reported b.el:3 -- the COMBINED line against a filename that has no line 3. A file:line that does not match the file is worse than no line at all. 105/105 native, 37/37 integration, fixpoint ok, compiler self-checks clean. --- lang/el-compiler/src/codegen.el | 8 ++-- lang/el-compiler/src/compiler.el | 40 +++++++++++++++++++ lang/el-compiler/src/lexer.el | 27 ++++++++++++- lang/el-compiler/src/parser.el | 37 +++++++++++------ lang/tests/integration/definitions_query.sh | 4 +- lang/tests/native/test_compiler.el | 34 ++++++++++------ lang/tools/check/definitions.sh | 44 +++++++++++++++------ 7 files changed, 148 insertions(+), 46 deletions(-) diff --git a/lang/el-compiler/src/codegen.el b/lang/el-compiler/src/codegen.el index 288e1c2..c8e4853 100644 --- a/lang/el-compiler/src/codegen.el +++ b/lang/el-compiler/src/codegen.el @@ -2802,7 +2802,7 @@ fn cg_fn(stmt: Map) -> Void { // functions into one translation unit. cc catches it, but names the // generated helpers first and the user's fn fourth. Recording it lets the // collision be reported at El level, in El terms. - record_call(fn_name, "defines") + record_call(fn_name, "defines_at:" + stmt["line"]) // Emit which constructs this fn carries, so the prohibition check can be a // query over relations instead of a walk inside the emitter. let cdl = stmt["decorators"] @@ -3630,7 +3630,7 @@ fn route_sort_desc(recs: [Map]) -> [Map] { // be synthesized in the streaming backend, which discards per-fn ASTs. Handles // decorator STACKING: `@route(...) @manager fn` still records the route. fn scan_routes(tokens: [Any]) -> [Map] { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let recs: [Map] = native_list_empty() let has_pending: Bool = false let pending_args: [String] = native_list_empty() @@ -3804,7 +3804,7 @@ fn declare_prohibition(construct: String, names_csv: String) -> Void { // to walk. fn scan_declared_decorators(tokens: [Any]) -> Void { declare_prohibition("manager", "dharma_emit,dharma_field") - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let has_pending_p: Bool = false let pending_prohibit: String = "" let pos: Int = 0 @@ -4015,7 +4015,7 @@ fn emit_streaming_preamble(sigs: [Map], source: String) -> Void { // sigs: pre-scanned signature list from scan_fn_sigs(tokens) // source: original source string (for string literal lookup) fn codegen_streaming(tokens: [Any], sigs: [Map], source: String) -> String { - let total_tokens: Int = native_list_len(tokens) / 2 + let total_tokens: Int = native_list_len(tokens) / 3 // Emit preamble (forward decls, file-scope lets, #includes) // Arena scope: free intermediate strings built during preamble emission. diff --git a/lang/el-compiler/src/compiler.el b/lang/el-compiler/src/compiler.el index 14c3ccd..b06405c 100644 --- a/lang/el-compiler/src/compiler.el +++ b/lang/el-compiler/src/compiler.el @@ -414,6 +414,12 @@ fn parse_import_line(trimmed: String, dir: String) -> String { // Accumulates chunks into lists and joins once at the end to avoid the O(n²) // memory growth caused by repeated `prefix = prefix + chunk` concatenation. fn resolve_imports(src_path: String) -> String { + // Only the OUTERMOST call publishes provenance. Nested calls number their + // lines from 1 within themselves, so their spans are meaningless once the + // text is spliced into the parent. + let depth: String = state_get("__elc_prov_depth") + if str_eq(depth, "") { state_set("__elc_prov_depth", "1") } + let is_top: Bool = str_eq(depth, "") let seen_key: String = "__elc_imp__:" + src_path let already: String = state_get(seen_key) if !str_eq(already, "") { return "" } @@ -443,6 +449,7 @@ fn resolve_imports(src_path: String) -> String { // Collect chunks into lists — O(1) amortized per append. // Join once at the end — O(n) single pass. let prefix_chunks: [String] = native_list_empty() + let prefix_paths: [String] = native_list_empty() let body_chunks: [String] = native_list_empty() let i: Int = 0 while i < n { @@ -454,21 +461,54 @@ fn resolve_imports(src_path: String) -> String { // Only check .elh for imported files — never for the entry file itself. let imp_elh_path: String = str_slice(imp_path, 0, str_len(imp_path) - 3) + ".elh" let imp_elh: String = fs_read(imp_elh_path) + // Provenance: record which line range of the combined source came + // from which file, so a diagnostic can name the FILE and not just a + // line in a string that no longer exists on disk. if !str_eq(imp_elh, "") { // Header exists: mark the .el as seen (so it won't be re-inlined // if something else also imports it) and use the header text. let seen_imp_key: String = "__elc_imp__:" + imp_path state_set(seen_imp_key, "1") let prefix_chunks = native_list_append(prefix_chunks, imp_elh) + let prefix_paths = native_list_append(prefix_paths, imp_path) } else { let imp_body: String = resolve_imports(imp_path) let prefix_chunks = native_list_append(prefix_chunks, imp_body) + let prefix_paths = native_list_append(prefix_paths, imp_path) } } else { let body_chunks = native_list_append(body_chunks, line + "\n") } let i = i + 1 } + // Walk the assembled chunks once and publish spans . + // LIMIT: nested imports return a single string, so their internal + // boundaries are already lost by the time we see them -- a definition + // inside a transitively imported file is attributed to the direct import. + // Local, not accumulated in state: a nested call numbers its lines from 1 + // within itself, so letting it append to a shared buffer republishes + // meaningless spans under the parent's name. + let prov: String = "" + let line_at: Int = 1 + let ci: Int = 0 + let nchunks: Int = native_list_len(prefix_chunks) + while ci < nchunks { + let chunk: String = native_list_get(prefix_chunks, ci) + let nlines: Int = str_count_lines(chunk) + let src: String = native_list_get(prefix_paths, ci) + let prov = prov + src + " spans " + native_int_to_str(line_at) + " " + native_int_to_str(line_at + nlines - 1) + "\n" + let line_at = line_at + nlines + let ci = ci + 1 + } + let prov = prov + src_path + " spans " + native_int_to_str(line_at) + " 999999\n" + if is_top { + let prov_out: String = env("EL_RELATIONS_OUT") + if !str_eq(prov_out, "") { + let existing: String = "" + if fs_exists(prov_out) { let existing = fs_read(prov_out) } + fs_write(prov_out, existing + prov) + } + } return str_join(prefix_chunks, "") + str_join(body_chunks, "") } diff --git a/lang/el-compiler/src/lexer.el b/lang/el-compiler/src/lexer.el index a14f7e2..358f486 100644 --- a/lang/el-compiler/src/lexer.el +++ b/lang/el-compiler/src/lexer.el @@ -138,9 +138,18 @@ fn lex_is_whitespace(ch: String) -> Bool { // tok_append — append a (kind, value) pair to a flat token list. // Returns the updated list. Gamma combines flat-list + char-code for max savings. +// A token is (kind, value, line). The line comes from state rather than a +// parameter so the ~200 existing tok_append call sites are untouched -- the +// lexer advances __lex_line as it walks, and every token minted takes the line +// it was minted on. +// +// WHY AT ALL: before this a token carried no position, so no diagnostic in El +// could name a place. Every error named a symbol and never a line, and after +// textual inlining there was no way to say which FILE a definition came from. fn tok_append(tokens: [Any], kind: String, value: String) -> [Any] { let tokens = native_list_append(tokens, kind) - native_list_append(tokens, value) + let tokens = native_list_append(tokens, value) + native_list_append(tokens, state_get("__lex_line")) } // -- Keyword lookup ------------------------------------------------------------ @@ -532,6 +541,12 @@ fn scan_interp_brace(src: String, start: Int, total: Int) -> Map { // interp_tokens_append_all - copy every (kind, value) pair from flat src list // into flat dst list, skipping the trailing Eof pair that lex() always appends. +// Splices re-lexed interpolation tokens into the stream. This walks the token +// list DIRECTLY rather than through tok_append, so it carries its own copy of +// the stride -- which is why giving tokens a line broke the compiler's second +// generation and not its first: the compiler's own source uses string +// interpolation, so gen1 (built by the old compiler) was fine and gen2 emitted +// a corrupted stream. fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] { let src_len: Int = native_list_len(src) let j = 0 @@ -542,9 +557,11 @@ fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] { let j = src_len } else { let val: String = native_list_get(src, j + 1) + let ln: String = native_list_get(src, j + 2) let result = native_list_append(result, kind) let result = native_list_append(result, val) - let j = j + 2 + let result = native_list_append(result, ln) + let j = j + 3 } } result @@ -775,8 +792,14 @@ fn lex(source: String) -> [Any] { let total: Int = str_len(source) let tokens: [Any] = native_list_empty() let i: Int = 0 + state_set("__lex_line", "1") + let line_no: Int = 1 while i < total { + if str_eq(str_slice(source, i, i + 1), "\n") { + let line_no = line_no + 1 + state_set("__lex_line", native_int_to_str(line_no)) + } let c: Int = str_char_code(source, i) // Skip whitespace (space=32, tab=9, newline=10, CR=13) diff --git a/lang/el-compiler/src/parser.el b/lang/el-compiler/src/parser.el index 68c69d3..f72015f 100644 --- a/lang/el-compiler/src/parser.el +++ b/lang/el-compiler/src/parser.el @@ -17,8 +17,8 @@ // programs. All callers use these helpers -- only these three need updating. fn tok_at(tokens: [Any], pos: Int) -> Map { - let kind: String = native_list_get(tokens, pos * 2) - let value: String = native_list_get(tokens, pos * 2 + 1) + let kind: String = native_list_get(tokens, pos * 3) + let value: String = native_list_get(tokens, pos * 3 + 1) { "kind": kind, "value": value } } @@ -28,25 +28,32 @@ fn tok_kind(tokens: [Any], pos: Int) -> String { // single trailing Eof token returns runtime null (el_list_get OOB -> 0), // which matches no delimiter, letting inner parse loops append AST nodes // forever on malformed input -> unbounded allocation -> OOM. - let n: Int = native_list_len(tokens) / 2 + let n: Int = native_list_len(tokens) / 3 if pos < 0 { return "Eof" } if pos >= n { return "Eof" } - native_list_get(tokens, pos * 2) + native_list_get(tokens, pos * 3) +} + +fn tok_line(tokens: [Any], pos: Int) -> String { + let n: Int = native_list_len(tokens) / 3 + if pos < 0 { return "0" } + if pos >= n { return "0" } + native_list_get(tokens, pos * 3 + 2) } fn tok_value(tokens: [Any], pos: Int) -> String { - let n: Int = native_list_len(tokens) / 2 + let n: Int = native_list_len(tokens) / 3 if pos < 0 { return "" } if pos >= n { return "" } - native_list_get(tokens, pos * 2 + 1) + native_list_get(tokens, pos * 3 + 1) } // parse_progress_fatal — robustness backstop. Called by the token-consuming @@ -1230,7 +1237,7 @@ fn parse_block(tokens: [Any], pos: Int) -> Map { // Runaway backstop: a block can hold at most (token count) statements, since // every iteration consumes >= 1 token. If we exceed that, the cursor has run // off the end without terminating (malformed input) -> fail fast, don't hang. - let blk_total: Int = native_list_len(tokens) / 2 + let blk_total: Int = native_list_len(tokens) / 3 let blk_iters: Int = 0 while running { let blk_iters = blk_iters + 1 @@ -1550,7 +1557,10 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let p = r2["pos"] // r2 result map fully consumed — release to free peak heap. el_release(r2) - return make_result({ "stmt": "FnDef", "name": name, "params": params, "body": body, "ret_type": ret_type }, p) + // The definition carries the line it was written on. Without it no + // diagnostic can name a place, and after textual inlining there is no + // way to say which FILE a definition came from. + return make_result({ "stmt": "FnDef", "name": name, "params": params, "body": body, "ret_type": ret_type, "line": tok_line(tokens, pos) }, p) } // type definition: `type Name = { field: Type, ... }` @@ -1842,6 +1852,7 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { "params": inner["params"], "body": inner["body"], "ret_type": inner["ret_type"], + "line": inner["line"], "decorator": dec_name, "decorators": dlist } @@ -2158,7 +2169,7 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { fn parse(tokens: [Any]) -> [Map] { // Flat list: 2 entries per token, so divide by 2 for token count. - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let stmts: [Map] = native_list_empty() let pos: Int = 0 let running = true @@ -2201,7 +2212,7 @@ fn parse_one(tokens: [Any], pos: Int) -> Map { // On entry, pos must point at the LBrace token. // Returns the position of the token AFTER the matching RBrace. fn skip_to_rbrace(tokens: [Any], pos: Int) -> Int { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let p: Int = pos + 1 let depth: Int = 1 let going: Bool = true @@ -2253,7 +2264,7 @@ fn is_stmt_start_kind(k: String) -> Bool { // token that could start a new top-level statement, staying depth-aware // so that braces inside expressions don't fool us. fn skip_expr_to_stmt_boundary(tokens: [Any], pos: Int) -> Int { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let p: Int = pos let depth: Int = 0 let going: Bool = true @@ -2419,7 +2430,7 @@ fn scan_params_el(tokens: [Any], pos: Int) -> Map { // // Peak memory: O(tokens) with no expression AST allocation. fn scan_fn_sigs_el(tokens: [Any]) -> [Map] { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let sigs: [Map] = native_list_empty() let pos: Int = 0 let going: Bool = true @@ -2561,7 +2572,7 @@ fn scan_params_c(tokens: [Any], pos: Int) -> Map { // // The scan allocates only small string values per entry, keeping peak RSS low. fn scan_fn_sigs(tokens: [Any]) -> [Map] { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let sigs: [Map] = native_list_empty() let pos: Int = 0 let going: Bool = true diff --git a/lang/tests/integration/definitions_query.sh b/lang/tests/integration/definitions_query.sh index 2f31c3c..e2fb523 100755 --- a/lang/tests/integration/definitions_query.sh +++ b/lang/tests/integration/definitions_query.sh @@ -17,9 +17,11 @@ out=$("$LANG_DIR/tools/check/definitions.sh" "$W/r.txt" 2>&1); rc=$? chk "a collision across modules is caught at El level" "1" "$rc" chk "the colliding name is reported" "1" "$(echo "$out" | grep -c "'helper' is defined 2 times")" chk "and the reason is given" "1" "$(echo "$out" | grep -c 'no namespacing')" +chk "both source FILES are named" "1" "$(echo "$out" | grep -c 'a.el:1')" +chk "with file-local line numbers, not combined ones" "1" "$(echo "$out" | grep -c 'b.el:1')" printf 'fn only_once() -> Int { return 1 }\nfn main() { println(int_to_str(only_once())) }\n' > "$W/c.el" EL_RELATIONS_OUT="$W/r2.txt" "$ELC" "$W/c.el" >/dev/null 2>&1 "$LANG_DIR/tools/check/definitions.sh" "$W/r2.txt" >/dev/null 2>&1 chk "a clean program exits 0" "0" "$?" -echo; echo " 4 assertions, $((4-F)) passed, $F failed"; exit $F +echo; echo " 6 assertions, $((6-F)) passed, $F failed"; exit $F diff --git a/lang/tests/native/test_compiler.el b/lang/tests/native/test_compiler.el index cf95597..5f2c304 100644 --- a/lang/tests/native/test_compiler.el +++ b/lang/tests/native/test_compiler.el @@ -18,7 +18,9 @@ import "../../el-compiler/src/compiler.el" // ── Lexer helpers ───────────────────────────────────────────────────────────── fn tok_count(tokens: [Any]) -> Int { - native_list_len(tokens) / 2 + // A token is (kind, value, line). This helper carried its own copy of the + // stride, so it escaped a search scoped to the compiler sources. + native_list_len(tokens) / 3 } // ── Codegen helper: capture compile() stdout to a string ───────────────────── @@ -259,22 +261,28 @@ test "lex-multiline-source" { assert tok_kind(tokens, 0) == "Let", "first token is Let" } -test "lex-flat-stride-2-layout" { - // Verify that the flat stride-2 layout: token i has kind at index 2*i, value at 2*i+1 +test "lex-flat-stride-3-layout" { + // A token is (kind, value, line): token i has kind at 3*i, value at 3*i+1, + // line at 3*i+2. Before 2026-08-17 a token carried no position at all, so + // no diagnostic in El could name a place. let tokens: [Any] = lex("fn foo") - // tokens[0] = "Fn", tokens[1] = "fn", tokens[2] = "Ident", tokens[3] = "foo", ... let raw_len: Int = native_list_len(tokens) - assert raw_len == 6, "fn + foo + Eof = 3 tokens = 6 raw entries" - let kind0: String = native_list_get(tokens, 0) - let val0: String = native_list_get(tokens, 1) - let kind1: String = native_list_get(tokens, 2) - let val1: String = native_list_get(tokens, 3) - assert kind0 == "Fn", "raw[0] is Fn kind" - assert val0 == "fn", "raw[1] is fn value" - assert kind1 == "Ident", "raw[2] is Ident kind" - assert val1 == "foo", "raw[3] is foo value" + assert raw_len == 9, "fn + foo + Eof = 3 tokens = 9 raw entries" + assert native_list_get(tokens, 0) == "Fn", "raw[0] is the kind" + assert native_list_get(tokens, 1) == "fn", "raw[1] is the value" + assert native_list_get(tokens, 2) == "1", "raw[2] is the line" + assert native_list_get(tokens, 3) == "Ident", "raw[3] is the next kind" + assert native_list_get(tokens, 5) == "1", "still line 1" } +test "lexer-tracks-line-numbers" { + let tokens: [Any] = lex("fn a\nfn b\nfn c") + assert tok_line(tokens, 0) == "1", "first fn is on line 1" + assert tok_line(tokens, 2) == "2", "second fn is on line 2" + assert tok_line(tokens, 4) == "3", "third fn is on line 3" +} + + // ── Parser tests ────────────────────────────────────────────────────────────── fn get_first_stmt_kind(src: String) -> String { diff --git a/lang/tools/check/definitions.sh b/lang/tools/check/definitions.sh index b97c7f2..64f6834 100755 --- a/lang/tools/check/definitions.sh +++ b/lang/tools/check/definitions.sh @@ -1,24 +1,42 @@ #!/usr/bin/env bash -# definitions.sh — catch duplicate top-level definitions at El level. +# definitions.sh — catch duplicate top-level definitions, and name the files. # # El has no namespacing. `import` is textual inlining, so two modules defining -# the same name emit two C functions into one translation unit. cc does catch -# it, but reports the generated helpers (__el_body_f, __env_f, __thunk_f) before -# the user's own function, so the first three errors name symbols the user never -# wrote. +# the same name emit two C functions into one translation unit. cc catches it, +# but reports the generated helpers (__el_body_f, __env_f, __thunk_f) before the +# user's own function, so the first three errors name symbols nobody wrote. # -# LIMIT, stated rather than hidden: textual inlining destroys file provenance. -# By the time codegen runs there is one source string and no record of which -# file a definition came from, so this can say WHICH name collides but not which -# files. Naming the files needs provenance threaded through resolve_imports. +# Naming the FILES needed provenance threaded end to end: tokens had no line +# numbers at all, so no diagnostic in El could name a place. Now a token is +# (kind, value, line), FnDef carries its line, and resolve_imports publishes +# which line range of the combined source came from which file. +# +# LIMIT: a nested import returns one string, so a definition inside a +# transitively imported file is attributed to the direct import. set -uo pipefail REL="${1:?usage: definitions.sh }" [ -f "$REL" ] || exit 0 + +# line in the COMBINED source -> "file:line-within-that-file". Reporting the +# combined line against a filename would point at a line that file does not +# have, which is worse than reporting no line at all. +locate() { + awk -v L="$1" '$2=="spans" && $3<=L && $4>=L {printf "%s:%d", $1, L-$3+1; found=1; exit} + END{ if(!found) printf "" }' "$REL" +} + V=0 -while read -r name count; do - [ "$count" -gt 1 ] || continue - printf "duplicate definition: '%s' is defined %s times — El has no namespacing, so imported modules share one global scope\n" "$name" "$count" +while read -r name; do + lines=$(grep -E "^$name calls defines_at:" "$REL" | sed 's/.*defines_at://' | sort -un) + n=$(echo "$lines" | wc -l | tr -d ' ') + [ "$n" -gt 1 ] || continue + printf "duplicate definition: '%s' is defined %s times — El has no namespacing, so imported modules share one global scope\n" "$name" "$n" + for l in $lines; do + loc=$(locate "$l") + [ -n "$loc" ] && printf " %s\n" "$loc" || printf " combined line %s\n" "$l" + done V=$((V+1)) -done < <(grep ' calls defines$' "$REL" | awk '{print $1}' | sort | uniq -c | awk '{print $2, $1}') +done < <(grep ' calls defines_at:' "$REL" | awk '{print $1}' | sort -u) + [ "$V" -eq 0 ] && echo "definitions: clean" exit "$V"