diff --git a/lang/el-compiler/src/codegen.el b/lang/el-compiler/src/codegen.el index 288e1c2..c8e4853 100644 --- a/lang/el-compiler/src/codegen.el +++ b/lang/el-compiler/src/codegen.el @@ -2802,7 +2802,7 @@ fn cg_fn(stmt: Map) -> Void { // functions into one translation unit. cc catches it, but names the // generated helpers first and the user's fn fourth. Recording it lets the // collision be reported at El level, in El terms. - record_call(fn_name, "defines") + record_call(fn_name, "defines_at:" + stmt["line"]) // Emit which constructs this fn carries, so the prohibition check can be a // query over relations instead of a walk inside the emitter. let cdl = stmt["decorators"] @@ -3630,7 +3630,7 @@ fn route_sort_desc(recs: [Map]) -> [Map] { // be synthesized in the streaming backend, which discards per-fn ASTs. Handles // decorator STACKING: `@route(...) @manager fn` still records the route. fn scan_routes(tokens: [Any]) -> [Map] { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let recs: [Map] = native_list_empty() let has_pending: Bool = false let pending_args: [String] = native_list_empty() @@ -3804,7 +3804,7 @@ fn declare_prohibition(construct: String, names_csv: String) -> Void { // to walk. fn scan_declared_decorators(tokens: [Any]) -> Void { declare_prohibition("manager", "dharma_emit,dharma_field") - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let has_pending_p: Bool = false let pending_prohibit: String = "" let pos: Int = 0 @@ -4015,7 +4015,7 @@ fn emit_streaming_preamble(sigs: [Map], source: String) -> Void { // sigs: pre-scanned signature list from scan_fn_sigs(tokens) // source: original source string (for string literal lookup) fn codegen_streaming(tokens: [Any], sigs: [Map], source: String) -> String { - let total_tokens: Int = native_list_len(tokens) / 2 + let total_tokens: Int = native_list_len(tokens) / 3 // Emit preamble (forward decls, file-scope lets, #includes) // Arena scope: free intermediate strings built during preamble emission. diff --git a/lang/el-compiler/src/compiler.el b/lang/el-compiler/src/compiler.el index 14c3ccd..b06405c 100644 --- a/lang/el-compiler/src/compiler.el +++ b/lang/el-compiler/src/compiler.el @@ -414,6 +414,12 @@ fn parse_import_line(trimmed: String, dir: String) -> String { // Accumulates chunks into lists and joins once at the end to avoid the O(n²) // memory growth caused by repeated `prefix = prefix + chunk` concatenation. fn resolve_imports(src_path: String) -> String { + // Only the OUTERMOST call publishes provenance. Nested calls number their + // lines from 1 within themselves, so their spans are meaningless once the + // text is spliced into the parent. + let depth: String = state_get("__elc_prov_depth") + if str_eq(depth, "") { state_set("__elc_prov_depth", "1") } + let is_top: Bool = str_eq(depth, "") let seen_key: String = "__elc_imp__:" + src_path let already: String = state_get(seen_key) if !str_eq(already, "") { return "" } @@ -443,6 +449,7 @@ fn resolve_imports(src_path: String) -> String { // Collect chunks into lists — O(1) amortized per append. // Join once at the end — O(n) single pass. let prefix_chunks: [String] = native_list_empty() + let prefix_paths: [String] = native_list_empty() let body_chunks: [String] = native_list_empty() let i: Int = 0 while i < n { @@ -454,21 +461,54 @@ fn resolve_imports(src_path: String) -> String { // Only check .elh for imported files — never for the entry file itself. let imp_elh_path: String = str_slice(imp_path, 0, str_len(imp_path) - 3) + ".elh" let imp_elh: String = fs_read(imp_elh_path) + // Provenance: record which line range of the combined source came + // from which file, so a diagnostic can name the FILE and not just a + // line in a string that no longer exists on disk. if !str_eq(imp_elh, "") { // Header exists: mark the .el as seen (so it won't be re-inlined // if something else also imports it) and use the header text. let seen_imp_key: String = "__elc_imp__:" + imp_path state_set(seen_imp_key, "1") let prefix_chunks = native_list_append(prefix_chunks, imp_elh) + let prefix_paths = native_list_append(prefix_paths, imp_path) } else { let imp_body: String = resolve_imports(imp_path) let prefix_chunks = native_list_append(prefix_chunks, imp_body) + let prefix_paths = native_list_append(prefix_paths, imp_path) } } else { let body_chunks = native_list_append(body_chunks, line + "\n") } let i = i + 1 } + // Walk the assembled chunks once and publish spans . + // LIMIT: nested imports return a single string, so their internal + // boundaries are already lost by the time we see them -- a definition + // inside a transitively imported file is attributed to the direct import. + // Local, not accumulated in state: a nested call numbers its lines from 1 + // within itself, so letting it append to a shared buffer republishes + // meaningless spans under the parent's name. + let prov: String = "" + let line_at: Int = 1 + let ci: Int = 0 + let nchunks: Int = native_list_len(prefix_chunks) + while ci < nchunks { + let chunk: String = native_list_get(prefix_chunks, ci) + let nlines: Int = str_count_lines(chunk) + let src: String = native_list_get(prefix_paths, ci) + let prov = prov + src + " spans " + native_int_to_str(line_at) + " " + native_int_to_str(line_at + nlines - 1) + "\n" + let line_at = line_at + nlines + let ci = ci + 1 + } + let prov = prov + src_path + " spans " + native_int_to_str(line_at) + " 999999\n" + if is_top { + let prov_out: String = env("EL_RELATIONS_OUT") + if !str_eq(prov_out, "") { + let existing: String = "" + if fs_exists(prov_out) { let existing = fs_read(prov_out) } + fs_write(prov_out, existing + prov) + } + } return str_join(prefix_chunks, "") + str_join(body_chunks, "") } diff --git a/lang/el-compiler/src/lexer.el b/lang/el-compiler/src/lexer.el index a14f7e2..358f486 100644 --- a/lang/el-compiler/src/lexer.el +++ b/lang/el-compiler/src/lexer.el @@ -138,9 +138,18 @@ fn lex_is_whitespace(ch: String) -> Bool { // tok_append — append a (kind, value) pair to a flat token list. // Returns the updated list. Gamma combines flat-list + char-code for max savings. +// A token is (kind, value, line). The line comes from state rather than a +// parameter so the ~200 existing tok_append call sites are untouched -- the +// lexer advances __lex_line as it walks, and every token minted takes the line +// it was minted on. +// +// WHY AT ALL: before this a token carried no position, so no diagnostic in El +// could name a place. Every error named a symbol and never a line, and after +// textual inlining there was no way to say which FILE a definition came from. fn tok_append(tokens: [Any], kind: String, value: String) -> [Any] { let tokens = native_list_append(tokens, kind) - native_list_append(tokens, value) + let tokens = native_list_append(tokens, value) + native_list_append(tokens, state_get("__lex_line")) } // -- Keyword lookup ------------------------------------------------------------ @@ -532,6 +541,12 @@ fn scan_interp_brace(src: String, start: Int, total: Int) -> Map { // interp_tokens_append_all - copy every (kind, value) pair from flat src list // into flat dst list, skipping the trailing Eof pair that lex() always appends. +// Splices re-lexed interpolation tokens into the stream. This walks the token +// list DIRECTLY rather than through tok_append, so it carries its own copy of +// the stride -- which is why giving tokens a line broke the compiler's second +// generation and not its first: the compiler's own source uses string +// interpolation, so gen1 (built by the old compiler) was fine and gen2 emitted +// a corrupted stream. fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] { let src_len: Int = native_list_len(src) let j = 0 @@ -542,9 +557,11 @@ fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] { let j = src_len } else { let val: String = native_list_get(src, j + 1) + let ln: String = native_list_get(src, j + 2) let result = native_list_append(result, kind) let result = native_list_append(result, val) - let j = j + 2 + let result = native_list_append(result, ln) + let j = j + 3 } } result @@ -775,8 +792,14 @@ fn lex(source: String) -> [Any] { let total: Int = str_len(source) let tokens: [Any] = native_list_empty() let i: Int = 0 + state_set("__lex_line", "1") + let line_no: Int = 1 while i < total { + if str_eq(str_slice(source, i, i + 1), "\n") { + let line_no = line_no + 1 + state_set("__lex_line", native_int_to_str(line_no)) + } let c: Int = str_char_code(source, i) // Skip whitespace (space=32, tab=9, newline=10, CR=13) diff --git a/lang/el-compiler/src/parser.el b/lang/el-compiler/src/parser.el index 68c69d3..f72015f 100644 --- a/lang/el-compiler/src/parser.el +++ b/lang/el-compiler/src/parser.el @@ -17,8 +17,8 @@ // programs. All callers use these helpers -- only these three need updating. fn tok_at(tokens: [Any], pos: Int) -> Map { - let kind: String = native_list_get(tokens, pos * 2) - let value: String = native_list_get(tokens, pos * 2 + 1) + let kind: String = native_list_get(tokens, pos * 3) + let value: String = native_list_get(tokens, pos * 3 + 1) { "kind": kind, "value": value } } @@ -28,25 +28,32 @@ fn tok_kind(tokens: [Any], pos: Int) -> String { // single trailing Eof token returns runtime null (el_list_get OOB -> 0), // which matches no delimiter, letting inner parse loops append AST nodes // forever on malformed input -> unbounded allocation -> OOM. - let n: Int = native_list_len(tokens) / 2 + let n: Int = native_list_len(tokens) / 3 if pos < 0 { return "Eof" } if pos >= n { return "Eof" } - native_list_get(tokens, pos * 2) + native_list_get(tokens, pos * 3) +} + +fn tok_line(tokens: [Any], pos: Int) -> String { + let n: Int = native_list_len(tokens) / 3 + if pos < 0 { return "0" } + if pos >= n { return "0" } + native_list_get(tokens, pos * 3 + 2) } fn tok_value(tokens: [Any], pos: Int) -> String { - let n: Int = native_list_len(tokens) / 2 + let n: Int = native_list_len(tokens) / 3 if pos < 0 { return "" } if pos >= n { return "" } - native_list_get(tokens, pos * 2 + 1) + native_list_get(tokens, pos * 3 + 1) } // parse_progress_fatal — robustness backstop. Called by the token-consuming @@ -1230,7 +1237,7 @@ fn parse_block(tokens: [Any], pos: Int) -> Map { // Runaway backstop: a block can hold at most (token count) statements, since // every iteration consumes >= 1 token. If we exceed that, the cursor has run // off the end without terminating (malformed input) -> fail fast, don't hang. - let blk_total: Int = native_list_len(tokens) / 2 + let blk_total: Int = native_list_len(tokens) / 3 let blk_iters: Int = 0 while running { let blk_iters = blk_iters + 1 @@ -1550,7 +1557,10 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let p = r2["pos"] // r2 result map fully consumed — release to free peak heap. el_release(r2) - return make_result({ "stmt": "FnDef", "name": name, "params": params, "body": body, "ret_type": ret_type }, p) + // The definition carries the line it was written on. Without it no + // diagnostic can name a place, and after textual inlining there is no + // way to say which FILE a definition came from. + return make_result({ "stmt": "FnDef", "name": name, "params": params, "body": body, "ret_type": ret_type, "line": tok_line(tokens, pos) }, p) } // type definition: `type Name = { field: Type, ... }` @@ -1842,6 +1852,7 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { "params": inner["params"], "body": inner["body"], "ret_type": inner["ret_type"], + "line": inner["line"], "decorator": dec_name, "decorators": dlist } @@ -2158,7 +2169,7 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { fn parse(tokens: [Any]) -> [Map] { // Flat list: 2 entries per token, so divide by 2 for token count. - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let stmts: [Map] = native_list_empty() let pos: Int = 0 let running = true @@ -2201,7 +2212,7 @@ fn parse_one(tokens: [Any], pos: Int) -> Map { // On entry, pos must point at the LBrace token. // Returns the position of the token AFTER the matching RBrace. fn skip_to_rbrace(tokens: [Any], pos: Int) -> Int { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let p: Int = pos + 1 let depth: Int = 1 let going: Bool = true @@ -2253,7 +2264,7 @@ fn is_stmt_start_kind(k: String) -> Bool { // token that could start a new top-level statement, staying depth-aware // so that braces inside expressions don't fool us. fn skip_expr_to_stmt_boundary(tokens: [Any], pos: Int) -> Int { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let p: Int = pos let depth: Int = 0 let going: Bool = true @@ -2419,7 +2430,7 @@ fn scan_params_el(tokens: [Any], pos: Int) -> Map { // // Peak memory: O(tokens) with no expression AST allocation. fn scan_fn_sigs_el(tokens: [Any]) -> [Map] { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let sigs: [Map] = native_list_empty() let pos: Int = 0 let going: Bool = true @@ -2561,7 +2572,7 @@ fn scan_params_c(tokens: [Any], pos: Int) -> Map { // // The scan allocates only small string values per entry, keeping peak RSS low. fn scan_fn_sigs(tokens: [Any]) -> [Map] { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let sigs: [Map] = native_list_empty() let pos: Int = 0 let going: Bool = true diff --git a/lang/tests/integration/definitions_query.sh b/lang/tests/integration/definitions_query.sh index 2f31c3c..e2fb523 100755 --- a/lang/tests/integration/definitions_query.sh +++ b/lang/tests/integration/definitions_query.sh @@ -17,9 +17,11 @@ out=$("$LANG_DIR/tools/check/definitions.sh" "$W/r.txt" 2>&1); rc=$? chk "a collision across modules is caught at El level" "1" "$rc" chk "the colliding name is reported" "1" "$(echo "$out" | grep -c "'helper' is defined 2 times")" chk "and the reason is given" "1" "$(echo "$out" | grep -c 'no namespacing')" +chk "both source FILES are named" "1" "$(echo "$out" | grep -c 'a.el:1')" +chk "with file-local line numbers, not combined ones" "1" "$(echo "$out" | grep -c 'b.el:1')" printf 'fn only_once() -> Int { return 1 }\nfn main() { println(int_to_str(only_once())) }\n' > "$W/c.el" EL_RELATIONS_OUT="$W/r2.txt" "$ELC" "$W/c.el" >/dev/null 2>&1 "$LANG_DIR/tools/check/definitions.sh" "$W/r2.txt" >/dev/null 2>&1 chk "a clean program exits 0" "0" "$?" -echo; echo " 4 assertions, $((4-F)) passed, $F failed"; exit $F +echo; echo " 6 assertions, $((6-F)) passed, $F failed"; exit $F diff --git a/lang/tests/native/test_compiler.el b/lang/tests/native/test_compiler.el index cf95597..5f2c304 100644 --- a/lang/tests/native/test_compiler.el +++ b/lang/tests/native/test_compiler.el @@ -18,7 +18,9 @@ import "../../el-compiler/src/compiler.el" // ── Lexer helpers ───────────────────────────────────────────────────────────── fn tok_count(tokens: [Any]) -> Int { - native_list_len(tokens) / 2 + // A token is (kind, value, line). This helper carried its own copy of the + // stride, so it escaped a search scoped to the compiler sources. + native_list_len(tokens) / 3 } // ── Codegen helper: capture compile() stdout to a string ───────────────────── @@ -259,22 +261,28 @@ test "lex-multiline-source" { assert tok_kind(tokens, 0) == "Let", "first token is Let" } -test "lex-flat-stride-2-layout" { - // Verify that the flat stride-2 layout: token i has kind at index 2*i, value at 2*i+1 +test "lex-flat-stride-3-layout" { + // A token is (kind, value, line): token i has kind at 3*i, value at 3*i+1, + // line at 3*i+2. Before 2026-08-17 a token carried no position at all, so + // no diagnostic in El could name a place. let tokens: [Any] = lex("fn foo") - // tokens[0] = "Fn", tokens[1] = "fn", tokens[2] = "Ident", tokens[3] = "foo", ... let raw_len: Int = native_list_len(tokens) - assert raw_len == 6, "fn + foo + Eof = 3 tokens = 6 raw entries" - let kind0: String = native_list_get(tokens, 0) - let val0: String = native_list_get(tokens, 1) - let kind1: String = native_list_get(tokens, 2) - let val1: String = native_list_get(tokens, 3) - assert kind0 == "Fn", "raw[0] is Fn kind" - assert val0 == "fn", "raw[1] is fn value" - assert kind1 == "Ident", "raw[2] is Ident kind" - assert val1 == "foo", "raw[3] is foo value" + assert raw_len == 9, "fn + foo + Eof = 3 tokens = 9 raw entries" + assert native_list_get(tokens, 0) == "Fn", "raw[0] is the kind" + assert native_list_get(tokens, 1) == "fn", "raw[1] is the value" + assert native_list_get(tokens, 2) == "1", "raw[2] is the line" + assert native_list_get(tokens, 3) == "Ident", "raw[3] is the next kind" + assert native_list_get(tokens, 5) == "1", "still line 1" } +test "lexer-tracks-line-numbers" { + let tokens: [Any] = lex("fn a\nfn b\nfn c") + assert tok_line(tokens, 0) == "1", "first fn is on line 1" + assert tok_line(tokens, 2) == "2", "second fn is on line 2" + assert tok_line(tokens, 4) == "3", "third fn is on line 3" +} + + // ── Parser tests ────────────────────────────────────────────────────────────── fn get_first_stmt_kind(src: String) -> String { diff --git a/lang/tools/check/definitions.sh b/lang/tools/check/definitions.sh index b97c7f2..64f6834 100755 --- a/lang/tools/check/definitions.sh +++ b/lang/tools/check/definitions.sh @@ -1,24 +1,42 @@ #!/usr/bin/env bash -# definitions.sh — catch duplicate top-level definitions at El level. +# definitions.sh — catch duplicate top-level definitions, and name the files. # # El has no namespacing. `import` is textual inlining, so two modules defining -# the same name emit two C functions into one translation unit. cc does catch -# it, but reports the generated helpers (__el_body_f, __env_f, __thunk_f) before -# the user's own function, so the first three errors name symbols the user never -# wrote. +# the same name emit two C functions into one translation unit. cc catches it, +# but reports the generated helpers (__el_body_f, __env_f, __thunk_f) before the +# user's own function, so the first three errors name symbols nobody wrote. # -# LIMIT, stated rather than hidden: textual inlining destroys file provenance. -# By the time codegen runs there is one source string and no record of which -# file a definition came from, so this can say WHICH name collides but not which -# files. Naming the files needs provenance threaded through resolve_imports. +# Naming the FILES needed provenance threaded end to end: tokens had no line +# numbers at all, so no diagnostic in El could name a place. Now a token is +# (kind, value, line), FnDef carries its line, and resolve_imports publishes +# which line range of the combined source came from which file. +# +# LIMIT: a nested import returns one string, so a definition inside a +# transitively imported file is attributed to the direct import. set -uo pipefail REL="${1:?usage: definitions.sh }" [ -f "$REL" ] || exit 0 + +# line in the COMBINED source -> "file:line-within-that-file". Reporting the +# combined line against a filename would point at a line that file does not +# have, which is worse than reporting no line at all. +locate() { + awk -v L="$1" '$2=="spans" && $3<=L && $4>=L {printf "%s:%d", $1, L-$3+1; found=1; exit} + END{ if(!found) printf "" }' "$REL" +} + V=0 -while read -r name count; do - [ "$count" -gt 1 ] || continue - printf "duplicate definition: '%s' is defined %s times — El has no namespacing, so imported modules share one global scope\n" "$name" "$count" +while read -r name; do + lines=$(grep -E "^$name calls defines_at:" "$REL" | sed 's/.*defines_at://' | sort -un) + n=$(echo "$lines" | wc -l | tr -d ' ') + [ "$n" -gt 1 ] || continue + printf "duplicate definition: '%s' is defined %s times — El has no namespacing, so imported modules share one global scope\n" "$name" "$n" + for l in $lines; do + loc=$(locate "$l") + [ -n "$loc" ] && printf " %s\n" "$loc" || printf " combined line %s\n" "$l" + done V=$((V+1)) -done < <(grep ' calls defines$' "$REL" | awk '{print $1}' | sort | uniq -c | awk '{print $2, $1}') +done < <(grep ' calls defines_at:' "$REL" | awk '{print $1}' | sort -u) + [ "$V" -eq 0 ] && echo "definitions: clean" exit "$V"