// compiler.el — el self-hosting compiler pipeline // // Wires lexer -> parser -> codegen into a single compile() function. // This is the bootstrap entry point: compiled once by the Rust el-compiler, // then self-hosted from that point forward. // // Two backends: // - C (default) compile() -> emits C source linked against el_runtime.c // - JS (--target=js) compile_js() -> emits JS source linked against el_runtime.js // // Compile the C output with: // cc -o .c el_runtime.c // // Run the JS output with: // node .js (after copying el_runtime.js next to it) import "lexer.el" import "parser.el" import "codegen.el" import "codegen-js.el" // compile — full pipeline (C target): source string -> C source string // Uses JIT function-at-a-time streaming: parse one decl → emit C → discard AST. // Peak memory is O(one function's AST) instead of O(whole program AST). fn compile(source: String) -> String { // Top-level arena scope: activates the string arena before lex() so that // ALL strdup allocations (token strings, sig strings, codegen fragments) // are tracked and freed on pop. Without this, lex() and scan_fn_sigs() // run before any push, leaving _tl_arena_active=0 and leaking every // token string. Also prevents inner pop(mark=0) calls from deactivating // the arena between per-function scopes. let top_mark: Any = el_arena_push() let tokens: [Any] = lex(source) // Fast pre-scan: collect fn signatures + program kind without building // full expression ASTs. O(tokens) time, minimal allocation. let sigs: [Map] = scan_fn_sigs(tokens) // Stream parse-emit: parse one decl at a time, emit C, discard. // All output written to stdout via println before pop. codegen_streaming(tokens, sigs, source) el_arena_pop(top_mark) "" } // compile_test — like compile() but sets __test_mode so codegen_streaming // compiles test { } blocks instead of skipping them, and emits the test // harness main() instead of the normal int main(). fn compile_test(source: String) -> String { state_set("__test_mode", "1") let top_mark: Any = el_arena_push() let tokens: [Any] = lex(source) let sigs: [Map] = scan_fn_sigs(tokens) codegen_streaming(tokens, sigs, source) el_arena_pop(top_mark) state_set("__test_mode", "") "" } // compile_js — full pipeline (JS target, module mode): source string -> JS source string fn compile_js(source: String) -> String { let tokens: [Any] = lex(source) let stmts: [Map] = parse(tokens) // Token list is no longer needed after parsing — release it to free memory. el_release(tokens) codegen_js(stmts, source) } // compile_js_with_bundle — JS target in bundle mode. // Reads el_runtime.js from runtime_path and inlines it inside an IIFE. fn compile_js_with_bundle(source: String, runtime_path: String) -> String { let tokens: [Any] = lex(source) let stmts: [Map] = parse(tokens) el_release(tokens) let runtime_content: String = fs_read(runtime_path) if str_eq(runtime_content, "") { println("el-compiler: warning: --bundle: could not read runtime at " + runtime_path) println("el-compiler: warning: bundle output will be incomplete") } codegen_js_bundle(stmts, source, runtime_content) } // compile_dispatch — pick a backend based on the requested target. // tgt = "c" | "js" // (The parameter is named `tgt` because `target` is a reserved keyword // in El's lexer — it would be tokenised as `Target`, breaking the // parser's identifier resolution.) fn compile_dispatch(tgt: String, source: String) -> String { if str_eq(tgt, "js") { return compile_js(source) } compile(source) } // compile_dispatch_bundle — like compile_dispatch but bundle mode for JS. fn compile_dispatch_bundle(tgt: String, source: String, runtime_path: String) -> String { if str_eq(tgt, "js") { return compile_js_with_bundle(source, runtime_path) } compile(source) } // Detect a `--target=` flag in argv and return the target. // Returns "c" if none specified or unrecognized. fn detect_target(argv: [String]) -> String { let n: Int = native_list_len(argv) let i = 0 while i < n { let a: String = native_list_get(argv, i) if str_starts_with(a, "--target=") { let v: String = str_slice(a, 9, str_len(a)) return v } let i = i + 1 } return "c" } // Strip flags from argv, leaving only positional arguments. fn strip_flags(argv: [String]) -> [String] { let out: [String] = native_list_empty() let n: Int = native_list_len(argv) let i = 0 while i < n { let a: String = native_list_get(argv, i) if !str_starts_with(a, "--") { let out = native_list_append(out, a) } let i = i + 1 } return out } // Detect --emit-header flag in argv. fn detect_emit_header(argv: [String]) -> Bool { let n: Int = native_list_len(argv) let i = 0 while i < n { let a: String = native_list_get(argv, i) if str_eq(a, "--emit-header") { return true } let i = i + 1 } return false } // Detect --bundle flag in argv. fn detect_bundle(argv: [String]) -> Bool { let n: Int = native_list_len(argv) let i = 0 while i < n { let a: String = native_list_get(argv, i) if str_eq(a, "--bundle") { return true } let i = i + 1 } return false } // Detect --minify flag in argv. fn detect_minify(argv: [String]) -> Bool { let n: Int = native_list_len(argv) let i = 0 while i < n { let a: String = native_list_get(argv, i) if str_eq(a, "--minify") { return true } let i = i + 1 } return false } // Detect --obfuscate flag in argv. fn detect_obfuscate(argv: [String]) -> Bool { let n: Int = native_list_len(argv) let i = 0 while i < n { let a: String = native_list_get(argv, i) if str_eq(a, "--obfuscate") { return true } let i = i + 1 } return false } // Detect --test flag in argv. fn detect_test(argv: [String]) -> Bool { let n: Int = native_list_len(argv) let i = 0 while i < n { let a: String = native_list_get(argv, i) if str_eq(a, "--test") { return true } let i = i + 1 } return false } // Build a unique temp file path: /tmp/elc--. fn make_temp_path(suffix: String) -> String { let pid: Int = getpid_now() let ts: Int = time_now() "/tmp/elc-" + native_int_to_str(pid) + "-" + native_int_to_str(ts) + "." + suffix } // Reserved globals that terser and javascript-obfuscator must not mangle. // These are referenced from HTML onclick= attributes and other direct window usage. fn js_reserved_names() -> String { "neuronDemoToggle,neuronDemoSend,neuronDemoReset,signInWith,signInWithEmail,signUpWithEmail,sendMagicLink,signOut,resetPassword,sendResetEmail,updatePassword,showSignIn,showSignUp,hideReset,setSort,addFamilyMember,removeFamilyMember,copyForPlatform,entHeadcountChange,NEURON_CFG" } // Find a CLI tool by checking node_modules paths first, then falling back to npx. // src_dir is the directory of the source file being compiled. // Returns the command string to invoke the tool, or "" if not found. fn find_node_tool(tool_name: String, src_dir: String) -> String { // 1. Check ./node_modules/.bin/ relative to source file let cand1: String = src_dir + "/node_modules/.bin/" + tool_name let check1: String = str_trim(exec_capture("test -x " + cand1 + " && echo yes 2>/dev/null")) if str_eq(check1, "yes") { return cand1 } // 2. Check ../node_modules/.bin/ (monorepo layout) let parent_dir: String = dirname_of(src_dir) let cand2: String = parent_dir + "/node_modules/.bin/" + tool_name let check2: String = str_trim(exec_capture("test -x " + cand2 + " && echo yes 2>/dev/null")) if str_eq(check2, "yes") { return cand2 } // 3. Fall back to npx if it is on PATH. npx will use the globally cached // package or download on first use. Use --no to avoid auto-install if // the package is not already cached; if that fails, try with --yes. let npx_path: String = str_trim(exec_capture("which npx 2>/dev/null")) if !str_eq(npx_path, "") { return "npx --yes " + tool_name } return "" } // apply_minify — run terser on js_path, write result to out_path. // Returns true on success, false on failure. fn apply_minify(js_path: String, out_path: String, src_dir: String) -> Bool { let terser: String = find_node_tool("terser", src_dir) if str_eq(terser, "") { println("el-compiler: error: terser not found. Run 'npm install terser' in your project directory.") return false } let names: String = js_reserved_names() // Single-quote the mangle reserved list so the shell does not glob-expand // the bracket expression. The compress options are safe without quoting. let compress_opts: String = "passes=2,drop_console=false,drop_debugger=true" let mangle_reserved: String = "'reserved=[" + names + "]'" let cmd: String = terser + " " + js_path + " --compress " + compress_opts + " --mangle " + mangle_reserved + " --output " + out_path let ret: Int = exec_command(cmd) if ret == 0 { return true } println("el-compiler: error: terser failed (exit " + native_int_to_str(ret) + ")") return false } // apply_obfuscate — run javascript-obfuscator on js_path, write result to out_path. // Returns true on success, false on failure. fn apply_obfuscate(js_path: String, out_path: String, src_dir: String) -> Bool { let obfuscator: String = find_node_tool("javascript-obfuscator", src_dir) if str_eq(obfuscator, "") { println("el-compiler: error: javascript-obfuscator not found. Run 'npm install javascript-obfuscator' in your project directory.") return false } let names: String = js_reserved_names() let cmd: String = obfuscator + " " + js_path + " --output " + out_path + " --compact true --simplify true --string-array true --string-array-encoding base64 --string-array-threshold 0.75 --identifier-names-generator hexadecimal --rename-globals false --self-defending false --reserved-names " + names let ret: Int = exec_command(cmd) if ret == 0 { return true } println("el-compiler: error: javascript-obfuscator failed (exit " + native_int_to_str(ret) + ")") return false } // Resolve the runtime path for --bundle mode. // Looks for el_runtime.js next to the source file first; // if not found there, looks next to the elc binary itself. // Returns "" if not found anywhere (caller emits a warning). fn resolve_runtime_path(src_path: String) -> String { let src_dir: String = dirname_of(src_path) let candidate: String = src_dir + "/el_runtime.js" let existing: String = fs_read(candidate) if !str_eq(existing, "") { return candidate } return "" } // Reconstruct an El type annotation string from a parsed type node. fn type_node_to_el(t: Map) -> String { let k: String = t["kind"] if str_eq(k, "Simple") { return t["name"] } if str_eq(k, "List") { let inner: String = type_node_to_el(t["inner"]) return "[" + inner + "]" } if str_eq(k, "Map") { let kt: String = type_node_to_el(t["key"]) let vt: String = type_node_to_el(t["val"]) return "Map<" + kt + ", " + vt + ">" } "Any" } // emit_header — write a .elh file from parsed statements. // Scans for FnDef nodes and emits 'extern fn' declarations. // NOTE: This function requires the full AST. Prefer emit_header_from_sigs // for the --emit-header path — it works from a token-level scan without // building expression ASTs, avoiding OOM on large files. fn emit_header(stmts: [Map], hdr_path: String) -> Void { let n: Int = native_list_len(stmts) let i = 0 let parts: [String] = native_list_empty() let parts = native_list_append(parts, "// auto-generated by elc --emit-header — do not edit\n") while i < n { let stmt = native_list_get(stmts, i) let kind: String = stmt["stmt"] if str_eq(kind, "FnDef") { let name: String = stmt["name"] if !str_eq(name, "main") { let params = stmt["params"] let ret_type: String = stmt["ret_type"] // build param list let np: Int = native_list_len(params) let pi = 0 let param_parts: [String] = native_list_empty() while pi < np { let param = native_list_get(params, pi) let pname: String = param["name"] let ptype: String = param["type"] if str_eq(ptype, "") { let ptype = "Any" } let param_parts = native_list_append(param_parts, pname + ": " + ptype) let pi = pi + 1 } let params_str: String = str_join(param_parts, ", ") let ret_str: String = ret_type if str_eq(ret_str, "") { let ret_str = "Any" } let sig: String = "extern fn " + name + "(" + params_str + ") -> " + ret_str let parts = native_list_append(parts, sig + "\n") } } let i = i + 1 } let content: String = str_join(parts, "") let ok: Bool = fs_write(hdr_path, content) } // emit_header_from_sigs — write a .elh file from pre-scanned El signatures. // Uses the output of scan_fn_sigs_el() — no full AST required. // Peak memory is O(tokens) rather than O(whole-program AST), which prevents // OOM on large files with HTML template bodies or deep BinOp chains. fn emit_header_from_sigs(sigs: [Map], hdr_path: String) -> Void { let n: Int = native_list_len(sigs) let i: Int = 0 let parts: [String] = native_list_empty() let parts = native_list_append(parts, "// auto-generated by elc --emit-header — do not edit\n") while i < n { let sig = native_list_get(sigs, i) let kind: String = sig["kind"] if str_eq(kind, "fn") { let name: String = sig["name"] let params_el: String = sig["params_el"] let ret_el: String = sig["ret_el"] if str_eq(ret_el, "") { let ret_el = "Any" } let line: String = "extern fn " + name + "(" + params_el + ") -> " + ret_el let parts = native_list_append(parts, line + "\n") } let i = i + 1 } let content: String = str_join(parts, "") let ok: Bool = fs_write(hdr_path, content) } // ── Import resolution ──────────────────────────────────────────────────────── // // elc supports two forms of import: // import "path/to/file.el" — quoted relative path // from module import { Name } — bare module name resolves to module.el // in the entry source's directory // // Codegen treats Import statements as no-ops (declarations only), so to // actually link bodies across files we textually concatenate every imported // source ahead of the entry source before lex/parse. resolve_imports does a // depth-first traversal with deduplication so any module that gets pulled in // transitively is included exactly once. fn dirname_of(path: String) -> String { let n: Int = str_len(path) let i: Int = n - 1 while i >= 0 { let c: String = str_slice(path, i, i + 1) if str_eq(c, "/") { return str_slice(path, 0, i) } let i = i - 1 } return "." } // Extract the resolved file path from a single trimmed source line. Returns // "" if the line is not an import. fn parse_import_line(trimmed: String, dir: String) -> String { if str_starts_with(trimmed, "import \"") { let after: String = str_slice(trimmed, 8, str_len(trimmed)) let q: Int = str_index_of(after, "\"") if q > 0 { let mod: String = str_slice(after, 0, q) return dir + "/" + mod } } if str_starts_with(trimmed, "from ") { let after: String = str_slice(trimmed, 5, str_len(trimmed)) // module name is the first whitespace-delimited token let sp: Int = str_index_of(after, " ") if sp > 0 { let mod_raw: String = str_slice(after, 0, sp) let mod: String = str_trim(mod_raw) if !str_eq(mod, "") { return dir + "/" + mod + ".el" } } } return "" } // Recursively resolve imports starting from src_path. Returns the combined // source text with every imported module's body inlined ahead of the entry // source, deduplicated by absolute path. Uses state_set to track which paths // have already been pulled in for this run. // // Accumulates chunks into lists and joins once at the end to avoid the O(n²) // memory growth caused by repeated `prefix = prefix + chunk` concatenation. fn resolve_imports(src_path: String) -> String { let seen_key: String = "__elc_imp__:" + src_path let already: String = state_get(seen_key) if !str_eq(already, "") { return "" } state_set(seen_key, "1") // A missing file must be a hard error, never an empty string. // // fs_read returns "" both for "file is empty" and "file does not exist", and // this function used the value without distinguishing them. So a broken // import path — a typo, a moved file, a relative path resolved from the // wrong working directory — compiled CLEANLY: exit 0, empty stderr, and a // program silently missing everything it imported. Observed 2026-08-15: // eleven consecutive "successful" compiles that had included no runtime at // all, and a wrong conclusion drawn from them before anyone noticed. // // Missing dependency, confident success. fs_exists separates the two cases, // so a genuinely empty file still resolves to "" and is fine. if !fs_exists(src_path) { println("elc: cannot resolve import: " + src_path) exit_program(1) } let source: String = fs_read(src_path) let dir: String = dirname_of(src_path) let lines: [String] = str_split(source, "\n") let n: Int = native_list_len(lines) // Collect chunks into lists — O(1) amortized per append. // Join once at the end — O(n) single pass. let prefix_chunks: [String] = native_list_empty() let body_chunks: [String] = native_list_empty() let i: Int = 0 while i < n { let line: String = native_list_get(lines, i) let trimmed: String = str_trim(line) let imp_path: String = parse_import_line(trimmed, dir) if !str_eq(imp_path, "") { // Use pre-compiled header if available (separate compilation). // Only check .elh for imported files — never for the entry file itself. let imp_elh_path: String = str_slice(imp_path, 0, str_len(imp_path) - 3) + ".elh" let imp_elh: String = fs_read(imp_elh_path) if !str_eq(imp_elh, "") { // Header exists: mark the .el as seen (so it won't be re-inlined // if something else also imports it) and use the header text. let seen_imp_key: String = "__elc_imp__:" + imp_path state_set(seen_imp_key, "1") let prefix_chunks = native_list_append(prefix_chunks, imp_elh) } else { let imp_body: String = resolve_imports(imp_path) let prefix_chunks = native_list_append(prefix_chunks, imp_body) } } else { let body_chunks = native_list_append(body_chunks, line + "\n") } let i = i + 1 } return str_join(prefix_chunks, "") + str_join(body_chunks, "") } // run_with_postprocess — codegen + minify + optional obfuscate pipeline. // // Called from main() when --minify or --obfuscate is active. Redirects stdout // to a temp file during codegen so the output can be passed through the // external tools (terser, javascript-obfuscator) before final emission. // // Pipeline: codegen -> terser -> (javascript-obfuscator) -> stdout or file fn run_with_postprocess(tgt: String, source: String, src_path: String, do_bundle: Bool, do_obfuscate: Bool, argc: Int, positional: [String]) -> Void { let src_dir: String = dirname_of(src_path) let tmp_gen: String = make_temp_path("js") let tmp_min: String = make_temp_path("min.js") // Redirect stdout to tmp_gen so codegen println output is captured. stdout_to_file(tmp_gen) if do_bundle { let runtime_path: String = resolve_runtime_path(src_path) compile_dispatch_bundle(tgt, source, runtime_path) } else { compile_dispatch(tgt, source) } stdout_restore() // Run terser: tmp_gen -> tmp_min let ok_min: Bool = apply_minify(tmp_gen, tmp_min, src_dir) if !ok_min { exec_command("rm -f " + tmp_gen + " " + tmp_min) exit(1) } // Determine final result path (either tmp_min or post-obfuscation file). // Use state to pass the final path out of the optional obfuscation branch. state_set("__elc_final_js", tmp_min) if do_obfuscate { let tmp_obf: String = make_temp_path("obf.js") let ok_obf: Bool = apply_obfuscate(tmp_min, tmp_obf, src_dir) if !ok_obf { exec_command("rm -f " + tmp_gen + " " + tmp_min + " " + tmp_obf) exit(1) } state_set("__elc_final_js", tmp_obf) } let final_path: String = state_get("__elc_final_js") let final_js: String = fs_read(final_path) // Clean up all temp files. exec_command("rm -f " + tmp_gen + " " + tmp_min) if do_obfuscate { exec_command("rm -f " + final_path) } if argc >= 2 { let out_path: String = native_list_get(positional, 1) let ok: Bool = fs_write(out_path, final_js) if ok { return } else { println("el-compiler: failed to write output") exit(1) } } // No output file: print final JS to stdout. print(final_js) } // main — CLI entry point. // // elc # emit C to stdout // elc --target=js # emit JS (module) to stdout // elc --target=js --bundle # emit self-contained JS (IIFE) to stdout // elc --target=js --bundle --minify # emit minified IIFE to stdout // elc --target=js --bundle --obfuscate # emit minified+obfuscated IIFE to stdout // elc --target=c # write C to file // elc --target=js # write JS to file // elc --target=js --bundle # write bundled JS to file // elc --target=js --bundle --minify # write minified JS to file fn main() -> Void { let argv: [String] = args() // Use `tgt` not `target`: `target` is a reserved keyword in the lexer // (Section 1.5 of the language spec). detect_target itself is fine // because the function-name position has no token-class restriction. let tgt: String = detect_target(argv) let do_emit_header: Bool = detect_emit_header(argv) let do_bundle: Bool = detect_bundle(argv) let do_minify: Bool = detect_minify(argv) let do_obfuscate: Bool = detect_obfuscate(argv) let do_test: Bool = detect_test(argv) // --obfuscate implies --minify: obfuscating unminified code is pointless. if do_obfuscate { let do_minify = true } let positional: [String] = strip_flags(argv) let argc: Int = native_list_len(positional) if argc < 1 { println("el-compiler: usage: elc [--target=c|js] [--bundle] [--minify] [--obfuscate] [--emit-header] [--test] []") exit(1) } // --minify and --obfuscate require --target=js if do_minify { if !str_eq(tgt, "js") { println("el-compiler: error: --minify and --obfuscate require --target=js") exit(1) } } let src_path: String = native_list_get(positional, 0) // When --emit-header is requested, lex the source file and do a // token-level signature scan (no full AST) to write a .elh file. // This avoids OOM on large files with HTML template bodies or deep // BinOp chains (e.g. checkout.el) — parse() builds O(whole-program AST) // while scan_fn_sigs_el keeps peak memory at O(tokens). if do_emit_header { el_mem_check() let raw_source: String = fs_read(src_path) let hdr_tokens: [Any] = lex(raw_source) let hdr_sigs: [Map] = scan_fn_sigs_el(hdr_tokens) el_release(hdr_tokens) let hdr_path: String = str_slice(src_path, 0, str_len(src_path) - 3) + ".elh" emit_header_from_sigs(hdr_sigs, hdr_path) el_release(hdr_sigs) } let source: String = resolve_imports(src_path) // When post-processing (--minify or --obfuscate) is requested, redirect // stdout to a temp file so codegen output can be captured and piped through // the external tools. After codegen, restore stdout before emitting the // final result. if do_minify { run_with_postprocess(tgt, source, src_path, do_bundle, do_obfuscate, argc, positional) exit(0) } // --test mode: compile with test harness (C target only). if do_test { compile_test(source) exit(0) } // Standard path (no post-processing). let out: String = "" if do_bundle { let runtime_path: String = resolve_runtime_path(src_path) let out = compile_dispatch_bundle(tgt, source, runtime_path) } else { let out = compile_dispatch(tgt, source) } if argc >= 2 { let out_path: String = native_list_get(positional, 1) let ok: Bool = fs_write(out_path, out) if ok { exit(0) } else { println("el-compiler: failed to write output") exit(1) } } // No output path: codegen streamed to stdout already; out is "". }