Files
el/lang/el-compiler/src/compiler.el
T
will.anderson 53e0b99d5f fix(elc): add el_mem_check() memory guard — abort before OS OOM-kill
Add el_mem_check() to el_runtime.c: reads ELC_MAX_MEM_MB (default 512),
checks RSS via getrusage (macOS bytes / Linux KB normalised to MB), prints
a clear diagnostic to stderr and exits(1) if exceeded.

Wire it into two places:
- compiler.el: upfront check at --emit-header entry point
- codegen.el: per-function check in the streaming loop after each
  el_arena_pop, so runaway growth is caught at the earliest function
  boundary rather than after the machine is already dying.
2026-05-08 08:21:38 -05:00

621 lines
25 KiB
EmacsLisp

// compiler.el el self-hosting compiler pipeline
//
// Wires lexer -> parser -> codegen into a single compile() function.
// This is the bootstrap entry point: compiled once by the Rust el-compiler,
// then self-hosted from that point forward.
//
// Two backends:
// - C (default) compile() -> emits C source linked against el_runtime.c
// - JS (--target=js) compile_js() -> emits JS source linked against el_runtime.js
//
// Compile the C output with:
// cc -o <prog> <prog>.c el_runtime.c
//
// Run the JS output with:
// node <prog>.js (after copying el_runtime.js next to it)
import "lexer.el"
import "parser.el"
import "codegen.el"
import "codegen-js.el"
// compile full pipeline (C target): source string -> C source string
// Uses JIT function-at-a-time streaming: parse one decl emit C discard AST.
// Peak memory is O(one function's AST) instead of O(whole program AST).
fn compile(source: String) -> String {
// Top-level arena scope: activates the string arena before lex() so that
// ALL strdup allocations (token strings, sig strings, codegen fragments)
// are tracked and freed on pop. Without this, lex() and scan_fn_sigs()
// run before any push, leaving _tl_arena_active=0 and leaking every
// token string. Also prevents inner pop(mark=0) calls from deactivating
// the arena between per-function scopes.
let top_mark: Any = el_arena_push()
let tokens: [Any] = lex(source)
// Fast pre-scan: collect fn signatures + program kind without building
// full expression ASTs. O(tokens) time, minimal allocation.
let sigs: [Map<String, Any>] = scan_fn_sigs(tokens)
// Stream parse-emit: parse one decl at a time, emit C, discard.
// All output written to stdout via println before pop.
codegen_streaming(tokens, sigs, source)
el_arena_pop(top_mark)
""
}
// compile_test like compile() but sets __test_mode so codegen_streaming
// compiles test { } blocks instead of skipping them, and emits the test
// harness main() instead of the normal int main().
fn compile_test(source: String) -> String {
state_set("__test_mode", "1")
let top_mark: Any = el_arena_push()
let tokens: [Any] = lex(source)
let sigs: [Map<String, Any>] = scan_fn_sigs(tokens)
codegen_streaming(tokens, sigs, source)
el_arena_pop(top_mark)
state_set("__test_mode", "")
""
}
// compile_js full pipeline (JS target, module mode): source string -> JS source string
fn compile_js(source: String) -> String {
let tokens: [Any] = lex(source)
let stmts: [Map<String, Any>] = parse(tokens)
// Token list is no longer needed after parsing release it to free memory.
el_release(tokens)
codegen_js(stmts, source)
}
// compile_js_with_bundle JS target in bundle mode.
// Reads el_runtime.js from runtime_path and inlines it inside an IIFE.
fn compile_js_with_bundle(source: String, runtime_path: String) -> String {
let tokens: [Any] = lex(source)
let stmts: [Map<String, Any>] = parse(tokens)
el_release(tokens)
let runtime_content: String = fs_read(runtime_path)
if str_eq(runtime_content, "") {
println("el-compiler: warning: --bundle: could not read runtime at " + runtime_path)
println("el-compiler: warning: bundle output will be incomplete")
}
codegen_js_bundle(stmts, source, runtime_content)
}
// compile_dispatch pick a backend based on the requested target.
// tgt = "c" | "js"
// (The parameter is named `tgt` because `target` is a reserved keyword
// in El's lexer it would be tokenised as `Target`, breaking the
// parser's identifier resolution.)
fn compile_dispatch(tgt: String, source: String) -> String {
if str_eq(tgt, "js") { return compile_js(source) }
compile(source)
}
// compile_dispatch_bundle like compile_dispatch but bundle mode for JS.
fn compile_dispatch_bundle(tgt: String, source: String, runtime_path: String) -> String {
if str_eq(tgt, "js") { return compile_js_with_bundle(source, runtime_path) }
compile(source)
}
// Detect a `--target=<lang>` flag in argv and return the target.
// Returns "c" if none specified or unrecognized.
fn detect_target(argv: [String]) -> String {
let n: Int = native_list_len(argv)
let i = 0
while i < n {
let a: String = native_list_get(argv, i)
if str_starts_with(a, "--target=") {
let v: String = str_slice(a, 9, str_len(a))
return v
}
let i = i + 1
}
return "c"
}
// Strip flags from argv, leaving only positional arguments.
fn strip_flags(argv: [String]) -> [String] {
let out: [String] = native_list_empty()
let n: Int = native_list_len(argv)
let i = 0
while i < n {
let a: String = native_list_get(argv, i)
if !str_starts_with(a, "--") {
let out = native_list_append(out, a)
}
let i = i + 1
}
return out
}
// Detect --emit-header flag in argv.
fn detect_emit_header(argv: [String]) -> Bool {
let n: Int = native_list_len(argv)
let i = 0
while i < n {
let a: String = native_list_get(argv, i)
if str_eq(a, "--emit-header") { return true }
let i = i + 1
}
return false
}
// Detect --bundle flag in argv.
fn detect_bundle(argv: [String]) -> Bool {
let n: Int = native_list_len(argv)
let i = 0
while i < n {
let a: String = native_list_get(argv, i)
if str_eq(a, "--bundle") { return true }
let i = i + 1
}
return false
}
// Detect --minify flag in argv.
fn detect_minify(argv: [String]) -> Bool {
let n: Int = native_list_len(argv)
let i = 0
while i < n {
let a: String = native_list_get(argv, i)
if str_eq(a, "--minify") { return true }
let i = i + 1
}
return false
}
// Detect --obfuscate flag in argv.
fn detect_obfuscate(argv: [String]) -> Bool {
let n: Int = native_list_len(argv)
let i = 0
while i < n {
let a: String = native_list_get(argv, i)
if str_eq(a, "--obfuscate") { return true }
let i = i + 1
}
return false
}
// Detect --test flag in argv.
fn detect_test(argv: [String]) -> Bool {
let n: Int = native_list_len(argv)
let i = 0
while i < n {
let a: String = native_list_get(argv, i)
if str_eq(a, "--test") { return true }
let i = i + 1
}
return false
}
// Build a unique temp file path: /tmp/elc-<pid>-<timestamp>.<suffix>
fn make_temp_path(suffix: String) -> String {
let pid: Int = getpid_now()
let ts: Int = time_now()
"/tmp/elc-" + native_int_to_str(pid) + "-" + native_int_to_str(ts) + "." + suffix
}
// Reserved globals that terser and javascript-obfuscator must not mangle.
// These are referenced from HTML onclick= attributes and other direct window usage.
fn js_reserved_names() -> String {
"neuronDemoToggle,neuronDemoSend,neuronDemoReset,signInWith,signInWithEmail,signUpWithEmail,sendMagicLink,signOut,resetPassword,sendResetEmail,updatePassword,showSignIn,showSignUp,hideReset,setSort,addFamilyMember,removeFamilyMember,copyForPlatform,entHeadcountChange,NEURON_CFG"
}
// Find a CLI tool by checking node_modules paths first, then falling back to npx.
// src_dir is the directory of the source file being compiled.
// Returns the command string to invoke the tool, or "" if not found.
fn find_node_tool(tool_name: String, src_dir: String) -> String {
// 1. Check ./node_modules/.bin/<tool> relative to source file
let cand1: String = src_dir + "/node_modules/.bin/" + tool_name
let check1: String = str_trim(exec_capture("test -x " + cand1 + " && echo yes 2>/dev/null"))
if str_eq(check1, "yes") { return cand1 }
// 2. Check ../node_modules/.bin/<tool> (monorepo layout)
let parent_dir: String = dirname_of(src_dir)
let cand2: String = parent_dir + "/node_modules/.bin/" + tool_name
let check2: String = str_trim(exec_capture("test -x " + cand2 + " && echo yes 2>/dev/null"))
if str_eq(check2, "yes") { return cand2 }
// 3. Fall back to npx if it is on PATH. npx will use the globally cached
// package or download on first use. Use --no to avoid auto-install if
// the package is not already cached; if that fails, try with --yes.
let npx_path: String = str_trim(exec_capture("which npx 2>/dev/null"))
if !str_eq(npx_path, "") { return "npx --yes " + tool_name }
return ""
}
// apply_minify run terser on js_path, write result to out_path.
// Returns true on success, false on failure.
fn apply_minify(js_path: String, out_path: String, src_dir: String) -> Bool {
let terser: String = find_node_tool("terser", src_dir)
if str_eq(terser, "") {
println("el-compiler: error: terser not found. Run 'npm install terser' in your project directory.")
return false
}
let names: String = js_reserved_names()
// Single-quote the mangle reserved list so the shell does not glob-expand
// the bracket expression. The compress options are safe without quoting.
let compress_opts: String = "passes=2,drop_console=false,drop_debugger=true"
let mangle_reserved: String = "'reserved=[" + names + "]'"
let cmd: String = terser + " " + js_path + " --compress " + compress_opts + " --mangle " + mangle_reserved + " --output " + out_path
let ret: Int = exec_command(cmd)
if ret == 0 { return true }
println("el-compiler: error: terser failed (exit " + native_int_to_str(ret) + ")")
return false
}
// apply_obfuscate run javascript-obfuscator on js_path, write result to out_path.
// Returns true on success, false on failure.
fn apply_obfuscate(js_path: String, out_path: String, src_dir: String) -> Bool {
let obfuscator: String = find_node_tool("javascript-obfuscator", src_dir)
if str_eq(obfuscator, "") {
println("el-compiler: error: javascript-obfuscator not found. Run 'npm install javascript-obfuscator' in your project directory.")
return false
}
let names: String = js_reserved_names()
let cmd: String = obfuscator + " " + js_path + " --output " + out_path + " --compact true --simplify true --string-array true --string-array-encoding base64 --string-array-threshold 0.75 --identifier-names-generator hexadecimal --rename-globals false --self-defending false --reserved-names " + names
let ret: Int = exec_command(cmd)
if ret == 0 { return true }
println("el-compiler: error: javascript-obfuscator failed (exit " + native_int_to_str(ret) + ")")
return false
}
// Resolve the runtime path for --bundle mode.
// Looks for el_runtime.js next to the source file first;
// if not found there, looks next to the elc binary itself.
// Returns "" if not found anywhere (caller emits a warning).
fn resolve_runtime_path(src_path: String) -> String {
let src_dir: String = dirname_of(src_path)
let candidate: String = src_dir + "/el_runtime.js"
let existing: String = fs_read(candidate)
if !str_eq(existing, "") {
return candidate
}
return ""
}
// Reconstruct an El type annotation string from a parsed type node.
fn type_node_to_el(t: Map<String, Any>) -> String {
let k: String = t["kind"]
if str_eq(k, "Simple") { return t["name"] }
if str_eq(k, "List") {
let inner: String = type_node_to_el(t["inner"])
return "[" + inner + "]"
}
if str_eq(k, "Map") {
let kt: String = type_node_to_el(t["key"])
let vt: String = type_node_to_el(t["val"])
return "Map<" + kt + ", " + vt + ">"
}
"Any"
}
// emit_header write a .elh file from parsed statements.
// Scans for FnDef nodes and emits 'extern fn' declarations.
// NOTE: This function requires the full AST. Prefer emit_header_from_sigs
// for the --emit-header path it works from a token-level scan without
// building expression ASTs, avoiding OOM on large files.
fn emit_header(stmts: [Map<String, Any>], hdr_path: String) -> Void {
let n: Int = native_list_len(stmts)
let i = 0
let parts: [String] = native_list_empty()
let parts = native_list_append(parts, "// auto-generated by elc --emit-header — do not edit\n")
while i < n {
let stmt = native_list_get(stmts, i)
let kind: String = stmt["stmt"]
if str_eq(kind, "FnDef") {
let name: String = stmt["name"]
if !str_eq(name, "main") {
let params = stmt["params"]
let ret_type: String = stmt["ret_type"]
// build param list
let np: Int = native_list_len(params)
let pi = 0
let param_parts: [String] = native_list_empty()
while pi < np {
let param = native_list_get(params, pi)
let pname: String = param["name"]
let ptype: String = param["type"]
if str_eq(ptype, "") { let ptype = "Any" }
let param_parts = native_list_append(param_parts, pname + ": " + ptype)
let pi = pi + 1
}
let params_str: String = str_join(param_parts, ", ")
let ret_str: String = ret_type
if str_eq(ret_str, "") { let ret_str = "Any" }
let sig: String = "extern fn " + name + "(" + params_str + ") -> " + ret_str
let parts = native_list_append(parts, sig + "\n")
}
}
let i = i + 1
}
let content: String = str_join(parts, "")
let ok: Bool = fs_write(hdr_path, content)
}
// emit_header_from_sigs write a .elh file from pre-scanned El signatures.
// Uses the output of scan_fn_sigs_el() no full AST required.
// Peak memory is O(tokens) rather than O(whole-program AST), which prevents
// OOM on large files with HTML template bodies or deep BinOp chains.
fn emit_header_from_sigs(sigs: [Map<String, Any>], hdr_path: String) -> Void {
let n: Int = native_list_len(sigs)
let i: Int = 0
let parts: [String] = native_list_empty()
let parts = native_list_append(parts, "// auto-generated by elc --emit-header — do not edit\n")
while i < n {
let sig = native_list_get(sigs, i)
let kind: String = sig["kind"]
if str_eq(kind, "fn") {
let name: String = sig["name"]
let params_el: String = sig["params_el"]
let ret_el: String = sig["ret_el"]
if str_eq(ret_el, "") { let ret_el = "Any" }
let line: String = "extern fn " + name + "(" + params_el + ") -> " + ret_el
let parts = native_list_append(parts, line + "\n")
}
let i = i + 1
}
let content: String = str_join(parts, "")
let ok: Bool = fs_write(hdr_path, content)
}
// Import resolution
//
// elc supports two forms of import:
// import "path/to/file.el" quoted relative path
// from module import { Name } bare module name resolves to module.el
// in the entry source's directory
//
// Codegen treats Import statements as no-ops (declarations only), so to
// actually link bodies across files we textually concatenate every imported
// source ahead of the entry source before lex/parse. resolve_imports does a
// depth-first traversal with deduplication so any module that gets pulled in
// transitively is included exactly once.
fn dirname_of(path: String) -> String {
let n: Int = str_len(path)
let i: Int = n - 1
while i >= 0 {
let c: String = str_slice(path, i, i + 1)
if str_eq(c, "/") {
return str_slice(path, 0, i)
}
let i = i - 1
}
return "."
}
// Extract the resolved file path from a single trimmed source line. Returns
// "" if the line is not an import.
fn parse_import_line(trimmed: String, dir: String) -> String {
if str_starts_with(trimmed, "import \"") {
let after: String = str_slice(trimmed, 8, str_len(trimmed))
let q: Int = str_index_of(after, "\"")
if q > 0 {
let mod: String = str_slice(after, 0, q)
return dir + "/" + mod
}
}
if str_starts_with(trimmed, "from ") {
let after: String = str_slice(trimmed, 5, str_len(trimmed))
// module name is the first whitespace-delimited token
let sp: Int = str_index_of(after, " ")
if sp > 0 {
let mod_raw: String = str_slice(after, 0, sp)
let mod: String = str_trim(mod_raw)
if !str_eq(mod, "") {
return dir + "/" + mod + ".el"
}
}
}
return ""
}
// Recursively resolve imports starting from src_path. Returns the combined
// source text with every imported module's body inlined ahead of the entry
// source, deduplicated by absolute path. Uses state_set to track which paths
// have already been pulled in for this run.
//
// Accumulates chunks into lists and joins once at the end to avoid the O(n²)
// memory growth caused by repeated `prefix = prefix + chunk` concatenation.
fn resolve_imports(src_path: String) -> String {
let seen_key: String = "__elc_imp__:" + src_path
let already: String = state_get(seen_key)
if !str_eq(already, "") { return "" }
state_set(seen_key, "1")
let source: String = fs_read(src_path)
let dir: String = dirname_of(src_path)
let lines: [String] = str_split(source, "\n")
let n: Int = native_list_len(lines)
// Collect chunks into lists O(1) amortized per append.
// Join once at the end O(n) single pass.
let prefix_chunks: [String] = native_list_empty()
let body_chunks: [String] = native_list_empty()
let i: Int = 0
while i < n {
let line: String = native_list_get(lines, i)
let trimmed: String = str_trim(line)
let imp_path: String = parse_import_line(trimmed, dir)
if !str_eq(imp_path, "") {
// Use pre-compiled header if available (separate compilation).
// Only check .elh for imported files never for the entry file itself.
let imp_elh_path: String = str_slice(imp_path, 0, str_len(imp_path) - 3) + ".elh"
let imp_elh: String = fs_read(imp_elh_path)
if !str_eq(imp_elh, "") {
// Header exists: mark the .el as seen (so it won't be re-inlined
// if something else also imports it) and use the header text.
let seen_imp_key: String = "__elc_imp__:" + imp_path
state_set(seen_imp_key, "1")
let prefix_chunks = native_list_append(prefix_chunks, imp_elh)
} else {
let imp_body: String = resolve_imports(imp_path)
let prefix_chunks = native_list_append(prefix_chunks, imp_body)
}
} else {
let body_chunks = native_list_append(body_chunks, line + "\n")
}
let i = i + 1
}
return str_join(prefix_chunks, "") + str_join(body_chunks, "")
}
// run_with_postprocess codegen + minify + optional obfuscate pipeline.
//
// Called from main() when --minify or --obfuscate is active. Redirects stdout
// to a temp file during codegen so the output can be passed through the
// external tools (terser, javascript-obfuscator) before final emission.
//
// Pipeline: codegen -> terser -> (javascript-obfuscator) -> stdout or file
fn run_with_postprocess(tgt: String, source: String, src_path: String, do_bundle: Bool, do_obfuscate: Bool, argc: Int, positional: [String]) -> Void {
let src_dir: String = dirname_of(src_path)
let tmp_gen: String = make_temp_path("js")
let tmp_min: String = make_temp_path("min.js")
// Redirect stdout to tmp_gen so codegen println output is captured.
stdout_to_file(tmp_gen)
if do_bundle {
let runtime_path: String = resolve_runtime_path(src_path)
compile_dispatch_bundle(tgt, source, runtime_path)
} else {
compile_dispatch(tgt, source)
}
stdout_restore()
// Run terser: tmp_gen -> tmp_min
let ok_min: Bool = apply_minify(tmp_gen, tmp_min, src_dir)
if !ok_min {
exec_command("rm -f " + tmp_gen + " " + tmp_min)
exit(1)
}
// Determine final result path (either tmp_min or post-obfuscation file).
// Use state to pass the final path out of the optional obfuscation branch.
state_set("__elc_final_js", tmp_min)
if do_obfuscate {
let tmp_obf: String = make_temp_path("obf.js")
let ok_obf: Bool = apply_obfuscate(tmp_min, tmp_obf, src_dir)
if !ok_obf {
exec_command("rm -f " + tmp_gen + " " + tmp_min + " " + tmp_obf)
exit(1)
}
state_set("__elc_final_js", tmp_obf)
}
let final_path: String = state_get("__elc_final_js")
let final_js: String = fs_read(final_path)
// Clean up all temp files.
exec_command("rm -f " + tmp_gen + " " + tmp_min)
if do_obfuscate {
exec_command("rm -f " + final_path)
}
if argc >= 2 {
let out_path: String = native_list_get(positional, 1)
let ok: Bool = fs_write(out_path, final_js)
if ok {
return
} else {
println("el-compiler: failed to write output")
exit(1)
}
}
// No output file: print final JS to stdout.
print(final_js)
}
// main CLI entry point.
//
// elc <source.el> # emit C to stdout
// elc --target=js <source.el> # emit JS (module) to stdout
// elc --target=js --bundle <source.el> # emit self-contained JS (IIFE) to stdout
// elc --target=js --bundle --minify <source.el> # emit minified IIFE to stdout
// elc --target=js --bundle --obfuscate <source.el> # emit minified+obfuscated IIFE to stdout
// elc --target=c <source.el> <out.c> # write C to file
// elc --target=js <source.el> <out.js> # write JS to file
// elc --target=js --bundle <source.el> <out.js> # write bundled JS to file
// elc --target=js --bundle --minify <source.el> <out.min.js> # write minified JS to file
fn main() -> Void {
let argv: [String] = args()
// Use `tgt` not `target`: `target` is a reserved keyword in the lexer
// (Section 1.5 of the language spec). detect_target itself is fine
// because the function-name position has no token-class restriction.
let tgt: String = detect_target(argv)
let do_emit_header: Bool = detect_emit_header(argv)
let do_bundle: Bool = detect_bundle(argv)
let do_minify: Bool = detect_minify(argv)
let do_obfuscate: Bool = detect_obfuscate(argv)
let do_test: Bool = detect_test(argv)
// --obfuscate implies --minify: obfuscating unminified code is pointless.
if do_obfuscate {
let do_minify = true
}
let positional: [String] = strip_flags(argv)
let argc: Int = native_list_len(positional)
if argc < 1 {
println("el-compiler: usage: elc [--target=c|js] [--bundle] [--minify] [--obfuscate] [--emit-header] [--test] <source.el> [<output>]")
exit(1)
}
// --minify and --obfuscate require --target=js
if do_minify {
if !str_eq(tgt, "js") {
println("el-compiler: error: --minify and --obfuscate require --target=js")
exit(1)
}
}
let src_path: String = native_list_get(positional, 0)
// When --emit-header is requested, lex the source file and do a
// token-level signature scan (no full AST) to write a .elh file.
// This avoids OOM on large files with HTML template bodies or deep
// BinOp chains (e.g. checkout.el) parse() builds O(whole-program AST)
// while scan_fn_sigs_el keeps peak memory at O(tokens).
if do_emit_header {
el_mem_check()
let raw_source: String = fs_read(src_path)
let hdr_tokens: [Any] = lex(raw_source)
let hdr_sigs: [Map<String, Any>] = scan_fn_sigs_el(hdr_tokens)
el_release(hdr_tokens)
let hdr_path: String = str_slice(src_path, 0, str_len(src_path) - 3) + ".elh"
emit_header_from_sigs(hdr_sigs, hdr_path)
el_release(hdr_sigs)
}
let source: String = resolve_imports(src_path)
// When post-processing (--minify or --obfuscate) is requested, redirect
// stdout to a temp file so codegen output can be captured and piped through
// the external tools. After codegen, restore stdout before emitting the
// final result.
if do_minify {
run_with_postprocess(tgt, source, src_path, do_bundle, do_obfuscate, argc, positional)
exit(0)
}
// --test mode: compile with test harness (C target only).
if do_test {
compile_test(source)
exit(0)
}
// Standard path (no post-processing).
let out: String = ""
if do_bundle {
let runtime_path: String = resolve_runtime_path(src_path)
let out = compile_dispatch_bundle(tgt, source, runtime_path)
} else {
let out = compile_dispatch(tgt, source)
}
if argc >= 2 {
let out_path: String = native_list_get(positional, 1)
let ok: Bool = fs_write(out_path, out)
if ok {
exit(0)
} else {
println("el-compiler: failed to write output")
exit(1)
}
}
// No output path: codegen streamed to stdout already; out is "".
}