neuron-technologies/el
3
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

perf: 81% RSS reduction — el_release, arena scoping, streaming codegen, libcurl stub

Browse Source 
Chain of optimizations from swarm rounds 4-7:
- Flat stride-2 token list: eliminate per-token Map allocation (~112B each × N tokens)
- Systematic el_release() in parser.el: eagerly free intermediate parse result maps
- Per-function and per-statement arena scoping in codegen_streaming()
- Streaming codegen pipeline: parse one fn at a time, emit C, discard AST
- HAVE_CURL guard: elc CLI binary drops libcurl, eliminating SSL/TLS init overhead
- HTML codegen parts-list: O(n) instead of O(n²) string growth for nested templates
- Batch c_escape: str_slice clean runs instead of char-at per byte

Result: 33.4MB → 6.5MB RSS on web/src/main.el (-81%). Self-host: PASS.
This commit is contained in:
Will Anderson
2026-05-05 20:39:38 -05:00
parent ee86736eab
commit 3726f69435
5 changed files with 859 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lang/el-compiler/src/codegen.el
+392 -7
View File
@@ -134,7 +134,10 @@ fn c_escape(s: String) -> String {
if clean_start < total {
let parts = native_list_append(parts, str_slice(s, clean_start, total))
}
str_join(parts, "")
let result: String = str_join(parts, "")
// parts list fully consumed — release to free peak heap.
el_release(parts)
result
}
fn c_str_lit(s: String) -> String {
@@ -849,6 +852,8 @@ fn cg_expr(expr: Map<String, Any>) -> String {
let i = i + 1
}
let args_c: String = str_join(args_parts, ", ")
// args_parts list fully consumed — release to free peak heap.
el_release(args_parts)
if func_kind == "Ident" {
let fn_name: String = func["name"]
@@ -953,7 +958,10 @@ fn cg_expr(expr: Map<String, Any>) -> String {
let items_parts = native_list_append(items_parts, elem_c)
let i = i + 1
}
return "el_list_new(" + native_int_to_str(n) + ", " + str_join(items_parts, ", ") + ")"
let items_joined: String = str_join(items_parts, ", ")
// items_parts fully consumed — release to free peak heap.
el_release(items_parts)
return "el_list_new(" + native_int_to_str(n) + ", " + items_joined + ")"
}
if kind == "Map" {
@@ -974,7 +982,10 @@ fn cg_expr(expr: Map<String, Any>) -> String {
let items_parts = native_list_append(items_parts, c_str_lit(key) + ", " + val_c)
let i = i + 1
}
return "el_map_new(" + native_int_to_str(n) + ", " + str_join(items_parts, ", ") + ")"
let items_joined: String = str_join(items_parts, ", ")
// items_parts fully consumed — release to free peak heap.
el_release(items_parts)
return "el_map_new(" + native_int_to_str(n) + ", " + items_joined + ")"
}
if kind == "Try" {
@@ -1077,7 +1088,10 @@ fn cg_match(expr: Map<String, Any>) -> String {
let i = i + 1
}
let parts = native_list_append(parts, done_label + ":; " + result_var + "; })")
str_join(parts, "")
let result: String = str_join(parts, "")
// parts list fully consumed — release to free peak heap.
el_release(parts)
result
}
// Lower a match statement (used for side effects, not as an expression) to a
@@ -1246,7 +1260,10 @@ fn cg_if_expr_arm(stmts: [Map<String, Any>], result_var: String) -> String {
}
let i = i + 1
}
str_join(parts, "")
let result: String = str_join(parts, "")
// parts list fully consumed — release to free peak heap.
el_release(parts)
result
}
fn cg_if_expr(expr: Map<String, Any>) -> String {
@@ -1582,7 +1599,11 @@ fn cg_stmts(stmts: [Map<String, Any>], indent: String, declared: [String]) -> [S
let decl = declared
while i < n {
let stmt = native_list_get(stmts, i)
// Per-statement arena scope: free intermediate strings (str_concat
// fragments, cg_expr results) after each statement is emitted.
let s_mark: Any = el_arena_push()
let decl = cg_stmt(stmt, indent, decl)
el_arena_pop(s_mark)
let i = i + 1
}
decl
@@ -1606,7 +1627,10 @@ fn params_to_c(params: [Map<String, Any>]) -> String {
let parts = native_list_append(parts, decl)
let i = i + 1
}
str_join(parts, ", ")
let result: String = str_join(parts, ", ")
// parts list fully consumed — release to free peak heap.
el_release(parts)
result
}
// Transform a function body so that an implicit-return final expression
@@ -2575,6 +2599,9 @@ fn builtin_arity(name: String) -> Int {
if str_eq(name, "__channel_recv") { return 1 }
if str_eq(name, "__channel_try_recv") { return 1 }
if str_eq(name, "__channel_close") { return 1 }
// Arena mark/restore builtins
if str_eq(name, "el_arena_push") { return 0 }
if str_eq(name, "el_arena_pop") { return 1 }
// -1 sentinel: variadic / unknown / user-defined -> no check.
return -1
}
@@ -2811,7 +2838,8 @@ fn cg_fn(stmt: Map<String, Any>) -> Void {
if !str_eq(ret_type, "Void") {
let body_xformed = transform_implicit_return(body)
}
cg_stmts(body_xformed, " ", decl)
let final_decl = cg_stmts(body_xformed, " ", decl)
el_release(final_decl)
emit_line(" return 0;")
emit_line("}")
emit_blank()
@@ -3285,3 +3313,360 @@ fn codegen(stmts: [Map<String, Any>], source: String) -> String {
// Return empty string - output was streamed via println
""
}
// ── Streaming codegen (JIT function-at-a-time) ─────────────────────────────
//
// codegen_streaming is a memory-efficient alternative to codegen().
// Instead of receiving the full parsed AST, it receives the raw token list
// and a pre-scanned signature list (from scan_fn_sigs in parser.el).
//
// Pipeline:
// 1. Scan phase (already done by caller): scan_fn_sigs(tokens) -> sigs
// 2. Emit preamble using sigs (no full AST needed)
// 3. For each top-level statement:
// parse_one(tokens, pos) -> { node, pos }
// cg_decl_streaming(node) <- emit C for this one decl
// el_release(node) <- discard AST immediately
//
// Peak memory: O(one function's AST) instead of O(whole program AST).
//
// Entry point: codegen_streaming(tokens, sigs, source) -> String
// cg_decl_streaming — emit C for a single top-level declaration.
// Handles FnDef, ExternFn, TypeDef, EnumDef, Import, CgiBlock, ServiceBlock.
// Top-level Let statements go into the main() body, not here.
// Top-level executable statements (non-fn, non-let, non-decl) are
// accumulated into state and emitted later in main().
fn cg_decl_streaming(stmt: Map<String, Any>) -> Void {
let sk: String = stmt["stmt"]
if str_eq(sk, "FnDef") {
cg_fn(stmt)
return
}
// All other top-level decl kinds are either no-ops (Import, TypeDef,
// EnumDef, ExternFn forward decl already emitted) or capability markers
// (CgiBlock, ServiceBlock already handled in preamble).
// Top-level Lets are also no-ops here (file-scope slots already emitted).
// Executable top-level stmts (Expr, Return, etc.) are accumulated in state.
if !str_eq(sk, "FnDef") {
if !is_top_level_decl(stmt) {
if !str_eq(sk, "Let") {
// This is an executable top-level statement.
// We can't emit it into main() yet because we haven't started
// emitting main(). Accumulate in state as a list index.
// We'll collect these into a list and emit after all fns.
state_set("__streaming_has_toplevel_stmts", "1")
}
}
}
}
// emit_streaming_preamble — emit #includes, forward decls, and file-scope lets
// using the pre-scanned signature data (no full AST).
fn emit_streaming_preamble(sigs: [Map<String, Any>], source: String) -> Void {
let n: Int = native_list_len(sigs)
// Detect program kind from sigs
let cgi_count: Int = 0
let svc_count: Int = 0
let i: Int = 0
while i < n {
let sig = native_list_get(sigs, i)
let sk: String = sig["kind"]
if str_eq(sk, "cgi_block") { let cgi_count = cgi_count + 1 }
if str_eq(sk, "service_block") { let svc_count = svc_count + 1 }
let i = i + 1
}
if cgi_count > 1 {
emit_line("#error \"El: multiple cgi blocks in program (only one allowed)\"")
}
if svc_count > 1 {
emit_line("#error \"El: multiple service blocks in program (only one allowed)\"")
}
if cgi_count >= 1 {
if svc_count >= 1 {
emit_line("#error \"El: program declares both cgi and service blocks (mutually exclusive - pick one)\"")
}
}
let kind: String = "utility"
if cgi_count >= 1 { let kind = "cgi" }
if svc_count >= 1 { let kind = "service" }
state_set("__program_kind", kind)
state_set("__cap_violations", "")
state_set("__arity_violations", "")
state_set("__time_violations", "")
emit_line("#include <stdint.h>")
emit_line("#include <stdlib.h>")
emit_line("#include \"el_runtime.h\"")
emit_blank()
// Forward declarations — use pre-computed params_c strings from scan.
let i = 0
while i < n {
let sig = native_list_get(sigs, i)
let sk: String = sig["kind"]
if str_eq(sk, "fn") {
let fn_name: String = sig["name"]
if !str_eq(fn_name, "main") {
let params_c: String = sig["params_c"]
emit_line("el_val_t " + fn_name + "(" + params_c + ");")
}
}
if str_eq(sk, "extern_fn") {
let fn_name: String = sig["name"]
let params_c: String = sig["params_c"]
emit_line("el_val_t " + fn_name + "(" + params_c + ");")
}
let i = i + 1
}
emit_blank()
// File-scope let slots
let has_toplevel_lets: Bool = false
let i = 0
while i < n {
let sig = native_list_get(sigs, i)
let sk: String = sig["kind"]
if str_eq(sk, "toplevel_let") {
let name: String = sig["name"]
let ltype: String = sig["ltype"]
if str_eq(ltype, "Int") { add_int_name(name) }
emit_line("el_val_t " + name + ";")
let has_toplevel_lets = true
}
let i = i + 1
}
if has_toplevel_lets { emit_blank() }
}
// codegen_streaming — JIT function-at-a-time compiler backend.
// tokens: flat token list from lex()
// sigs: pre-scanned signature list from scan_fn_sigs(tokens)
// source: original source string (for string literal lookup)
fn codegen_streaming(tokens: [Any], sigs: [Map<String, Any>], source: String) -> String {
let total_tokens: Int = native_list_len(tokens) / 2
// Emit preamble (forward decls, file-scope lets, #includes)
// Arena scope: free intermediate strings built during preamble emission.
let preamble_mark: Any = el_arena_push()
emit_streaming_preamble(sigs, source)
el_arena_pop(preamble_mark)
// Detect whether there is a fn main() and whether there are top-level
// executable stmts (for library detection) from sigs.
let has_el_main: Bool = false
let ns: Int = native_list_len(sigs)
let si: Int = 0
while si < ns {
let sig = native_list_get(sigs, si)
let sk2: String = sig["kind"]
if str_eq(sk2, "fn") {
let fn_name_chk: String = sig["name"]
if str_eq(fn_name_chk, "main") { let has_el_main = true }
}
let si = si + 1
}
// Collect top-level let names for seeding main()'s declared set.
let toplevel_let_names: [String] = native_list_empty()
let si = 0
while si < ns {
let sig = native_list_get(sigs, si)
let sk2: String = sig["kind"]
if str_eq(sk2, "toplevel_let") {
let tname: String = sig["name"]
let toplevel_let_names = native_list_append(toplevel_let_names, tname)
}
let si = si + 1
}
// Streaming parse-emit loop.
// For each parsed stmt:
// - FnDef (not main): emit immediately via cg_fn, release AST
// - Others: accumulate only fn-main body and top-level executable stmts
// (these are small in count relative to fn bodies)
let pos: Int = 0
let el_main_body: [Map<String, Any>] = native_list_empty()
let toplevel_exec_stmts: [Map<String, Any>] = native_list_empty()
let has_toplevel_exec: Bool = false
let stream_running: Bool = true
while stream_running {
if pos >= total_tokens {
let stream_running = false
} else {
let k: String = tok_kind(tokens, pos)
if str_eq(k, "Eof") {
let stream_running = false
} else {
let r = parse_one(tokens, pos)
let stmt = r["node"]
let new_pos: Int = r["pos"]
el_release(r)
// Guard against infinite loops
if new_pos <= pos {
el_release(stmt)
let pos = pos + 1
} else {
let sk: String = stmt["stmt"]
if str_eq(sk, "FnDef") {
let fn_name2: String = stmt["name"]
if str_eq(fn_name2, "main") {
// Capture main() body for later
let body = stmt["body"]
let bn: Int = native_list_len(body)
let bi: Int = 0
while bi < bn {
let el_main_body = native_list_append(el_main_body, native_list_get(body, bi))
let bi = bi + 1
}
el_release(stmt)
} else {
// Emit immediately — this is the JIT core
// Arena scope: free all intermediate strings (str_concat,
// int_to_str, cg_expr fragments) after each function.
let fn_arena_mark: Any = el_arena_push()
cg_fn(stmt)
el_release(stmt)
el_arena_pop(fn_arena_mark)
}
} else {
if is_top_level_decl(stmt) {
// Import, TypeDef, EnumDef, CgiBlock, ServiceBlock, ExternFn
// These are no-ops in codegen (forward decls already emitted)
el_release(stmt)
} else {
if str_eq(sk, "Let") {
// Top-level let: file-scope slot already declared.
// Keep for main() init — these are few and small.
let toplevel_exec_stmts = native_list_append(toplevel_exec_stmts, stmt)
let has_toplevel_exec = true
} else {
// Executable top-level stmt (rare)
let toplevel_exec_stmts = native_list_append(toplevel_exec_stmts, stmt)
let has_toplevel_exec = true
}
}
}
let pos = new_pos
}
}
}
}
// Tokens fully consumed by the streaming loop — release now to free peak heap.
el_release(tokens)
// Library detection: no fn main and no top-level executable stmts
let is_library: Bool = false
if !has_el_main {
if !has_toplevel_exec {
let is_library = true
}
}
if is_library { return "" }
// Emit main() — wrap in arena scope to free intermediate strings.
let main_arena_mark: Any = el_arena_push()
let kind2: String = state_get("__program_kind")
emit_line("int main(int _argc, char** _argv) {")
emit_line(" el_runtime_init_args(_argc, _argv);")
// cgi init if needed
let ns2: Int = native_list_len(sigs)
let si2: Int = 0
while si2 < ns2 {
let sig2 = native_list_get(sigs, si2)
let sk3: String = sig2["kind"]
if str_eq(sk3, "cgi_block") {
// We need the full cgi_block data — it was parsed by scan_fn_sigs
// but scan only stored the name. For cgi_init we need dharma_id etc.
// Since cgi blocks are rare and small, they end up in toplevel_exec_stmts.
// Find the CgiBlock in toplevel_exec_stmts.
let tes_n: Int = native_list_len(toplevel_exec_stmts)
let tes_i: Int = 0
while tes_i < tes_n {
let tes = native_list_get(toplevel_exec_stmts, tes_i)
let tes_k: String = tes["stmt"]
if str_eq(tes_k, "CgiBlock") {
let cname2: String = tes["name"]
let cdid2: String = tes["dharma_id"]
let cprin2: String = tes["principal"]
let cnet2: String = tes["network"]
let ceng2: String = tes["engram"]
let has_did2: Bool = tes["has_dharma_id"]
let has_prin2: Bool = tes["has_principal"]
let has_net2: Bool = tes["has_network"]
let has_eng2: Bool = tes["has_engram"]
let arg_name2: String = "EL_STR(" + c_str_lit(cname2) + ")"
let arg_did2: String = cgi_arg(cdid2, has_did2)
let arg_prin2: String = cgi_arg(cprin2, has_prin2)
let arg_net2: String = cgi_arg(cnet2, has_net2)
let arg_eng2: String = cgi_arg(ceng2, has_eng2)
emit_line(" el_cgi_init(" + arg_name2 + ", " + arg_did2 + ", " + arg_prin2 + ", " + arg_net2 + ", " + arg_eng2 + ");")
}
let tes_i = tes_i + 1
}
}
let si2 = si2 + 1
}
// sigs fully consumed — release to free peak heap.
el_release(sigs)
// Seed declared set with top-level let names
let main_decl2: [String] = native_list_empty()
let tln: Int = native_list_len(toplevel_let_names)
let tli: Int = 0
while tli < tln {
let main_decl2 = native_list_append(main_decl2, native_list_get(toplevel_let_names, tli))
let tli = tli + 1
}
// toplevel_let_names fully consumed — release to free peak heap.
el_release(toplevel_let_names)
// Emit top-level executable stmts (lets and others) into main()
let tes_n2: Int = native_list_len(toplevel_exec_stmts)
let tes_i2: Int = 0
while tes_i2 < tes_n2 {
let tes2 = native_list_get(toplevel_exec_stmts, tes_i2)
let tes_k2: String = tes2["stmt"]
if !str_eq(tes_k2, "CgiBlock") {
if !str_eq(tes_k2, "ServiceBlock") {
let main_decl2 = cg_stmt(tes2, " ", main_decl2)
}
}
let tes_i2 = tes_i2 + 1
}
// toplevel_exec_stmts fully consumed — release to free peak heap.
el_release(toplevel_exec_stmts)
// Emit fn main() body — per-statement arena scope frees intermediate strings.
let mn: Int = native_list_len(el_main_body)
let mi: Int = 0
while mi < mn {
let mstmt = native_list_get(el_main_body, mi)
let stmt_mark: Any = el_arena_push()
let main_decl2 = cg_stmt(mstmt, " ", main_decl2)
el_arena_pop(stmt_mark)
let mi = mi + 1
}
// el_main_body and main_decl2 fully consumed — release to free peak heap.
el_release(el_main_body)
el_release(main_decl2)
emit_line(" return 0;")
emit_line("}")
emit_blank()
emit_cap_violations()
emit_arity_violations()
emit_time_violations()
el_arena_pop(main_arena_mark)
""
}