fix: skip test blocks in codegen to prevent OOM on test files
test "name" { ... } blocks were not recognized by the self-hosted
compiler. The body { } was parsed as a Map literal, creating a huge
AST with O(n²) string concatenation in the toplevel_exec_stmts loop
(which had no arena scope). A 272-line test file would consume 400MB+
and a 720-line file importing the full compiler source caused 150GB
usage and crashed the machine.
Two fixes:
1. Skip Test tokens in codegen_streaming before parse_one() —
advance past "name" and skip_to_rbrace on the body block.
Test blocks are never compiled; self-hosted compiler has no test runner.
2. Add per-statement arena scope to toplevel_exec_stmts emission loop,
matching the el_main_body loop. Frees intermediate strings after
each statement to prevent O(n²) accumulation from any unrecognized
construct that reaches that path.
Result: test_string.el (272 lines, 27 test blocks): 0MB peak (was 400MB+).
test_compiler.el (720 lines + 8728 imported): 15MB peak (was 150GB).
This commit is contained in:
Vendored
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
@@ -3500,6 +3500,19 @@ fn codegen_streaming(tokens: [Any], sigs: [Map<String, Any>], source: String) ->
|
||||
if str_eq(k, "Eof") {
|
||||
let stream_running = false
|
||||
} else {
|
||||
if str_eq(k, "Test") {
|
||||
// Skip test "name" { ... } blocks entirely.
|
||||
// The self-hosted compiler does not implement test execution.
|
||||
// Without this skip, the body `{ ... }` would be parsed as a Map
|
||||
// literal, building a huge AST with O(n²) string allocation that
|
||||
// causes OOM on any file containing test blocks.
|
||||
let p: Int = pos + 1
|
||||
let k_name: String = tok_kind(tokens, p)
|
||||
if str_eq(k_name, "Str") { let p = p + 1 }
|
||||
let k_body: String = tok_kind(tokens, p)
|
||||
if str_eq(k_body, "LBrace") { let p = skip_to_rbrace(tokens, p) }
|
||||
let pos = p
|
||||
} else {
|
||||
let r = parse_one(tokens, pos)
|
||||
let stmt = r["node"]
|
||||
let new_pos: Int = r["pos"]
|
||||
@@ -3554,6 +3567,7 @@ fn codegen_streaming(tokens: [Any], sigs: [Map<String, Any>], source: String) ->
|
||||
|
||||
let pos = new_pos
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3630,6 +3644,9 @@ fn codegen_streaming(tokens: [Any], sigs: [Map<String, Any>], source: String) ->
|
||||
el_release(toplevel_let_names)
|
||||
|
||||
// Emit top-level executable stmts (lets and others) into main()
|
||||
// Per-statement arena scope mirrors el_main_body: frees intermediate strings
|
||||
// (str_concat fragments from cg_expr) after each statement, preventing O(n²)
|
||||
// accumulation when many stmts are present (e.g. from unrecognized constructs).
|
||||
let tes_n2: Int = native_list_len(toplevel_exec_stmts)
|
||||
let tes_i2: Int = 0
|
||||
while tes_i2 < tes_n2 {
|
||||
@@ -3637,7 +3654,9 @@ fn codegen_streaming(tokens: [Any], sigs: [Map<String, Any>], source: String) ->
|
||||
let tes_k2: String = tes2["stmt"]
|
||||
if !str_eq(tes_k2, "CgiBlock") {
|
||||
if !str_eq(tes_k2, "ServiceBlock") {
|
||||
let tes_mark: Any = el_arena_push()
|
||||
let main_decl2 = cg_stmt(tes2, " ", main_decl2)
|
||||
el_arena_pop(tes_mark)
|
||||
}
|
||||
}
|
||||
let tes_i2 = tes_i2 + 1
|
||||
|
||||
Reference in New Issue
Block a user