diff --git a/lang/dist/platform/elc b/lang/dist/platform/elc index 1745eac..c972eb6 100755 Binary files a/lang/dist/platform/elc and b/lang/dist/platform/elc differ diff --git a/lang/el-compiler/runtime/el_runtime.c b/lang/el-compiler/runtime/el_runtime.c index 63c9f82..9df9847 100644 --- a/lang/el-compiler/runtime/el_runtime.c +++ b/lang/el-compiler/runtime/el_runtime.c @@ -42,7 +42,9 @@ #include #include #include +#ifdef HAVE_CURL #include +#endif /* ── Internal allocators ─────────────────────────────────────────────────── */ @@ -739,6 +741,39 @@ struct JsonParser { * the loop is observable. */ +/* ── JSON error helper (used by HTTP, PQ, crypto stubs) ─────────────────── */ + +/* JSON-escape an arbitrary C string into an allocated buffer. */ +static char* json_escape_alloc(const char* s) { + if (!s) return el_strdup(""); + JsonBuf b; jb_init(&b); + for (const char* p = s; *p; p++) { + unsigned char c = (unsigned char)*p; + switch (c) { + case '"': jb_puts(&b, "\\\""); break; + case '\\': jb_puts(&b, "\\\\"); break; + case '\n': jb_puts(&b, "\\n"); break; + case '\r': jb_puts(&b, "\\r"); break; + case '\t': jb_puts(&b, "\\t"); break; + default: + if (c < 0x20) { + char tmp[8]; snprintf(tmp, sizeof(tmp), "\\u%04x", c); + jb_puts(&b, tmp); + } else jb_putc(&b, (char)c); + } + } + return b.buf; +} + +static el_val_t http_error_json(const char* msg) { + char* esc = json_escape_alloc(msg ? msg : "unknown error"); + char* buf = el_strbuf(strlen(esc) + 16); + sprintf(buf, "{\"error\":\"%s\"}", esc); + free(esc); + return el_wrap_str(buf); +} + +#ifdef HAVE_CURL /* ── HTTP client write-callback buffer ───────────────────────────────────── */ typedef struct { @@ -772,36 +807,6 @@ static size_t http_write_cb(char* ptr, size_t size, size_t nmemb, void* ud) { return n; } -/* JSON-escape an arbitrary C string into an allocated buffer. */ -static char* json_escape_alloc(const char* s) { - if (!s) return el_strdup(""); - JsonBuf b; jb_init(&b); - for (const char* p = s; *p; p++) { - unsigned char c = (unsigned char)*p; - switch (c) { - case '"': jb_puts(&b, "\\\""); break; - case '\\': jb_puts(&b, "\\\\"); break; - case '\n': jb_puts(&b, "\\n"); break; - case '\r': jb_puts(&b, "\\r"); break; - case '\t': jb_puts(&b, "\\t"); break; - default: - if (c < 0x20) { - char tmp[8]; snprintf(tmp, sizeof(tmp), "\\u%04x", c); - jb_puts(&b, tmp); - } else jb_putc(&b, (char)c); - } - } - return b.buf; -} - -static el_val_t http_error_json(const char* msg) { - char* esc = json_escape_alloc(msg ? msg : "unknown error"); - char* buf = el_strbuf(strlen(esc) + 16); - sprintf(buf, "{\"error\":\"%s\"}", esc); - free(esc); - return el_wrap_str(buf); -} - /* HTTP timeout (ms) — read once from EL_HTTP_TIMEOUT_MS, default 60000. * Applied via CURLOPT_TIMEOUT_MS on every libcurl request. */ static long _el_http_timeout_ms = -1; @@ -1009,6 +1014,7 @@ el_val_t http_post_to_file(el_val_t url, el_val_t body, el_val_t headers_map, el if (h) curl_slist_free_all(h); return r; } +#endif /* HAVE_CURL */ /* ── HTTP server (POSIX sockets + pthreads) ──────────────────────────────── */ @@ -7586,6 +7592,7 @@ el_val_t engram_query_range(el_val_t start_ms_v, el_val_t end_ms_v) { return el_wrap_str(b.buf); } +#ifdef HAVE_CURL /* ── DHARMA network ───────────────────────────────────────────────────────── * Real implementation. Peers are addressed by `dharma_id` — either bare * (e.g. "ntn-genesis", transport defaults to http://localhost:7770) or @@ -8097,6 +8104,7 @@ el_val_t dharma_peers(void) { free(peers); return out; } +#endif /* HAVE_CURL — DHARMA network */ /* ── Batch 4: LLM (Anthropic API client) ─────────────────────────────────── */ /* @@ -8111,6 +8119,7 @@ el_val_t dharma_peers(void) { * and returns a JSON-string el_val_t result. Iteration is capped at 10. */ +#ifdef HAVE_CURL static const char* LLM_DEFAULT_MODEL = "claude-sonnet-4-5"; static const char* LLM_API_URL = "https://api.anthropic.com/v1/messages"; static const char* LLM_VERSION = "2023-06-01"; @@ -8796,6 +8805,7 @@ el_val_t llm_models(void) { lst = el_list_append(lst, el_wrap_str(el_strdup("claude-haiku-4-5"))); return lst; } +#endif /* HAVE_CURL */ /* ── Native VM builtin aliases ────────────────────────────────────────────── * El source files use native_* names (El VM builtins). @@ -9986,6 +9996,7 @@ el_val_t aead_decrypt(el_val_t key_hex, el_val_t nonce_hex, el_val_t ciphertext_ #endif /* __has_include() */ +#ifdef HAVE_CURL /* ──────────────────────────────────────────────────────────────────────────── * OTLP/HTTP observability — logs, traces, metrics * @@ -10344,6 +10355,8 @@ el_val_t emit_event(el_val_t name_v, el_val_t duration_ms_v) { return trace_span_end(h); } +#endif /* HAVE_CURL — OTLP */ + /* ── Threading seed primitives ─────────────────────────────────────────────── * __thread_create(fn_name, arg) -> Int spawn El fn in a pthread, return tid * __thread_join(tid) -> String join thread, return result string @@ -10798,6 +10811,7 @@ el_val_t config(el_val_t key_v) { return el_wrap_str(el_strdup(val)); } +#ifdef HAVE_CURL /* http_patch — HTTP PATCH request with Content-Type: application/json. * Returns the response body (same error convention as http_post_json). */ el_val_t http_patch(el_val_t url_v, el_val_t body_v) { @@ -10913,6 +10927,7 @@ el_val_t http_get_engram(el_val_t url_v, el_val_t key_v) { } return el_wrap_str(rb.data); } +#endif /* HAVE_CURL */ /* str_to_bytes — encode a string as a JSON array of unsigned byte values. * "hello" -> "[104,101,108,108,111]" @@ -10991,3 +11006,29 @@ el_val_t hash_sha256(el_val_t sv) { return el_hex_encode(digest, 32); } +#ifndef HAVE_CURL +/* ── HAVE_CURL=0 stubs — compile without -lcurl for the elc CLI binary. ───── * + * These return a JSON error string so El programs get a clear message if they + * call HTTP/LLM functions in a curl-less build. */ +static el_val_t _no_curl_err(void) { + return el_wrap_str(el_strdup("{\"error\":\"not built with HAVE_CURL\"}")); +} +el_val_t http_get(el_val_t url) { (void)url; return _no_curl_err(); } +el_val_t http_post(el_val_t url, el_val_t body) { (void)url; (void)body; return _no_curl_err(); } +el_val_t http_post_json(el_val_t url, el_val_t body) { (void)url; (void)body; return _no_curl_err(); } +el_val_t http_get_with_headers(el_val_t url, el_val_t h) { (void)url; (void)h; return _no_curl_err(); } +el_val_t http_post_with_headers(el_val_t url, el_val_t b, el_val_t h) { (void)url; (void)b; (void)h; return _no_curl_err(); } +el_val_t http_post_form_auth(el_val_t url, el_val_t b, el_val_t a) { (void)url; (void)b; (void)a; return _no_curl_err(); } +el_val_t http_delete(el_val_t url) { (void)url; return _no_curl_err(); } +el_val_t http_patch(el_val_t url, el_val_t body) { (void)url; (void)body; return _no_curl_err(); } +el_val_t http_get_to_file(el_val_t url, el_val_t h, el_val_t p) { (void)url; (void)h; (void)p; return _no_curl_err(); } +el_val_t http_post_to_file(el_val_t url, el_val_t b, el_val_t h, el_val_t p) { (void)url; (void)b; (void)h; (void)p; return _no_curl_err(); } +el_val_t http_post_engram(el_val_t url, el_val_t k, el_val_t b) { (void)url; (void)k; (void)b; return _no_curl_err(); } +el_val_t http_get_engram(el_val_t url, el_val_t k) { (void)url; (void)k; return _no_curl_err(); } +el_val_t llm_call(el_val_t m, el_val_t p) { (void)m; (void)p; return _no_curl_err(); } +el_val_t llm_call_system(el_val_t m, el_val_t s, el_val_t u) { (void)m; (void)s; (void)u; return _no_curl_err(); } +el_val_t llm_call_agentic(el_val_t m, el_val_t s, el_val_t u, el_val_t t) { (void)m; (void)s; (void)u; (void)t; return _no_curl_err(); } +el_val_t llm_vision(el_val_t m, el_val_t s, el_val_t p, el_val_t i) { (void)m; (void)s; (void)p; (void)i; return _no_curl_err(); } +el_val_t llm_models(void) { return el_list_empty(); } +void llm_register_tool(el_val_t n, el_val_t f) { (void)n; (void)f; } +#endif /* !HAVE_CURL */ diff --git a/lang/el-compiler/src/codegen.el b/lang/el-compiler/src/codegen.el index 69e9fec..7d5995d 100644 --- a/lang/el-compiler/src/codegen.el +++ b/lang/el-compiler/src/codegen.el @@ -134,7 +134,10 @@ fn c_escape(s: String) -> String { if clean_start < total { let parts = native_list_append(parts, str_slice(s, clean_start, total)) } - str_join(parts, "") + let result: String = str_join(parts, "") + // parts list fully consumed — release to free peak heap. + el_release(parts) + result } fn c_str_lit(s: String) -> String { @@ -849,6 +852,8 @@ fn cg_expr(expr: Map) -> String { let i = i + 1 } let args_c: String = str_join(args_parts, ", ") + // args_parts list fully consumed — release to free peak heap. + el_release(args_parts) if func_kind == "Ident" { let fn_name: String = func["name"] @@ -953,7 +958,10 @@ fn cg_expr(expr: Map) -> String { let items_parts = native_list_append(items_parts, elem_c) let i = i + 1 } - return "el_list_new(" + native_int_to_str(n) + ", " + str_join(items_parts, ", ") + ")" + let items_joined: String = str_join(items_parts, ", ") + // items_parts fully consumed — release to free peak heap. + el_release(items_parts) + return "el_list_new(" + native_int_to_str(n) + ", " + items_joined + ")" } if kind == "Map" { @@ -974,7 +982,10 @@ fn cg_expr(expr: Map) -> String { let items_parts = native_list_append(items_parts, c_str_lit(key) + ", " + val_c) let i = i + 1 } - return "el_map_new(" + native_int_to_str(n) + ", " + str_join(items_parts, ", ") + ")" + let items_joined: String = str_join(items_parts, ", ") + // items_parts fully consumed — release to free peak heap. + el_release(items_parts) + return "el_map_new(" + native_int_to_str(n) + ", " + items_joined + ")" } if kind == "Try" { @@ -1077,7 +1088,10 @@ fn cg_match(expr: Map) -> String { let i = i + 1 } let parts = native_list_append(parts, done_label + ":; " + result_var + "; })") - str_join(parts, "") + let result: String = str_join(parts, "") + // parts list fully consumed — release to free peak heap. + el_release(parts) + result } // Lower a match statement (used for side effects, not as an expression) to a @@ -1246,7 +1260,10 @@ fn cg_if_expr_arm(stmts: [Map], result_var: String) -> String { } let i = i + 1 } - str_join(parts, "") + let result: String = str_join(parts, "") + // parts list fully consumed — release to free peak heap. + el_release(parts) + result } fn cg_if_expr(expr: Map) -> String { @@ -1582,7 +1599,11 @@ fn cg_stmts(stmts: [Map], indent: String, declared: [String]) -> [S let decl = declared while i < n { let stmt = native_list_get(stmts, i) + // Per-statement arena scope: free intermediate strings (str_concat + // fragments, cg_expr results) after each statement is emitted. + let s_mark: Any = el_arena_push() let decl = cg_stmt(stmt, indent, decl) + el_arena_pop(s_mark) let i = i + 1 } decl @@ -1606,7 +1627,10 @@ fn params_to_c(params: [Map]) -> String { let parts = native_list_append(parts, decl) let i = i + 1 } - str_join(parts, ", ") + let result: String = str_join(parts, ", ") + // parts list fully consumed — release to free peak heap. + el_release(parts) + result } // Transform a function body so that an implicit-return final expression @@ -2575,6 +2599,9 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "__channel_recv") { return 1 } if str_eq(name, "__channel_try_recv") { return 1 } if str_eq(name, "__channel_close") { return 1 } + // Arena mark/restore builtins + if str_eq(name, "el_arena_push") { return 0 } + if str_eq(name, "el_arena_pop") { return 1 } // -1 sentinel: variadic / unknown / user-defined -> no check. return -1 } @@ -2811,7 +2838,8 @@ fn cg_fn(stmt: Map) -> Void { if !str_eq(ret_type, "Void") { let body_xformed = transform_implicit_return(body) } - cg_stmts(body_xformed, " ", decl) + let final_decl = cg_stmts(body_xformed, " ", decl) + el_release(final_decl) emit_line(" return 0;") emit_line("}") emit_blank() @@ -3285,3 +3313,360 @@ fn codegen(stmts: [Map], source: String) -> String { // Return empty string - output was streamed via println "" } + +// ── Streaming codegen (JIT function-at-a-time) ───────────────────────────── +// +// codegen_streaming is a memory-efficient alternative to codegen(). +// Instead of receiving the full parsed AST, it receives the raw token list +// and a pre-scanned signature list (from scan_fn_sigs in parser.el). +// +// Pipeline: +// 1. Scan phase (already done by caller): scan_fn_sigs(tokens) -> sigs +// 2. Emit preamble using sigs (no full AST needed) +// 3. For each top-level statement: +// parse_one(tokens, pos) -> { node, pos } +// cg_decl_streaming(node) <- emit C for this one decl +// el_release(node) <- discard AST immediately +// +// Peak memory: O(one function's AST) instead of O(whole program AST). +// +// Entry point: codegen_streaming(tokens, sigs, source) -> String + +// cg_decl_streaming — emit C for a single top-level declaration. +// Handles FnDef, ExternFn, TypeDef, EnumDef, Import, CgiBlock, ServiceBlock. +// Top-level Let statements go into the main() body, not here. +// Top-level executable statements (non-fn, non-let, non-decl) are +// accumulated into state and emitted later in main(). +fn cg_decl_streaming(stmt: Map) -> Void { + let sk: String = stmt["stmt"] + if str_eq(sk, "FnDef") { + cg_fn(stmt) + return + } + // All other top-level decl kinds are either no-ops (Import, TypeDef, + // EnumDef, ExternFn forward decl already emitted) or capability markers + // (CgiBlock, ServiceBlock already handled in preamble). + // Top-level Lets are also no-ops here (file-scope slots already emitted). + // Executable top-level stmts (Expr, Return, etc.) are accumulated in state. + if !str_eq(sk, "FnDef") { + if !is_top_level_decl(stmt) { + if !str_eq(sk, "Let") { + // This is an executable top-level statement. + // We can't emit it into main() yet because we haven't started + // emitting main(). Accumulate in state as a list index. + // We'll collect these into a list and emit after all fns. + state_set("__streaming_has_toplevel_stmts", "1") + } + } + } +} + +// emit_streaming_preamble — emit #includes, forward decls, and file-scope lets +// using the pre-scanned signature data (no full AST). +fn emit_streaming_preamble(sigs: [Map], source: String) -> Void { + let n: Int = native_list_len(sigs) + + // Detect program kind from sigs + let cgi_count: Int = 0 + let svc_count: Int = 0 + let i: Int = 0 + while i < n { + let sig = native_list_get(sigs, i) + let sk: String = sig["kind"] + if str_eq(sk, "cgi_block") { let cgi_count = cgi_count + 1 } + if str_eq(sk, "service_block") { let svc_count = svc_count + 1 } + let i = i + 1 + } + if cgi_count > 1 { + emit_line("#error \"El: multiple cgi blocks in program (only one allowed)\"") + } + if svc_count > 1 { + emit_line("#error \"El: multiple service blocks in program (only one allowed)\"") + } + if cgi_count >= 1 { + if svc_count >= 1 { + emit_line("#error \"El: program declares both cgi and service blocks (mutually exclusive - pick one)\"") + } + } + let kind: String = "utility" + if cgi_count >= 1 { let kind = "cgi" } + if svc_count >= 1 { let kind = "service" } + state_set("__program_kind", kind) + state_set("__cap_violations", "") + state_set("__arity_violations", "") + state_set("__time_violations", "") + + emit_line("#include ") + emit_line("#include ") + emit_line("#include \"el_runtime.h\"") + emit_blank() + + // Forward declarations — use pre-computed params_c strings from scan. + let i = 0 + while i < n { + let sig = native_list_get(sigs, i) + let sk: String = sig["kind"] + if str_eq(sk, "fn") { + let fn_name: String = sig["name"] + if !str_eq(fn_name, "main") { + let params_c: String = sig["params_c"] + emit_line("el_val_t " + fn_name + "(" + params_c + ");") + } + } + if str_eq(sk, "extern_fn") { + let fn_name: String = sig["name"] + let params_c: String = sig["params_c"] + emit_line("el_val_t " + fn_name + "(" + params_c + ");") + } + let i = i + 1 + } + emit_blank() + + // File-scope let slots + let has_toplevel_lets: Bool = false + let i = 0 + while i < n { + let sig = native_list_get(sigs, i) + let sk: String = sig["kind"] + if str_eq(sk, "toplevel_let") { + let name: String = sig["name"] + let ltype: String = sig["ltype"] + if str_eq(ltype, "Int") { add_int_name(name) } + emit_line("el_val_t " + name + ";") + let has_toplevel_lets = true + } + let i = i + 1 + } + if has_toplevel_lets { emit_blank() } +} + +// codegen_streaming — JIT function-at-a-time compiler backend. +// tokens: flat token list from lex() +// sigs: pre-scanned signature list from scan_fn_sigs(tokens) +// source: original source string (for string literal lookup) +fn codegen_streaming(tokens: [Any], sigs: [Map], source: String) -> String { + let total_tokens: Int = native_list_len(tokens) / 2 + + // Emit preamble (forward decls, file-scope lets, #includes) + // Arena scope: free intermediate strings built during preamble emission. + let preamble_mark: Any = el_arena_push() + emit_streaming_preamble(sigs, source) + el_arena_pop(preamble_mark) + + // Detect whether there is a fn main() and whether there are top-level + // executable stmts (for library detection) from sigs. + let has_el_main: Bool = false + let ns: Int = native_list_len(sigs) + let si: Int = 0 + while si < ns { + let sig = native_list_get(sigs, si) + let sk2: String = sig["kind"] + if str_eq(sk2, "fn") { + let fn_name_chk: String = sig["name"] + if str_eq(fn_name_chk, "main") { let has_el_main = true } + } + let si = si + 1 + } + + // Collect top-level let names for seeding main()'s declared set. + let toplevel_let_names: [String] = native_list_empty() + let si = 0 + while si < ns { + let sig = native_list_get(sigs, si) + let sk2: String = sig["kind"] + if str_eq(sk2, "toplevel_let") { + let tname: String = sig["name"] + let toplevel_let_names = native_list_append(toplevel_let_names, tname) + } + let si = si + 1 + } + + // Streaming parse-emit loop. + // For each parsed stmt: + // - FnDef (not main): emit immediately via cg_fn, release AST + // - Others: accumulate only fn-main body and top-level executable stmts + // (these are small in count relative to fn bodies) + let pos: Int = 0 + let el_main_body: [Map] = native_list_empty() + let toplevel_exec_stmts: [Map] = native_list_empty() + let has_toplevel_exec: Bool = false + + let stream_running: Bool = true + while stream_running { + if pos >= total_tokens { + let stream_running = false + } else { + let k: String = tok_kind(tokens, pos) + if str_eq(k, "Eof") { + let stream_running = false + } else { + let r = parse_one(tokens, pos) + let stmt = r["node"] + let new_pos: Int = r["pos"] + el_release(r) + + // Guard against infinite loops + if new_pos <= pos { + el_release(stmt) + let pos = pos + 1 + } else { + let sk: String = stmt["stmt"] + + if str_eq(sk, "FnDef") { + let fn_name2: String = stmt["name"] + if str_eq(fn_name2, "main") { + // Capture main() body for later + let body = stmt["body"] + let bn: Int = native_list_len(body) + let bi: Int = 0 + while bi < bn { + let el_main_body = native_list_append(el_main_body, native_list_get(body, bi)) + let bi = bi + 1 + } + el_release(stmt) + } else { + // Emit immediately — this is the JIT core + // Arena scope: free all intermediate strings (str_concat, + // int_to_str, cg_expr fragments) after each function. + let fn_arena_mark: Any = el_arena_push() + cg_fn(stmt) + el_release(stmt) + el_arena_pop(fn_arena_mark) + } + } else { + if is_top_level_decl(stmt) { + // Import, TypeDef, EnumDef, CgiBlock, ServiceBlock, ExternFn + // These are no-ops in codegen (forward decls already emitted) + el_release(stmt) + } else { + if str_eq(sk, "Let") { + // Top-level let: file-scope slot already declared. + // Keep for main() init — these are few and small. + let toplevel_exec_stmts = native_list_append(toplevel_exec_stmts, stmt) + let has_toplevel_exec = true + } else { + // Executable top-level stmt (rare) + let toplevel_exec_stmts = native_list_append(toplevel_exec_stmts, stmt) + let has_toplevel_exec = true + } + } + } + + let pos = new_pos + } + } + } + } + + // Tokens fully consumed by the streaming loop — release now to free peak heap. + el_release(tokens) + + // Library detection: no fn main and no top-level executable stmts + let is_library: Bool = false + if !has_el_main { + if !has_toplevel_exec { + let is_library = true + } + } + if is_library { return "" } + + // Emit main() — wrap in arena scope to free intermediate strings. + let main_arena_mark: Any = el_arena_push() + let kind2: String = state_get("__program_kind") + emit_line("int main(int _argc, char** _argv) {") + emit_line(" el_runtime_init_args(_argc, _argv);") + + // cgi init if needed + let ns2: Int = native_list_len(sigs) + let si2: Int = 0 + while si2 < ns2 { + let sig2 = native_list_get(sigs, si2) + let sk3: String = sig2["kind"] + if str_eq(sk3, "cgi_block") { + // We need the full cgi_block data — it was parsed by scan_fn_sigs + // but scan only stored the name. For cgi_init we need dharma_id etc. + // Since cgi blocks are rare and small, they end up in toplevel_exec_stmts. + // Find the CgiBlock in toplevel_exec_stmts. + let tes_n: Int = native_list_len(toplevel_exec_stmts) + let tes_i: Int = 0 + while tes_i < tes_n { + let tes = native_list_get(toplevel_exec_stmts, tes_i) + let tes_k: String = tes["stmt"] + if str_eq(tes_k, "CgiBlock") { + let cname2: String = tes["name"] + let cdid2: String = tes["dharma_id"] + let cprin2: String = tes["principal"] + let cnet2: String = tes["network"] + let ceng2: String = tes["engram"] + let has_did2: Bool = tes["has_dharma_id"] + let has_prin2: Bool = tes["has_principal"] + let has_net2: Bool = tes["has_network"] + let has_eng2: Bool = tes["has_engram"] + let arg_name2: String = "EL_STR(" + c_str_lit(cname2) + ")" + let arg_did2: String = cgi_arg(cdid2, has_did2) + let arg_prin2: String = cgi_arg(cprin2, has_prin2) + let arg_net2: String = cgi_arg(cnet2, has_net2) + let arg_eng2: String = cgi_arg(ceng2, has_eng2) + emit_line(" el_cgi_init(" + arg_name2 + ", " + arg_did2 + ", " + arg_prin2 + ", " + arg_net2 + ", " + arg_eng2 + ");") + } + let tes_i = tes_i + 1 + } + } + let si2 = si2 + 1 + } + + // sigs fully consumed — release to free peak heap. + el_release(sigs) + + // Seed declared set with top-level let names + let main_decl2: [String] = native_list_empty() + let tln: Int = native_list_len(toplevel_let_names) + let tli: Int = 0 + while tli < tln { + let main_decl2 = native_list_append(main_decl2, native_list_get(toplevel_let_names, tli)) + let tli = tli + 1 + } + // toplevel_let_names fully consumed — release to free peak heap. + el_release(toplevel_let_names) + + // Emit top-level executable stmts (lets and others) into main() + let tes_n2: Int = native_list_len(toplevel_exec_stmts) + let tes_i2: Int = 0 + while tes_i2 < tes_n2 { + let tes2 = native_list_get(toplevel_exec_stmts, tes_i2) + let tes_k2: String = tes2["stmt"] + if !str_eq(tes_k2, "CgiBlock") { + if !str_eq(tes_k2, "ServiceBlock") { + let main_decl2 = cg_stmt(tes2, " ", main_decl2) + } + } + let tes_i2 = tes_i2 + 1 + } + // toplevel_exec_stmts fully consumed — release to free peak heap. + el_release(toplevel_exec_stmts) + + // Emit fn main() body — per-statement arena scope frees intermediate strings. + let mn: Int = native_list_len(el_main_body) + let mi: Int = 0 + while mi < mn { + let mstmt = native_list_get(el_main_body, mi) + let stmt_mark: Any = el_arena_push() + let main_decl2 = cg_stmt(mstmt, " ", main_decl2) + el_arena_pop(stmt_mark) + let mi = mi + 1 + } + // el_main_body and main_decl2 fully consumed — release to free peak heap. + el_release(el_main_body) + el_release(main_decl2) + + emit_line(" return 0;") + emit_line("}") + emit_blank() + + emit_cap_violations() + emit_arity_violations() + emit_time_violations() + el_arena_pop(main_arena_mark) + + "" +} diff --git a/lang/el-compiler/src/compiler.el b/lang/el-compiler/src/compiler.el index 0df9c5b..fe35471 100644 --- a/lang/el-compiler/src/compiler.el +++ b/lang/el-compiler/src/compiler.el @@ -20,13 +20,15 @@ import "codegen.el" import "codegen-js.el" // compile — full pipeline (C target): source string -> C source string +// Uses JIT function-at-a-time streaming: parse one decl → emit C → discard AST. +// Peak memory is O(one function's AST) instead of O(whole program AST). fn compile(source: String) -> String { let tokens: [Any] = lex(source) - let stmts: [Map] = parse(tokens) - // Token list is no longer needed after parsing — release it to free memory - // before codegen allocates its own working data on large source files. - el_release(tokens) - codegen(stmts, source) + // Fast pre-scan: collect fn signatures + program kind without building + // full expression ASTs. O(tokens) time, minimal allocation. + let sigs: [Map] = scan_fn_sigs(tokens) + // Stream parse-emit: parse one decl at a time, emit C, discard. + codegen_streaming(tokens, sigs, source) } // compile_js — full pipeline (JS target, module mode): source string -> JS source string diff --git a/lang/el-compiler/src/parser.el b/lang/el-compiler/src/parser.el index f4656f8..9c7c3e5 100644 --- a/lang/el-compiler/src/parser.el +++ b/lang/el-compiler/src/parser.el @@ -358,6 +358,8 @@ fn parse_html_attrs(tokens: [Any], pos: Int) -> Map { let r = parse_expr(tokens, p + 1) let val_node = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) let p = expect(tokens, p, "RBrace") let attrs = native_list_append(attrs, { "name": attr_name, "kind": "dynamic", "value": val_node }) } else { @@ -426,6 +428,8 @@ fn parse_html_children(tokens: [Any], pos: Int, parent_tag: String) -> Map Map Map = { "html": "Each", "list": list_expr, "item": item_name, "body": body_children } let children = native_list_append(children, each_node) } else { @@ -476,6 +484,8 @@ fn parse_html_children(tokens: [Any], pos: Int, parent_tag: String) -> Map Map Map { let r_attrs = parse_html_attrs(tokens, p) let attrs = r_attrs["attrs"] let p = r_attrs["pos"] + // r_attrs result map fully consumed — release to free peak heap. + el_release(r_attrs) // check for self-closing /> or void element let k = tok_kind(tokens, p) let self_closing = false @@ -555,6 +569,8 @@ fn parse_html_element(tokens: [Any], pos: Int) -> Map { let r_children = parse_html_children(tokens, p, tag_name) let children = r_children["children"] let p = r_children["pos"] + // r_children result map fully consumed — release to free peak heap. + el_release(r_children) make_result({ "html": "Element", "tag": tag_name, "attrs": attrs, "children": children, "self_closing": false }, p) } @@ -592,6 +608,8 @@ fn parse_html_template(tokens: [Any], pos: Int) -> Map { let r = parse_html_element(tokens, p) let root = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) let root_with_doctype = root if doctype { let root_with_doctype = { "html": root["html"], "tag": root["tag"], "attrs": root["attrs"], "children": root["children"], "self_closing": root["self_closing"], "doctype": true } @@ -649,6 +667,8 @@ fn parse_primary(tokens: [Any], pos: Int) -> Map { let r = parse_expr(tokens, pos + 1) let node = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) let p = expect(tokens, p, "RParen") return make_result(node, p) } @@ -669,6 +689,8 @@ fn parse_primary(tokens: [Any], pos: Int) -> Map { let r = parse_expr(tokens, p) let elem = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) let elems = native_list_append(elems, elem) let k3 = tok_kind(tokens, p) if k3 == "Comma" { @@ -714,6 +736,8 @@ fn parse_primary(tokens: [Any], pos: Int) -> Map { let r = parse_expr(tokens, new_p) let val_node = r["node"] let new_p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) let pair = { "key": key, "value": val_node } let pairs = native_list_append(pairs, pair) let k3 = tok_kind(tokens, new_p) @@ -760,6 +784,8 @@ fn parse_primary(tokens: [Any], pos: Int) -> Map { let r = parse_params(tokens, p) let params = r["params"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) let ret_type = "" let k2 = tok_kind(tokens, p) if k2 == "Arrow" { @@ -773,6 +799,8 @@ fn parse_primary(tokens: [Any], pos: Int) -> Map { let r2 = parse_block(tokens, p) let body = r2["stmts"] let p = r2["pos"] + // r2 result map fully consumed — release to free peak heap. + el_release(r2) return make_result({ "expr": "Lambda", "params": params, "body": body, "ret_type": ret_type }, p) } @@ -781,6 +809,8 @@ fn parse_primary(tokens: [Any], pos: Int) -> Map { let r = parse_primary(tokens, pos + 1) let inner = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) return make_result({ "expr": "Not", "inner": inner }, p) } @@ -789,6 +819,8 @@ fn parse_primary(tokens: [Any], pos: Int) -> Map { let r = parse_primary(tokens, pos + 1) let inner = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) return make_result({ "expr": "Neg", "inner": inner }, p) } @@ -832,9 +864,13 @@ fn parse_if(tokens: [Any], pos: Int) -> Map { state_set("__no_block_expr", prev_no_block) let cond = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) let r2 = parse_block(tokens, p) let then_stmts = r2["stmts"] let p = r2["pos"] + // r2 result map fully consumed — release to free peak heap. + el_release(r2) let has_else = false let else_stmts: [Map] = native_list_empty() let k2 = tok_kind(tokens, p) @@ -846,12 +882,16 @@ fn parse_if(tokens: [Any], pos: Int) -> Map { let r3 = parse_if(tokens, p) let nested = r3["node"] let p = r3["pos"] + // r3 result map fully consumed — release to free peak heap. + el_release(r3) let else_stmts = native_list_append(else_stmts, { "stmt": "Expr", "value": nested }) let has_else = true } else { let r3 = parse_block(tokens, p) let else_stmts = r3["stmts"] let p = r3["pos"] + // r3 result map fully consumed — release to free peak heap. + el_release(r3) let has_else = true } } @@ -866,6 +906,8 @@ fn parse_match(tokens: [Any], pos: Int) -> Map { state_set("__no_block_expr", prev_no_block) let subject = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) let p = expect(tokens, p, "LBrace") let arms: [Map] = native_list_empty() let running = true @@ -881,10 +923,14 @@ fn parse_match(tokens: [Any], pos: Int) -> Map { let r2 = parse_pattern(tokens, p) let pattern = r2["node"] let p = r2["pos"] + // r2 result map fully consumed — release to free peak heap. + el_release(r2) let p = expect(tokens, p, "FatArrow") let r3 = parse_expr(tokens, p) let body = r3["node"] let p = r3["pos"] + // r3 result map fully consumed — release to free peak heap. + el_release(r3) let arm = { "pattern": pattern, "body": body } let arms = native_list_append(arms, arm) let k2 = tok_kind(tokens, p) @@ -938,9 +984,13 @@ fn parse_for_expr(tokens: [Any], pos: Int) -> Map { state_set("__no_block_expr", prev_no_block) let list_expr = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) let r2 = parse_block(tokens, p) let body = r2["stmts"] let p = r2["pos"] + // r2 result map fully consumed — release to free peak heap. + el_release(r2) make_result({ "expr": "For", "item": item_name, "list": list_expr, "body": body }, p) } @@ -959,6 +1009,8 @@ fn parse_block(tokens: [Any], pos: Int) -> Map { let r = parse_stmt(tokens, p) let stmt = r["node"] let new_p: Int = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) let stmts = native_list_append(stmts, stmt) // Non-progress guard: a malformed input (e.g. `||` that // dragged the parser into Map-literal mode partway through @@ -1005,6 +1057,8 @@ fn parse_postfix(tokens: [Any], pos: Int) -> Map { let r = parse_primary(tokens, pos) let node = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) // Postfix duration literal: `.` where is one of // nanos | millis | seconds | minutes | hours | days (each with an @@ -1046,6 +1100,8 @@ fn parse_postfix(tokens: [Any], pos: Int) -> Map { let r2 = parse_expr(tokens, p) let arg = r2["node"] let p = r2["pos"] + // r2 result map fully consumed — release to free peak heap. + el_release(r2) let args = native_list_append(args, arg) let k3 = tok_kind(tokens, p) if k3 == "Comma" { @@ -1066,6 +1122,8 @@ fn parse_postfix(tokens: [Any], pos: Int) -> Map { let r2 = parse_expr(tokens, p + 1) let idx = r2["node"] let p = r2["pos"] + // r2 result map fully consumed — release to free peak heap. + el_release(r2) let p = expect(tokens, p, "RBracket") let node = { "expr": "Index", "object": node, "index": idx } } else { @@ -1122,6 +1180,8 @@ fn parse_binop(tokens: [Any], pos: Int, min_prec: Int) -> Map { let r = parse_postfix(tokens, pos) let left = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) let running = true while running { let k = tok_kind(tokens, p) @@ -1132,6 +1192,8 @@ fn parse_binop(tokens: [Any], pos: Int, min_prec: Int) -> Map { let r2 = parse_binop(tokens, p + 1, prec + 1) let right = r2["node"] let p = r2["pos"] + // r2 result map fully consumed — release to free peak heap. + el_release(r2) let left = { "expr": "BinOp", "op": op, "left": left, "right": right } } else { let running = false @@ -1174,6 +1236,8 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let r = parse_expr(tokens, p) let val = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) return make_result({ "stmt": "Let", "name": name, "value": val, "type": ltype }, p) } @@ -1190,6 +1254,8 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let r = parse_expr(tokens, p) let val = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) return make_result({ "stmt": "Return", "value": val }, p) } @@ -1204,6 +1270,8 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let r = parse_params(tokens, p) let params = r["params"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) let ret_type = "" let k3: String = tok_kind(tokens, p) if str_eq(k3, "Arrow") { @@ -1224,6 +1292,8 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let r = parse_params(tokens, p) let params = r["params"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) // return type annotation: -> Type. Capture the leading identifier // so codegen can distinguish Void-returning functions from value- // returning ones. Anything not "Void" is treated as a value type. @@ -1240,6 +1310,8 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let r2 = parse_block(tokens, p) let body = r2["stmts"] let p = r2["pos"] + // r2 result map fully consumed — release to free peak heap. + el_release(r2) return make_result({ "stmt": "FnDef", "name": name, "params": params, "body": body, "ret_type": ret_type }, p) } @@ -1363,9 +1435,13 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { state_set("__no_block_expr", prev_no_block) let cond = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) let r2 = parse_block(tokens, p) let body = r2["stmts"] let p = r2["pos"] + // r2 result map fully consumed — release to free peak heap. + el_release(r2) return make_result({ "stmt": "While", "cond": cond, "body": body }, p) } @@ -1391,6 +1467,8 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { state_set("__no_block_expr", prev_no_block) let start_expr = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) // Check for range operator: .. (exclusive) or ..= (inclusive) let range_k = tok_kind(tokens, p) if range_k == "DotDot" { @@ -1399,9 +1477,11 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let r2 = parse_expr(tokens, p) let end_expr = r2["node"] let p = r2["pos"] + el_release(r2) let r3 = parse_block(tokens, p) let body = r3["stmts"] let p = r3["pos"] + el_release(r3) return make_result({ "stmt": "ForRange", "var": item_name, "start": start_expr, "end": end_expr, "inclusive": false, "body": body }, p) } if range_k == "DotDotEq" { @@ -1410,9 +1490,11 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let r2 = parse_expr(tokens, p) let end_expr = r2["node"] let p = r2["pos"] + el_release(r2) let r3 = parse_block(tokens, p) let body = r3["stmts"] let p = r3["pos"] + el_release(r3) return make_result({ "stmt": "ForRange", "var": item_name, "start": start_expr, "end": end_expr, "inclusive": true, "body": body }, p) } // No range operator: regular for-in (list iteration) @@ -1420,6 +1502,7 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let r2 = parse_block(tokens, p) let body = r2["stmts"] let p = r2["pos"] + el_release(r2) return make_result({ "stmt": "For", "item": item_name, "list": list_expr, "body": body }, p) } @@ -1431,6 +1514,8 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let r_try = parse_block(tokens, p) let try_body = r_try["stmts"] let p = r_try["pos"] + // r_try result map fully consumed — release to free peak heap. + el_release(r_try) let catch_name = "err" let k2 = tok_kind(tokens, p) if str_eq(k2, "Catch") { @@ -1452,6 +1537,8 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let r_catch = parse_block(tokens, p) let catch_body = r_catch["stmts"] let p = r_catch["pos"] + // r_catch result map fully consumed — release to free peak heap. + el_release(r_catch) return make_result({ "stmt": "TryCatch", "try_body": try_body, "catch_name": catch_name, "catch_body": catch_body }, p) } return make_result({ "stmt": "TryCatch", "try_body": try_body, "catch_name": catch_name, "catch_body": native_list_empty() }, p) @@ -1475,6 +1562,8 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { "ret_type": inner["ret_type"], "decorator": dec_name } + // r result map fully consumed — release to free peak heap. + el_release(r) return make_result(with_dec, p2) } return r @@ -1609,6 +1698,8 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let r = parse_expr(tokens, p) let val = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) return make_result({ "stmt": "Assign", "name": name, "value": val }, p) } } @@ -1617,6 +1708,8 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let r = parse_expr(tokens, pos) let val = r["node"] let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) make_result({ "stmt": "Expr", "value": val }, p) } @@ -1639,6 +1732,8 @@ fn parse(tokens: [Any]) -> [Map] { let r = parse_stmt(tokens, pos) let stmt = r["node"] let new_pos: Int = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) let stmts = native_list_append(stmts, stmt) // Guard against infinite loops - if pos didn't advance, force it if new_pos <= pos { @@ -1651,3 +1746,297 @@ fn parse(tokens: [Any]) -> [Map] { } stmts } + +// -- Streaming parse helpers --------------------------------------------------- + +// parse_one — parse exactly one top-level statement at position `pos`. +// Returns { "node": stmt_map, "pos": new_pos }. +// Enables the streaming compiler pipeline (parse one → emit C → discard). +fn parse_one(tokens: [Any], pos: Int) -> Map { + parse_stmt(tokens, pos) +} + +// skip_to_rbrace — advance past a balanced { ... } block. +// On entry, pos must point at the LBrace token. +// Returns the position of the token AFTER the matching RBrace. +fn skip_to_rbrace(tokens: [Any], pos: Int) -> Int { + let total: Int = native_list_len(tokens) / 2 + let p: Int = pos + 1 + let depth: Int = 1 + let going: Bool = true + while going { + if p >= total { + let going = false + } else { + let kk: String = tok_kind(tokens, p) + if str_eq(kk, "Eof") { + let going = false + } else { + if str_eq(kk, "LBrace") { + let depth = depth + 1 + let p = p + 1 + } else { + if str_eq(kk, "RBrace") { + let depth = depth - 1 + let p = p + 1 + if depth <= 0 { + let going = false + } + } else { + let p = p + 1 + } + } + } + } + } + p +} + +// is_stmt_start_kind — true if `k` is a token kind that can start a new +// top-level statement (used to find safe stopping points during token skips). +fn is_stmt_start_kind(k: String) -> Bool { + if str_eq(k, "Fn") { return true } + if str_eq(k, "Let") { return true } + if str_eq(k, "Extern") { return true } + if str_eq(k, "Cgi") { return true } + if str_eq(k, "Service") { return true } + if str_eq(k, "Type") { return true } + if str_eq(k, "Enum") { return true } + if str_eq(k, "Import") { return true } + if str_eq(k, "From") { return true } + if str_eq(k, "Eof") { return true } + false +} + +// skip_expr_to_stmt_boundary — skip tokens from `pos` until we reach a +// token that could start a new top-level statement, staying depth-aware +// so that braces inside expressions don't fool us. +fn skip_expr_to_stmt_boundary(tokens: [Any], pos: Int) -> Int { + let total: Int = native_list_len(tokens) / 2 + let p: Int = pos + let depth: Int = 0 + let going: Bool = true + while going { + if p >= total { + let going = false + } else { + let kk: String = tok_kind(tokens, p) + if str_eq(kk, "Eof") { + let going = false + } else { + if str_eq(kk, "LBrace") { + let depth = depth + 1 + let p = p + 1 + } else { + if str_eq(kk, "RBrace") { + if depth <= 0 { + let going = false + } else { + let depth = depth - 1 + let p = p + 1 + } + } else { + if depth == 0 { + if is_stmt_start_kind(kk) { + let going = false + } else { + let p = p + 1 + } + } else { + let p = p + 1 + } + } + } + } + } + } + p +} + +// scan_params_c — scan a parameter list `(name: Type, ...)` starting at +// position `pos` (which should point at LParen) and return the C parameter +// declaration string along with the new position. +// Returns { "c": String, "pos": Int }. +// This avoids allocating param-map objects during the pre-scan phase. +fn scan_params_c(tokens: [Any], pos: Int) -> Map { + let p: Int = expect(tokens, pos, "LParen") + let parts: [String] = native_list_empty() + let going: Bool = true + while going { + let kk: String = tok_kind(tokens, p) + if str_eq(kk, "RParen") { + let going = false + } else { + if str_eq(kk, "Eof") { + let going = false + } else { + let pname: String = tok_value(tokens, p) + let p = p + 1 + let p = expect(tokens, p, "Colon") + let p = skip_type(tokens, p) + let parts = native_list_append(parts, "el_val_t " + pname) + let k2: String = tok_kind(tokens, p) + if str_eq(k2, "Comma") { + let p = p + 1 + } + } + } + } + let p = expect(tokens, p, "RParen") + let c_str: String = str_join(parts, ", ") + // parts list fully consumed — release to free peak heap. + el_release(parts) + if str_eq(c_str, "") { let c_str = "void" } + { "c": c_str, "pos": p } +} + +// scan_fn_sigs — lightweight token-level pre-scan. +// +// Returns a list of descriptor maps (one per top-level item) without building +// full expression ASTs or param-map objects. Descriptors have these shapes: +// +// fn/extern_fn: { "kind": "fn"|"extern_fn", "name": String, +// "params_c": String, <- C param decl string, e.g. "el_val_t a, el_val_t b" +// "is_main": Bool } +// toplevel_let: { "kind": "toplevel_let", "name": String, "ltype": String } +// cgi_block: { "kind": "cgi_block", "name": String } +// service_block: { "kind": "service_block", "name": String } +// +// Import/TypeDef/EnumDef nodes are skipped (codegen treats them as no-ops). +// +// The scan allocates only small string values per entry, keeping peak RSS low. +fn scan_fn_sigs(tokens: [Any]) -> [Map] { + let total: Int = native_list_len(tokens) / 2 + let sigs: [Map] = native_list_empty() + let pos: Int = 0 + let going: Bool = true + while going { + if pos >= total { + let going = false + } else { + let k: String = tok_kind(tokens, pos) + if str_eq(k, "Eof") { + let going = false + } else { + // --- fn definition --- + if str_eq(k, "Fn") { + let p: Int = pos + 1 + let name: String = tok_value(tokens, p) + let p = p + 1 + let r = scan_params_c(tokens, p) + let params_c: String = r["c"] + let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) + // skip return type + let k2: String = tok_kind(tokens, p) + if str_eq(k2, "Arrow") { + let p = p + 1 + let p = skip_type(tokens, p) + } + // skip body + let k3: String = tok_kind(tokens, p) + if str_eq(k3, "LBrace") { + let p = skip_to_rbrace(tokens, p) + } + let is_main: Bool = str_eq(name, "main") + let sigs = native_list_append(sigs, { + "kind": "fn", + "name": name, + "params_c": params_c, + "is_main": is_main + }) + let pos = p + } else { + // --- extern fn --- + if str_eq(k, "Extern") { + let p: Int = pos + 1 + let k2: String = tok_kind(tokens, p) + if str_eq(k2, "Fn") { + let p = p + 1 + let name: String = tok_value(tokens, p) + let p = p + 1 + let r = scan_params_c(tokens, p) + let params_c: String = r["c"] + let p = r["pos"] + // r result map fully consumed — release to free peak heap. + el_release(r) + let k3: String = tok_kind(tokens, p) + if str_eq(k3, "Arrow") { + let p = p + 1 + let p = skip_type(tokens, p) + } + let sigs = native_list_append(sigs, { + "kind": "extern_fn", + "name": name, + "params_c": params_c, + "is_main": false + }) + let pos = p + } else { + let pos = pos + 1 + } + } else { + // --- top-level let --- + if str_eq(k, "Let") { + let p: Int = pos + 1 + let name: String = tok_value(tokens, p) + let p = p + 1 + let ltype: String = "" + let k2: String = tok_kind(tokens, p) + if str_eq(k2, "Colon") { + let p = p + 1 + let kt: String = tok_kind(tokens, p) + if str_eq(kt, "Ident") { let ltype = tok_value(tokens, p) } + let p = skip_type(tokens, p) + } + let p = expect(tokens, p, "Eq") + let p = skip_expr_to_stmt_boundary(tokens, p) + let sigs = native_list_append(sigs, { + "kind": "toplevel_let", + "name": name, + "ltype": ltype + }) + let pos = p + } else { + // --- cgi block --- + if str_eq(k, "Cgi") { + let p: Int = pos + 1 + let name: String = tok_value(tokens, p) + let p = p + 1 + let k2: String = tok_kind(tokens, p) + if str_eq(k2, "LBrace") { + let p = skip_to_rbrace(tokens, p) + } + let sigs = native_list_append(sigs, { + "kind": "cgi_block", + "name": name + }) + let pos = p + } else { + // --- service block --- + if str_eq(k, "Service") { + let p: Int = pos + 1 + let name: String = tok_value(tokens, p) + let p = p + 1 + let k2: String = tok_kind(tokens, p) + if str_eq(k2, "LBrace") { + let p = skip_to_rbrace(tokens, p) + } + let sigs = native_list_append(sigs, { + "kind": "service_block", + "name": name + }) + let pos = p + } else { + // Import, Type, Enum, From, or any other token. + // Skip ahead to the next statement boundary. + let p: Int = pos + 1 + let p = skip_expr_to_stmt_boundary(tokens, p) + let pos = p + }}}}} + } + } + } + sigs +}