Archived
Will Anderson
23bbc99e43
The compiler used to OOM at ~8.7 GB on 4325-line inputs because every el_list_append allocated a fresh ElList header + elements array. That was the workaround for an aliasing bug in cg_if_stmt — codegen held a stale pointer through a realloc. Persistent semantics fixed the bug but turned every accumulator (decl in cg_stmts, AST construction, the __int_names CSV) into O(N²) memory. Real fix in two coordinated parts: 1. Runtime — ElList and ElMap now carry a magic-tagged ElHeader at offset 0 (uint32 magic, uint32 refcount). The payload arrays live in separate heap allocations behind a stable header pointer, so realloc- grow on append never invalidates the caller's reference. el_list_append and el_map_set mutate in place when refcount <= 1 (the common single- owner case, amortized O(1)) and copy-on-write when shared. Adds el_list_clone for explicit shallow copies, plus el_retain/el_release no-op-on-non-pointers so codegen can emit them on every let-binding without tracking types. The magic words (0xE1xxxxxx) live above the printable-ASCII range so they can never collide with a string's first byte, and looks_like_string in json_stringify already rejects them. 2. Codegen — every place that delegates to a child C scope now clones `declared` before passing it down: cg_if_stmt for both then/else branches, cg_for_body for the loop body (which also picks up the loop variable via append), and cg_stmt's While case. Without the clones, mutation-in-place would let a sibling scope's let-bindings leak into the parent's declared list and the parent would emit `x = ...` against an undeclared name. The clones are cheap shallow copies of a list of strings. Result on the landing-combined.el (4325 lines): 8.7 GB → 3.5 GB peak, 0.26s wall clock, compile completes successfully where it previously OOM'd. Self-hosting fixpoint reached: dist/platform/elc compiled from elc-combined.el reproduces dist/platform/elc.c byte-for-byte on a second pass through itself. Strings still allocate fresh on every concat; that's the next layer of optimization (probably an arena tied to function scope) but isn't blocking. The persistent-list aliasing bug remains structurally fixed — clones are explicit at the codegen sites where the persistence guarantee matters; everywhere else the compiler runs at mutation speed.
359 lines
18 KiB
C
359 lines
18 KiB
C
/*
|
|
* el_runtime.h — El language C runtime header
|
|
*
|
|
* Declares all built-in functions available to compiled El programs.
|
|
* Include this in every generated .c file.
|
|
*
|
|
* Value model:
|
|
* All El values are represented as el_val_t (= int64_t).
|
|
* On 64-bit systems a pointer fits in int64_t.
|
|
* String values are cast: (el_val_t)(uintptr_t)"hello"
|
|
* Integer values are stored directly.
|
|
* This lets arithmetic work naturally while still passing strings around.
|
|
*
|
|
* Type conventions (El -> C):
|
|
* String -> el_val_t (holds const char* via uintptr_t cast)
|
|
* Int -> el_val_t
|
|
* Bool -> el_val_t (0 = false, nonzero = true)
|
|
* Any -> el_val_t
|
|
* Void -> void
|
|
*
|
|
* Macros for convenience:
|
|
* EL_STR(s) cast string literal to el_val_t
|
|
* EL_CSTR(v) cast el_val_t back to const char*
|
|
* EL_INT(v) identity — el_val_t is already int64_t
|
|
*
|
|
* Link requirements:
|
|
* -lcurl — required for the HTTP client (http_get, http_post, llm_*).
|
|
* -lpthread — required for the HTTP server (one detached thread per
|
|
* connection, capped at 64 concurrent).
|
|
*
|
|
* Canonical compile command:
|
|
* cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
|
|
* -o <out> <prog>.c el-compiler/runtime/el_runtime.c
|
|
*/
|
|
|
|
#pragma once
|
|
|
|
#include <stdint.h>
|
|
#include <stdlib.h>
|
|
|
|
typedef int64_t el_val_t;
|
|
|
|
#define EL_STR(s) ((el_val_t)(uintptr_t)(s))
|
|
#define EL_CSTR(v) ((const char*)(uintptr_t)(v))
|
|
#define EL_INT(v) (v)
|
|
#define EL_NULL ((el_val_t)0)
|
|
|
|
/* Float values share the el_val_t (int64) slot via a bit-cast.
|
|
* The codegen emits Float literals as `el_from_float(<dbl>)` so the
|
|
* underlying bits represent the IEEE 754 double. Float-aware builtins
|
|
* (math, format, json) round-trip via these helpers. */
|
|
static inline double el_to_float(el_val_t v) {
|
|
union { int64_t i; double f; } u;
|
|
u.i = (int64_t)v;
|
|
return u.f;
|
|
}
|
|
|
|
static inline el_val_t el_from_float(double f) {
|
|
union { double f; int64_t i; } u;
|
|
u.f = f;
|
|
return (el_val_t)u.i;
|
|
}
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
/* ── I/O ──────────────────────────────────────────────────────────────────── */
|
|
|
|
void println(el_val_t s);
|
|
void print(el_val_t s);
|
|
el_val_t readline(void);
|
|
|
|
/* ── String builtins ─────────────────────────────────────────────────────── */
|
|
|
|
el_val_t el_str_concat(el_val_t a, el_val_t b);
|
|
el_val_t str_eq(el_val_t a, el_val_t b);
|
|
el_val_t str_starts_with(el_val_t s, el_val_t prefix);
|
|
el_val_t str_ends_with(el_val_t s, el_val_t suffix);
|
|
el_val_t str_len(el_val_t s);
|
|
el_val_t str_concat(el_val_t a, el_val_t b);
|
|
el_val_t int_to_str(el_val_t n);
|
|
el_val_t str_to_int(el_val_t s);
|
|
el_val_t str_slice(el_val_t s, el_val_t start, el_val_t end);
|
|
el_val_t str_contains(el_val_t s, el_val_t sub);
|
|
el_val_t str_replace(el_val_t s, el_val_t from, el_val_t to);
|
|
el_val_t str_to_upper(el_val_t s);
|
|
el_val_t str_to_lower(el_val_t s);
|
|
el_val_t str_trim(el_val_t s);
|
|
|
|
/* ── Math ────────────────────────────────────────────────────────────────── */
|
|
|
|
el_val_t el_abs(el_val_t n);
|
|
el_val_t el_max(el_val_t a, el_val_t b);
|
|
el_val_t el_min(el_val_t a, el_val_t b);
|
|
|
|
/* ── Refcount (ARC) ──────────────────────────────────────────────────────────
|
|
* Lists and Maps carry a refcount. Strings and ints do not — el_retain and
|
|
* el_release are safe no-ops on non-refcounted values (they sniff a magic
|
|
* header at offset 0 and only act if the magic matches).
|
|
*
|
|
* Codegen emits these at let-binding shadowing, function entry (params), and
|
|
* function exit (locals other than the returned value). The refcount lets
|
|
* el_list_append and el_map_set mutate in place when uniquely owned (cheap)
|
|
* and copy-on-write when shared (preserves persistent semantics across
|
|
* accumulator patterns in the compiler itself). */
|
|
|
|
void el_retain(el_val_t v);
|
|
void el_release(el_val_t v);
|
|
|
|
/* ── List ────────────────────────────────────────────────────────────────── */
|
|
|
|
el_val_t el_list_new(el_val_t count, ...);
|
|
el_val_t el_list_len(el_val_t list);
|
|
el_val_t el_list_get(el_val_t list, el_val_t index);
|
|
el_val_t el_list_append(el_val_t list, el_val_t elem);
|
|
el_val_t el_list_empty(void);
|
|
el_val_t el_list_clone(el_val_t list);
|
|
|
|
/* ── Map ─────────────────────────────────────────────────────────────────── */
|
|
|
|
el_val_t el_map_new(el_val_t pair_count, ...);
|
|
el_val_t el_get_field(el_val_t map, el_val_t key);
|
|
el_val_t el_map_get(el_val_t map, el_val_t key);
|
|
el_val_t el_map_set(el_val_t map, el_val_t key, el_val_t value);
|
|
|
|
/* ── HTTP ─────────────────────────────────────────────────────────────────── */
|
|
|
|
el_val_t http_get(el_val_t url);
|
|
el_val_t http_post(el_val_t url, el_val_t body);
|
|
el_val_t http_post_json(el_val_t url, el_val_t json_body);
|
|
el_val_t http_get_with_headers(el_val_t url, el_val_t headers_map);
|
|
el_val_t http_post_with_headers(el_val_t url, el_val_t body, el_val_t headers_map);
|
|
el_val_t http_post_form_auth(el_val_t url, el_val_t form_body, el_val_t auth_header);
|
|
void http_serve(el_val_t port, el_val_t handler);
|
|
void http_set_handler(el_val_t name);
|
|
|
|
/* ── Filesystem ──────────────────────────────────────────────────────────── */
|
|
|
|
el_val_t fs_read(el_val_t path);
|
|
el_val_t fs_write(el_val_t path, el_val_t content);
|
|
el_val_t fs_list(el_val_t path);
|
|
|
|
/* ── JSON ────────────────────────────────────────────────────────────────── */
|
|
|
|
el_val_t json_get(el_val_t json, el_val_t key);
|
|
el_val_t json_parse(el_val_t s);
|
|
el_val_t json_stringify(el_val_t v);
|
|
el_val_t json_get_string(el_val_t json_str, el_val_t key);
|
|
el_val_t json_get_int(el_val_t json_str, el_val_t key);
|
|
el_val_t json_get_float(el_val_t json_str, el_val_t key);
|
|
el_val_t json_get_bool(el_val_t json_str, el_val_t key);
|
|
el_val_t json_get_raw(el_val_t json_str, el_val_t key);
|
|
el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value);
|
|
el_val_t json_array_len(el_val_t json_str);
|
|
|
|
/* ── Time ────────────────────────────────────────────────────────────────── */
|
|
|
|
el_val_t time_now(void);
|
|
el_val_t time_now_utc(void);
|
|
el_val_t time_format(el_val_t ts, el_val_t fmt);
|
|
el_val_t time_to_parts(el_val_t ts);
|
|
el_val_t time_from_parts(el_val_t secs, el_val_t ns, el_val_t tz);
|
|
el_val_t time_add(el_val_t ts, el_val_t n, el_val_t unit);
|
|
el_val_t time_diff(el_val_t ts1, el_val_t ts2, el_val_t unit);
|
|
|
|
/* ── UUID ────────────────────────────────────────────────────────────────── */
|
|
|
|
el_val_t uuid_new(void);
|
|
el_val_t uuid_v4(void);
|
|
|
|
/* ── Environment ─────────────────────────────────────────────────────────── */
|
|
|
|
el_val_t env(el_val_t key);
|
|
|
|
/* ── In-process state K/V ────────────────────────────────────────────────── */
|
|
|
|
el_val_t state_set(el_val_t key, el_val_t value);
|
|
el_val_t state_get(el_val_t key);
|
|
el_val_t state_del(el_val_t key);
|
|
el_val_t state_keys(void);
|
|
|
|
/* ── Float formatting ────────────────────────────────────────────────────── */
|
|
|
|
el_val_t float_to_str(el_val_t f);
|
|
el_val_t int_to_float(el_val_t n);
|
|
el_val_t float_to_int(el_val_t f);
|
|
el_val_t format_float(el_val_t f, el_val_t decimals);
|
|
el_val_t decimal_round(el_val_t f, el_val_t decimals);
|
|
el_val_t str_to_float(el_val_t s);
|
|
|
|
/* ── Math (Float-aware) ──────────────────────────────────────────────────── */
|
|
|
|
el_val_t math_sqrt(el_val_t f);
|
|
el_val_t math_log(el_val_t f);
|
|
el_val_t math_ln(el_val_t f);
|
|
el_val_t math_sin(el_val_t f);
|
|
el_val_t math_cos(el_val_t f);
|
|
el_val_t math_pi(void);
|
|
|
|
/* ── String additions ────────────────────────────────────────────────────── */
|
|
|
|
el_val_t str_index_of(el_val_t s, el_val_t sub);
|
|
el_val_t str_split(el_val_t s, el_val_t sep);
|
|
el_val_t str_char_at(el_val_t s, el_val_t i);
|
|
el_val_t str_char_code(el_val_t s, el_val_t i);
|
|
el_val_t str_pad_left(el_val_t s, el_val_t width, el_val_t pad);
|
|
el_val_t str_pad_right(el_val_t s, el_val_t width, el_val_t pad);
|
|
el_val_t str_format(el_val_t template, el_val_t data);
|
|
el_val_t str_lower(el_val_t s);
|
|
el_val_t str_upper(el_val_t s);
|
|
|
|
/* ── List additions ──────────────────────────────────────────────────────── */
|
|
|
|
el_val_t list_push(el_val_t list, el_val_t elem);
|
|
el_val_t list_push_front(el_val_t list, el_val_t elem);
|
|
el_val_t list_join(el_val_t list, el_val_t sep);
|
|
el_val_t list_range(el_val_t start, el_val_t end);
|
|
|
|
/* ── Bool helpers ────────────────────────────────────────────────────────── */
|
|
|
|
el_val_t bool_to_str(el_val_t b);
|
|
|
|
/* ── Process ─────────────────────────────────────────────────────────────── */
|
|
|
|
void exit_program(el_val_t code);
|
|
|
|
/* ── CGI identity ─────────────────────────────────────────────────────────────
|
|
* Called at the start of main() in CGI programs (those with a `cgi {}` block).
|
|
* Records the program's DHARMA identity before any other code executes. */
|
|
|
|
void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal,
|
|
el_val_t network, el_val_t engram);
|
|
|
|
/* ── DHARMA network builtins ─────────────────────────────────────────────────
|
|
* Available to CGI programs (declared with a `cgi {}` block).
|
|
*
|
|
* Peers are addressed by `dharma_id` of the form
|
|
* "<registry-id>@<transport-url>" e.g. "ntn-genesis@http://localhost:7770"
|
|
* If the @<url> portion is omitted, transport defaults to
|
|
* "http://localhost:7770" (the local CGI daemon assumption).
|
|
*
|
|
* Wire protocol (all peers expose):
|
|
* POST <url>/dharma/recv { channel, from, content } → response body
|
|
* POST <url>/dharma/event { type, payload, source, timestamp }
|
|
* POST <url>/api/activate { query } → list of nodes
|
|
*
|
|
* Hosting application's responsibility: an El program with a `cgi {}` block
|
|
* runs http_serve() with its own request handler; that handler should route
|
|
* "/dharma/event" requests by calling el_runtime_dharma_event_arrive() so
|
|
* incoming events feed dharma_field() queues. The runtime itself does not
|
|
* intercept any /dharma path. */
|
|
|
|
el_val_t dharma_connect(el_val_t cgi_id);
|
|
el_val_t dharma_send(el_val_t channel, el_val_t content);
|
|
el_val_t dharma_activate(el_val_t query);
|
|
void dharma_emit(el_val_t event_type, el_val_t payload);
|
|
el_val_t dharma_field(el_val_t event_type);
|
|
void dharma_strengthen(el_val_t cgi_id, el_val_t weight);
|
|
el_val_t dharma_relationship(el_val_t cgi_id);
|
|
el_val_t dharma_peers(void);
|
|
|
|
/* Public C API: called by an El program's HTTP handler when a /dharma/event
|
|
* request arrives. Pushes onto the per-event-type queue and signals any
|
|
* pending dharma_field() blockers. All three arguments must be NUL-terminated
|
|
* C strings (or NULL — then treated as empty). */
|
|
void el_runtime_dharma_event_arrive(const char* event_type,
|
|
const char* payload,
|
|
const char* source);
|
|
|
|
/* ── Engram local graph primitives ───────────────────────────────────────────
|
|
* Operate on the CGI's local Engram knowledge graph.
|
|
* `engram_activate` queries the local graph only; `dharma_activate` is
|
|
* network-wide across all connected CGI graphs. */
|
|
|
|
el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience);
|
|
el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
|
|
el_val_t salience, el_val_t importance, el_val_t confidence,
|
|
el_val_t tier, el_val_t tags);
|
|
el_val_t engram_get_node(el_val_t id);
|
|
void engram_strengthen(el_val_t node_id);
|
|
void engram_forget(el_val_t node_id);
|
|
el_val_t engram_node_count(void);
|
|
el_val_t engram_search(el_val_t query, el_val_t limit);
|
|
el_val_t engram_scan_nodes(el_val_t limit, el_val_t offset);
|
|
void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation);
|
|
el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id);
|
|
el_val_t engram_neighbors(el_val_t node_id);
|
|
el_val_t engram_neighbors_filtered(el_val_t node_id, el_val_t max_depth, el_val_t direction);
|
|
el_val_t engram_edge_count(void);
|
|
el_val_t engram_activate(el_val_t query, el_val_t depth);
|
|
el_val_t engram_save(el_val_t path);
|
|
el_val_t engram_load(el_val_t path);
|
|
|
|
/* JSON-string accessors — return pre-serialized JSON so HTTP handlers
|
|
* can pass results straight through without round-tripping ElList/ElMap
|
|
* through json_stringify. */
|
|
el_val_t engram_get_node_json(el_val_t id);
|
|
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
|
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
|
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
|
|
el_val_t engram_activate_json(el_val_t query, el_val_t depth);
|
|
el_val_t engram_stats_json(void);
|
|
|
|
/* ── LLM (Anthropic API client) ─────────────────────────────────────────────
|
|
* All functions call https://api.anthropic.com/v1/messages with the API key
|
|
* from env ANTHROPIC_API_KEY. Default model when empty: claude-sonnet-4-5. */
|
|
|
|
el_val_t llm_call(el_val_t model, el_val_t prompt);
|
|
el_val_t llm_call_system(el_val_t model, el_val_t system_prompt, el_val_t user_prompt);
|
|
el_val_t llm_call_agentic(el_val_t model, el_val_t system, el_val_t user, el_val_t tools);
|
|
el_val_t llm_vision(el_val_t model, el_val_t system, el_val_t prompt, el_val_t image_url_or_b64);
|
|
el_val_t llm_models(void);
|
|
|
|
/* Register a tool handler by name. The handler is looked up via dlsym
|
|
* (mirroring http_set_handler), so any El `fn <name>(input)` compiles to
|
|
* a global C symbol that this function can locate at runtime.
|
|
* Handler signature: `el_val_t handler(el_val_t input_json)` — receives
|
|
* the tool input as a JSON-string el_val_t and returns a JSON-string
|
|
* el_val_t result. Used by llm_call_agentic. */
|
|
void llm_register_tool(el_val_t name, el_val_t handler_fn_name);
|
|
|
|
/* ── args() ─────────────────────────────────────────────────────────────────
|
|
* Provides access to command-line arguments passed to the program.
|
|
* Populated by el_runtime_init_args() before main() runs. */
|
|
|
|
el_val_t args(void);
|
|
void el_runtime_init_args(int argc, char** argv);
|
|
|
|
/* ── Native VM builtin aliases (for compiled El source) ─────────────────────
|
|
* These match the El VM's native_* builtins so that El source compiled
|
|
* to C can call the same names without modification. */
|
|
|
|
el_val_t native_list_get(el_val_t list, el_val_t index);
|
|
el_val_t native_list_len(el_val_t list);
|
|
el_val_t native_list_append(el_val_t list, el_val_t elem);
|
|
el_val_t native_list_empty(void);
|
|
el_val_t native_list_clone(el_val_t list);
|
|
el_val_t native_string_chars(el_val_t s);
|
|
el_val_t native_int_to_str(el_val_t n);
|
|
|
|
/* ── Method-call shorthand aliases ──────────────────────────────────────────
|
|
* The El method-call convention `obj.method(args)` compiles to
|
|
* `method(obj, args)`. These aliases expose the runtime functions under
|
|
* the short names that result from method calls in El source.
|
|
*
|
|
* Example: `myList.append(x)` → `append(myList, x)` (calls this alias)
|
|
* `myList.len()` → `len(myList)` (calls this alias) */
|
|
|
|
el_val_t append(el_val_t list, el_val_t elem); /* el_list_append */
|
|
el_val_t len(el_val_t list); /* el_list_len */
|
|
el_val_t get(el_val_t list, el_val_t index); /* el_list_get */
|
|
el_val_t map_get(el_val_t map, el_val_t key); /* el_map_get */
|
|
el_val_t map_set(el_val_t map, el_val_t key, el_val_t value); /* el_map_set */
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|