From 37bcf7eb749786c0e5ffda7a1969ce533c3441e9 Mon Sep 17 00:00:00 2001 From: bigmerge Date: Sat, 15 Aug 2026 21:21:42 -0500 Subject: [PATCH] =?UTF-8?q?runtime:=20allocation=20accounting=20=E2=80=94?= =?UTF-8?q?=20the=20deterministic=20signal=20for=20complexity=20gating?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the three primitives the test-framework design (DESIGN.md §6.5) requires for gating on growth curves: el_alloc_count, el_alloc_bytes, el_peak_rss. Registered in codegen's builtin_arity and wrapped in el_seed.c per the project's C-builtin recipe. WHY COUNTS AND NOT WALL-CLOCK: a growth-curve gate has to be a hard build failure, which means the signal cannot flake. Wall-clock needs warmup, statistics, and a quiet machine; on shared CI it is unusable as a gate. Allocation counts are perfectly deterministic — same input, same number, every machine, every run. Fit them against n and a complexity regression becomes a build failure with zero noise. All four runtime string allocators (el_strdup, el_strbuf, and their _persist variants) funnel every allocation the language performs, so instrumenting there counts everything. WHY BYTES AS WELL AS COUNT — this is not redundancy, it is the whole gate. Measured with two El programs, one allocating once per item, one rebuilding its accumulator each iteration: n linear allocs / bytes quadratic allocs / bytes 100 100 / 290 100 / 5,150 200 200 / 690 200 / 20,300 400 400 / 1,490 400 / 80,600 800 800 / 3,090 800 / 321,200 The quadratic program's allocation COUNT is exactly linear — identical to the healthy one. Counting allocations alone would have missed it completely. Bytes catch it: each doubling of n quadruples bytes (ratios 3.94, 3.97, 3.99 -> converging on 4.0, i.e. O(n^2)), while the linear case converges on 2.0. That shape — count linear, per-allocation size growing — is the classic accidental quadratic, and it is exactly elc's defect: quadratic allocation VOLUME, which the old shipped compiler paid in RSS (27 GB, OOM) and the rebuilt one pays in malloc/free churn (42s on 1.4 MB). Volume was the invariant across both; RSS and wall-clock were just the two ways it surfaced. el_peak_rss is exported for context and is explicitly NOT a gating signal — it is perturbed by allocator internals, the page cache, and the OS. Gate on the deterministic numbers; report the physical one. Counters are unsynchronised by design: this is measurement, and a lock would change the thing being measured. Exact on the single-threaded compile path, approximate under threads. --- lang/el-compiler/src/codegen.el | 6 +++ lang/runtime/el_runtime.c | 70 ++++++++++++++++++++++++++++++++- lang/runtime/el_runtime.h | 6 +++ lang/runtime/el_seed.c | 7 ++++ 4 files changed, 87 insertions(+), 2 deletions(-) diff --git a/lang/el-compiler/src/codegen.el b/lang/el-compiler/src/codegen.el index 3c63dbd..1f6d758 100644 --- a/lang/el-compiler/src/codegen.el +++ b/lang/el-compiler/src/codegen.el @@ -2766,6 +2766,9 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "__engram_scan_nodes_json") { return 2 } if str_eq(name, "__engram_edges_json") { return 2 } if str_eq(name, "__engram_pool_stats_json") { return 0 } + if str_eq(name, "__el_alloc_count") { return 0 } + if str_eq(name, "__el_alloc_bytes") { return 0 } + if str_eq(name, "__el_peak_rss") { return 0 } if str_eq(name, "__generate") { return 1 } // Filesystem if str_eq(name, "fs_read") { return 1 } @@ -2866,6 +2869,9 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "engram_scan_nodes_json") { return 2 } if str_eq(name, "engram_edges_json") { return 2 } if str_eq(name, "engram_pool_stats_json") { return 0 } + if str_eq(name, "el_alloc_count") { return 0 } + if str_eq(name, "el_alloc_bytes") { return 0 } + if str_eq(name, "el_peak_rss") { return 0 } if str_eq(name, "engram_neighbors_json") { return 3 } if str_eq(name, "engram_activate_json") { return 2 } if str_eq(name, "engram_stats_json") { return 0 } diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index 7ae187b..7c948f1 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -155,21 +155,55 @@ el_val_t el_arena_pop(el_val_t mark) { return 0; } +/* ── Allocation accounting ─────────────────────────────────────────────────── + * + * Every string allocation in the runtime funnels through the four functions + * below, so counting here counts everything the language does. + * + * WHY THIS EXISTS: a growth-curve gate needs a signal that is DETERMINISTIC. + * Wall-clock needs statistics, warmup, and a quiet machine; it is noisy on + * shared CI and unusable as a hard build gate. Allocation COUNT has none of + * those problems — the same input allocates the same number of times on every + * machine, every run. Fit allocations against input size and a complexity + * regression becomes a build failure with zero flake. + * + * This is not hypothetical. elc's known defect is quadratic ALLOCATION VOLUME. + * The old shipped binary paid it in RSS (27 GB, OOM); the rebuilt one pays the + * same quadratic in malloc/free churn (42s on a 1.4 MB input). The allocation + * count was the invariant across both — RSS and wall-clock were just the two + * ways it surfaced. An `expect allocs O(n)` assertion on the compile path + * would have failed the build the day it was introduced. + * + * Peak RSS is exported too but is explicitly NOT the gating signal: it is + * perturbed by allocator behaviour, page cache, and the OS. Gate on counts, + * report RSS as context. + * + * Counters are plain unsigned longs, incremented on the allocating thread with + * no synchronisation: this is measurement, and a lock here would change the + * thing being measured. Under threads the count is approximate; for the + * single-threaded compile path it is exact. + * ──────────────────────────────────────────────────────────────────────────── */ +static unsigned long _el_alloc_count = 0; +static unsigned long _el_alloc_bytes = 0; + /* Persistent allocation — bypasses the arena (state_set, engram internals). */ static char* el_strdup_persist(const char* s) { - if (!s) return strdup(""); + if (!s) { _el_alloc_count++; _el_alloc_bytes += 1; return strdup(""); } + _el_alloc_count++; _el_alloc_bytes += strlen(s) + 1; return strdup(s); } static char* el_strbuf_persist(size_t n) { char* p = malloc(n + 1); if (!p) { fputs("el_runtime: out of memory\n", stderr); exit(1); } p[0] = '\0'; + _el_alloc_count++; _el_alloc_bytes += n + 1; return p; } static char* el_strdup(const char* s) { - if (!s) { char* p = strdup(""); el_arena_track(p); return p; } + if (!s) { char* p = strdup(""); _el_alloc_count++; _el_alloc_bytes += 1; el_arena_track(p); return p; } char* p = strdup(s); + _el_alloc_count++; _el_alloc_bytes += strlen(s) + 1; el_arena_track(p); return p; } @@ -178,6 +212,7 @@ static char* el_strbuf(size_t n) { char* p = malloc(n + 1); if (!p) { fputs("el_runtime: out of memory\n", stderr); exit(1); } p[0] = '\0'; + _el_alloc_count++; _el_alloc_bytes += n + 1; el_arena_track(p); return p; } @@ -18411,3 +18446,34 @@ el_val_t engram_pool_stats_json(void) { (unsigned)STORE_PAGE_SIZE); return el_wrap_str(el_strdup(b)); } + +/* ── Allocation/RSS introspection (test-framework complexity gate, §6.5) ───── + * + * el_alloc_count() — total runtime string allocations since process start. + * THE gating signal. Deterministic: same input => same count, every machine, + * every run. A benchmark harness samples it before and after an operation at + * several input sizes and fits the deltas against n; a curve worse than the + * declared one fails the build. No warmup, no statistics, no baseline file, + * no flake — none of which is true of wall-clock. + * + * el_alloc_bytes() — total bytes requested. Same determinism; catches the case + * where allocation COUNT stays linear but per-allocation SIZE grows, which is + * the classic accidental-quadratic shape (rebuilding a whole buffer per + * append). Count alone would miss it. + * + * el_peak_rss() — peak resident set in bytes. Context, NOT a gate: perturbed by + * allocator internals, the page cache, and the OS. Reported so a human can + * see the physical consequence; never fitted. + */ +el_val_t el_alloc_count(void) { return (el_val_t)(int64_t)_el_alloc_count; } +el_val_t el_alloc_bytes(void) { return (el_val_t)(int64_t)_el_alloc_bytes; } + +el_val_t el_peak_rss(void) { + struct rusage ru; + if (getrusage(RUSAGE_SELF, &ru) != 0) return (el_val_t)0; +#if defined(__APPLE__) || defined(__MACH__) + return (el_val_t)(int64_t)ru.ru_maxrss; /* macOS: bytes */ +#else + return (el_val_t)(int64_t)(ru.ru_maxrss * 1024L); /* Linux: KB -> bytes */ +#endif +} diff --git a/lang/runtime/el_runtime.h b/lang/runtime/el_runtime.h index 8fbc9bf..76f8f09 100644 --- a/lang/runtime/el_runtime.h +++ b/lang/runtime/el_runtime.h @@ -1017,6 +1017,12 @@ el_val_t stdout_to_file(el_val_t path); el_val_t stdout_restore(void); el_val_t el_mem_check(void); +/* Allocation accounting — the deterministic signal behind complexity gating. + * Gate on counts/bytes; peak RSS is context only. */ +el_val_t el_alloc_count(void); +el_val_t el_alloc_bytes(void); +el_val_t el_peak_rss(void); + /* Semantic retrieval surface. NOT interchangeable with engram_search_json, * which is lexical by design — see the note at the definition. */ el_val_t engram_recall_json(el_val_t query, el_val_t limit); diff --git a/lang/runtime/el_seed.c b/lang/runtime/el_seed.c index cb741e5..68dcf5b 100644 --- a/lang/runtime/el_seed.c +++ b/lang/runtime/el_seed.c @@ -1379,6 +1379,13 @@ el_val_t __engram_edges_json(el_val_t limit, el_val_t offset) { el_val_t engram_pool_stats_json(void); el_val_t __engram_pool_stats_json(void) { return engram_pool_stats_json(); } +el_val_t el_alloc_count(void); +el_val_t el_alloc_bytes(void); +el_val_t el_peak_rss(void); +el_val_t __el_alloc_count(void) { return el_alloc_count(); } +el_val_t __el_alloc_bytes(void) { return el_alloc_bytes(); } +el_val_t __el_peak_rss(void) { return el_peak_rss(); } + el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset) { return engram_scan_nodes_by_type_json(node_type, limit, offset); }