From 6a6b589ba01be87176e72a976a617086774df74a Mon Sep 17 00:00:00 2001 From: Neuron Date: Sat, 15 Aug 2026 21:45:13 -0500 Subject: [PATCH] bench: real black_box barrier + three-signal growth-curve gate Adds el_black_box (inline asm, +r constraint, memory clobber) and runtime/elbench.el: a growth-curve classifier that gates time AND allocation-count AND allocation-bytes, failing if any exceeds its declared curve. Refusal is a first-class verdict. The classifier REFUSES rather than classifying when the largest measurement is below the floor, or when a series is hard-flat across an 8x input range -- the shape produced when the optimiser deletes the work. Reporting O(1) there would be a confident answer with nothing behind it. Disagreeing ratios report INDETERMINATE rather than a guess. Deviation from DESIGN.md 6.2, stated in the source: uses consecutive ratios on a mandated geometric sweep rather than least-squares over candidate curves. Ratios are directly interpretable on a doubling sweep and need no floating point; the cost is weaker O(n) vs O(n log n) separation, reported as an ambiguous band rather than guessed. Documents the counter scope limit: engram_*.c and libcurl malloc are NOT tracked, so a flat curve over engram/HTTP-dominated work is not evidence of anything. 13 tests prove the classifier against real measured series from fitprobe.el -- including that an accumulator's allocation COUNT is linear while its bytes are quadratic, and that el #132's pure-CPU shape reads FLAT on both allocation signals and is caught only by time. --- lang/el-compiler/src/codegen.el | 1 + lang/runtime/el_runtime.c | 20 +++ lang/runtime/el_runtime.h | 1 + lang/runtime/elbench.el | 244 ++++++++++++++++++++++++++++++ lang/tests/native/test_elbench.el | 131 ++++++++++++++++ 5 files changed, 397 insertions(+) create mode 100644 lang/runtime/elbench.el create mode 100644 lang/tests/native/test_elbench.el diff --git a/lang/el-compiler/src/codegen.el b/lang/el-compiler/src/codegen.el index fe1056f..b4f9204 100644 --- a/lang/el-compiler/src/codegen.el +++ b/lang/el-compiler/src/codegen.el @@ -2887,6 +2887,7 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "el_alloc_count") { return 0 } if str_eq(name, "el_alloc_bytes") { return 0 } if str_eq(name, "el_peak_rss") { return 0 } + if str_eq(name, "el_black_box") { return 1 } if str_eq(name, "engram_neighbors_json") { return 3 } if str_eq(name, "engram_activate_json") { return 2 } if str_eq(name, "engram_stats_json") { return 0 } diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index 965b891..10c1233 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -18522,6 +18522,26 @@ el_val_t engram_pool_stats_json(void) { el_val_t el_alloc_count(void) { return (el_val_t)(int64_t)_el_alloc_count; } el_val_t el_alloc_bytes(void) { return (el_val_t)(int64_t)_el_alloc_bytes; } +/* el_black_box — optimisation barrier for benchmark bodies. + * + * WHY THIS IS NOT OPTIONAL. A benchmark whose result is unused is dead code, + * and CONSUMING THE RESULT IS NOT SUFFICIENT: clang recognises loop idioms and + * closes them to arithmetic. A nested `total = total + 1` loop measured at + * 0 microseconds for every n while returning a numerically correct n*n -- + * the answer was right and the work never happened. + * + * That is the same failure shape as a test that never ran reporting pass. The + * harness must own the barrier rather than trusting the benchmark author to + * defeat the optimiser. + * + * The constraint "+r" forces the value through a register the compiler must + * treat as both read and written by opaque code; the "memory" clobber stops + * loads and stores being reordered across it or elided. Emits no instructions. */ +el_val_t el_black_box(el_val_t v) { + __asm__ __volatile__("" : "+r"(v) : : "memory"); + return v; +} + el_val_t el_peak_rss(void) { struct rusage ru; if (getrusage(RUSAGE_SELF, &ru) != 0) return (el_val_t)0; diff --git a/lang/runtime/el_runtime.h b/lang/runtime/el_runtime.h index 76f8f09..25ea292 100644 --- a/lang/runtime/el_runtime.h +++ b/lang/runtime/el_runtime.h @@ -1022,6 +1022,7 @@ el_val_t el_mem_check(void); el_val_t el_alloc_count(void); el_val_t el_alloc_bytes(void); el_val_t el_peak_rss(void); +el_val_t el_black_box(el_val_t v); /* Semantic retrieval surface. NOT interchangeable with engram_search_json, * which is lexical by design — see the note at the definition. */ diff --git a/lang/runtime/elbench.el b/lang/runtime/elbench.el new file mode 100644 index 0000000..a19446f --- /dev/null +++ b/lang/runtime/elbench.el @@ -0,0 +1,244 @@ +// runtime/elbench.el — growth-curve classifier and complexity gate. +// +// Given a geometric sweep of input sizes and the measurements taken at each, +// classify the growth curve and decide whether it violates a declared bound. +// +// ── Why this exists ────────────────────────────────────────────────────────── +// +// Constant-factor regressions are annoying. Complexity regressions are outages. +// An O(n) lookup inside an O(n) loop is invisible at n=100 in a unit test and +// catastrophic at n=100000 in production. el #132 was exactly that: a strlen() +// inside a per-character accessor, quadratic, shipped for months. +// +// ── THREE signals, not one ─────────────────────────────────────────────────── +// +// The gate fits time AND allocation-count AND allocation-bytes, and fails if +// ANY of them exceeds its declared curve. This is not belt-and-braces; each +// signal is blind to a real defect class the others catch: +// +// * A copy-on-write accumulator rebuilding its buffer allocates ONCE per +// iteration — count is exactly linear — while bytes go quadratic. +// Count alone passes it. +// * el #132's strlen-per-character is pure CPU and allocates NOTHING. +// Both allocation signals read FLAT. Only time catches it. +// +// The deterministic signals (count, bytes) are preferable where they apply: +// no statistics, correct on the first run, machine-independent. They are +// simply not sufficient. +// +// ── SCOPE LIMIT — read this before trusting a flat curve ───────────────────── +// +// The allocation counters track EL-LEVEL allocation only: strings, ElList and +// ElMap bodies, their backing arrays, copy-on-write clones, and the realloc +// growth path. malloc inside engram_*.c and inside libcurl is NOT counted. +// +// A flat allocation curve over a workload dominated by engram or HTTP calls is +// therefore NOT evidence of anything. It means "no El-level allocation growth", +// not "no allocation growth". Gate El-level complexity with this; do not read +// third-party memory behaviour into it. +// +// ── Classification method ──────────────────────────────────────────────────── +// +// Sizes must form a geometric sweep (each n double the last). On such a sweep +// the ratio between consecutive measurements IS the growth exponent, directly: +// +// O(1) -> 1.0 O(log n) -> ~1.1 O(n) -> 2.0 +// O(n log n) -> ~2.2 O(n^2) -> 4.0 O(n^3) -> 8.0 +// +// DEVIATION FROM DESIGN.md 6.2, stated plainly: that section specified Google +// Benchmark's one-parameter least-squares fit over candidate curves. This uses +// consecutive ratios instead. The sweep is mandated geometric either way, and +// on a geometric sweep ratios are directly interpretable and need no floating +// point. The cost is weaker separation between O(n) and O(n log n), which is +// reported honestly as an ambiguous band rather than guessed at. Least-squares +// remains the better answer if that band ever needs to be resolved. +// +// All arithmetic is fixed-point, scaled by 1000 ("milli-ratio"), so a ratio of +// 2.0 is 2000. El values are int64; this avoids float-in-list handling. + +// Curve identifiers. Ordered by growth — the ordering IS the comparison used +// by the gate, so an index comparison decides "worse than declared". +// 0 = O(1) 1 = O(log n) 2 = O(n) 3 = O(n log n) 4 = O(n^2) 5 = O(n^3) + +fn elb_curve_name(c: Int) -> String { + if c == 0 { return "O(1)" } + if c == 1 { return "O(log n)" } + if c == 2 { return "O(n)" } + if c == 3 { return "O(n log n)" } + if c == 4 { return "O(n^2)" } + if c == 5 { return "O(n^3)" } + return "O(?)" +} + +fn elb_curve_from_name(s: String) -> Int { + if str_eq(s, "O(1)") { return 0 } + if str_eq(s, "O(log n)") { return 1 } + if str_eq(s, "O(n)") { return 2 } + if str_eq(s, "O(n log n)") { return 3 } + if str_eq(s, "O(n^2)") { return 4 } + if str_eq(s, "O(n^3)") { return 5 } + return -1 +} + +// elb_classify_ratio — map a milli-ratio-per-doubling onto a curve. +// +// Bands are deliberately wide at the top (a quadratic measured at 3.4x is +// still a quadratic) and deliberately overlap-averse at the bottom, where a +// misclassification between O(1) and O(log n) matters least. +fn elb_classify_ratio(milli: Int) -> Int { + if milli < 1300 { return 0 } + if milli < 1700 { return 1 } + if milli < 2400 { return 2 } + if milli < 3200 { return 3 } + if milli < 6000 { return 4 } + return 5 +} + +// elb_ratio — milli-ratio between two consecutive measurements. +// Returns -1 when the earlier measurement is zero (ratio undefined). +fn elb_ratio(prev: Int, cur: Int) -> Int { + if prev <= 0 { return -1 } + return (cur * 1000) / prev +} + +// ── The measurement floor ──────────────────────────────────────────────────── +// +// A benchmark whose largest measurement is at or near zero has not been +// measured. Reporting it as O(1) would be a confident answer with nothing +// behind it — the same failure as a test that never ran reporting pass, and +// exactly what happened when clang closed a nested loop to a multiply and the +// harness read 0 microseconds at every n. +// +// So: REFUSE. Never classify below the floor. +fn elb_below_floor(vals: [Int], floor: Int) -> Bool { + let n: Int = native_list_len(vals) + let i: Int = 0 + let mx: Int = 0 + while i < n { + let v: Int = native_list_get(vals, i) + if v > mx { let mx = v } + let i = i + 1 + } + if mx < floor { return true } + return false +} + +// elb_implausibly_flat — a measurement that does not move across a sweep whose +// input grew by 8x or more is not a flat curve, it is a broken measurement. +// Genuine O(1) work still shows noise; a hard-flat series means the work was +// optimised away, the timer has insufficient resolution, or the benchmark body +// never executed. +fn elb_implausibly_flat(vals: [Int]) -> Bool { + let n: Int = native_list_len(vals) + if n < 3 { return false } + let first: Int = native_list_get(vals, 0) + let last: Int = native_list_get(vals, n - 1) + if first == 0 { + if last == 0 { return true } + return false + } + let r: Int = (last * 1000) / first + if r < 1100 { return true } + return false +} + +// elb_spread_ok — do the consecutive ratios agree with each other? +// +// This is the ratio-method analogue of a normalised-RMS threshold. If the +// doublings disagree wildly the data is noise, a cache cliff, or a phase +// change, and the honest report is INDETERMINATE rather than a classification. +fn elb_spread_ok(ratios: [Int]) -> Bool { + let n: Int = native_list_len(ratios) + if n < 2 { return true } + let lo: Int = 999999 + let hi: Int = 0 + let i: Int = 0 + while i < n { + let r: Int = native_list_get(ratios, i) + if r >= 0 { + if r < lo { let lo = r } + if r > hi { let hi = r } + } + let i = i + 1 + } + if lo <= 0 { return false } + // Reject when the widest ratio is more than 2.2x the narrowest. That is + // enough slack for real timing noise and tight enough to separate a clean + // 2.0 series from a clean 4.0 series. + if (hi * 1000) / lo > 2200 { return false } + return true +} + +// elb_ratios — consecutive milli-ratios across the sweep. +fn elb_ratios(vals: [Int]) -> [Int] { + let out: [Int] = native_list_empty() + let n: Int = native_list_len(vals) + let i: Int = 1 + while i < n { + let out = native_list_append(out, + elb_ratio(native_list_get(vals, i - 1), native_list_get(vals, i))) + let i = i + 1 + } + return out +} + +// elb_mean_tail_ratio — mean of the LAST TWO ratios. +// +// The tail is used deliberately: asymptotic behaviour is what a complexity +// bound claims, and the small-n end of any sweep is dominated by fixed +// overhead. This is the same reason a benchmark harness discards warmup. +fn elb_mean_tail_ratio(ratios: [Int]) -> Int { + let n: Int = native_list_len(ratios) + if n == 0 { return -1 } + if n == 1 { return native_list_get(ratios, 0) } + let a: Int = native_list_get(ratios, n - 1) + let b: Int = native_list_get(ratios, n - 2) + if a < 0 { return b } + if b < 0 { return a } + return (a + b) / 2 +} + +// ── Verdicts ───────────────────────────────────────────────────────────────── +// +// 0 PASS measured curve is at or below the declared bound +// 1 FAIL measured curve is strictly worse than declared +// 2 INDETERMINATE ratios disagree; data is noise or a phase change +// 3 REFUSED below the measurement floor, or implausibly flat +// 4 BETTER measured strictly better than declared (warn, not fail) + +fn elb_verdict_name(v: Int) -> String { + if v == 0 { return "PASS" } + if v == 1 { return "FAIL" } + if v == 2 { return "INDETERMINATE" } + if v == 3 { return "REFUSED" } + if v == 4 { return "BETTER" } + return "?" +} + +// elb_gate — classify one signal against its declared bound. +// +// vals measurements, one per sweep point, in sweep order +// expect declared curve index (see elb_curve_name) +// floor minimum largest-measurement below which we refuse to classify +fn elb_gate(vals: [Int], expect: Int, floor: Int) -> Int { + if elb_below_floor(vals, floor) { return 3 } + if elb_implausibly_flat(vals) { return 3 } + let ratios: [Int] = elb_ratios(vals) + if !elb_spread_ok(ratios) { return 2 } + let m: Int = elb_mean_tail_ratio(ratios) + if m < 0 { return 2 } + let got: Int = elb_classify_ratio(m) + if got > expect { return 1 } + if got < expect { return 4 } + return 0 +} + +// elb_measured_curve — the classified curve for a signal, or -1 if unclassifiable. +fn elb_measured_curve(vals: [Int], floor: Int) -> Int { + if elb_below_floor(vals, floor) { return -1 } + if elb_implausibly_flat(vals) { return -1 } + let ratios: [Int] = elb_ratios(vals) + let m: Int = elb_mean_tail_ratio(ratios) + if m < 0 { return -1 } + return elb_classify_ratio(m) +} diff --git a/lang/tests/native/test_elbench.el b/lang/tests/native/test_elbench.el new file mode 100644 index 0000000..bd674d5 --- /dev/null +++ b/lang/tests/native/test_elbench.el @@ -0,0 +1,131 @@ +import "../../runtime/eltest.el" +import "../../runtime/elbench.el" + +// test_elbench.el — proves the growth-curve classifier against KNOWN curves. +// +// Every series below is real measured data from lang/tests/bench/fitprobe.el +// on a geometric sweep n = 200/400/800/1600. The classifier must be provable +// without depending on a live defect existing, which is the whole point of +// keeping controlled specimens. + +fn _s4(a: Int, b: Int, c: Int, d: Int) -> [Int] { + let l: [Int] = native_list_empty() + let l = native_list_append(l, a) + let l = native_list_append(l, b) + let l = native_list_append(l, c) + let l = native_list_append(l, d) + return l +} + +test "classifies a linear allocation series as O(n)" { + // fitprobe `linear`, allocation count + let v = _s4(208, 409, 810, 1611) + let c: Int = elb_measured_curve(v, 10) + assert c == 2, "linear allocs should classify O(n)" +} + +test "classifies a linear byte series as O(n)" { + // fitprobe `linear`, allocation bytes + let v = _s4(4786, 9682, 19474, 39658) + let c: Int = elb_measured_curve(v, 10) + assert c == 2, "linear bytes should classify O(n)" +} + +test "classifies a quadratic byte series as O(n^2)" { + // fitprobe `accum`, allocation bytes -- the accumulator-rebuild shape + let v = _s4(20300, 80600, 321200, 1282400) + let c: Int = elb_measured_curve(v, 10) + assert c == 4, "accum bytes should classify O(n^2)" +} + +test "accumulator count is linear -- proves count alone misses it" { + // Same run as above. The COUNT is exactly linear while bytes are + // quadratic. A count-only gate passes this defect clean. + let v = _s4(200, 400, 800, 1600) + let c: Int = elb_measured_curve(v, 10) + assert c == 2, "accum count classifies O(n)" + let g: Int = elb_gate(v, 2, 10) + assert g == 0, "count-only gate PASSES the quadratic" +} + +test "classifies a quadratic time series as O(n^2)" { + // fitprobe `compute` -- el #132's shape: n scans over n characters + let v = _s4(67, 205, 818, 3268) + let c: Int = elb_measured_curve(v, 10) + assert c == 4, "compute time should classify O(n^2)" +} + +test "REFUSES an all-zero series instead of calling it O(1)" { + // fitprobe `compute` allocation count. Pure CPU, allocates nothing. + // Reporting O(1) here would be a confident answer with nothing behind it. + let v = _s4(0, 0, 0, 0) + let g: Int = elb_gate(v, 2, 10) + assert g == 3, "all-zero series must be REFUSED" + // NOTE: bind before comparing. `call(...) == ` lowers to str_eq() + // on integers and segfaults -- see the elc == inference bug reported with + // this change. `let x = call(); x == y` is the safe form. + let got: Int = elb_measured_curve(v, 10) + assert got < 0, "unclassifiable returns -1" +} + +test "REFUSES an implausibly flat series" { + // The shape produced when clang closes a loop to a multiply: a real + // answer, no work done, no movement across an 8x input range. + let v = _s4(1000, 1001, 1002, 1003) + let g: Int = elb_gate(v, 2, 10) + assert g == 3, "hard-flat series must be REFUSED" +} + +test "gate FAILS a quadratic declared as linear" { + let v = _s4(20300, 80600, 321200, 1282400) + let g: Int = elb_gate(v, 2, 10) + assert g == 1, "O(n^2) measured vs O(n) declared must FAIL" +} + +test "gate PASSES a linear series declared as linear" { + let v = _s4(208, 409, 810, 1611) + let g: Int = elb_gate(v, 2, 10) + assert g == 0, "O(n) measured vs O(n) declared must PASS" +} + +test "gate reports BETTER when measured beats the declared bound" { + let v = _s4(208, 409, 810, 1611) + let g: Int = elb_gate(v, 4, 10) + assert g == 4, "O(n) measured vs O(n^2) declared is BETTER" +} + +test "gate reports INDETERMINATE on disagreeing ratios" { + // fitprobe `linear` WALL TIME at these sizes: 26/19/43/78 microseconds. + // Ratios 0.73, 2.26, 1.81 disagree well past the noise threshold. The + // honest answer is "cannot tell", not a classification -- this is exactly + // why benchmarks need auto-scaled iteration counts rather than one shot. + let v = _s4(26, 19, 43, 78) + let g: Int = elb_gate(v, 2, 10) + assert g == 2, "disagreeing ratios must be INDETERMINATE" +} + +test "black_box is a real barrier and returns its input" { + let bb: Int = el_black_box(42) + assert bb == 42, "black_box is value-preserving" + let s: Int = 0 + let i: Int = 0 + while i < 100 { + // Bind the call before using it in arithmetic: `x + call(...)` + // lowers to el_str_concat() on integers. Same inference defect + // as `call(...) == y` lowering to str_eq(). + let bx: Int = el_black_box(1) + let s = s + bx + let i = i + 1 + } + assert s == 100, "black_box does not disturb the computation" +} + +test "curve names round-trip" { + let k1: Int = elb_curve_from_name("O(n)") + assert k1 == 2, "O(n) parses" + let k2: Int = elb_curve_from_name("O(n^2)") + assert k2 == 4, "O(n^2) parses" + assert str_eq(elb_curve_name(4), "O(n^2)"), "O(n^2) renders" + let unk: Int = elb_curve_from_name("O(nonsense)") + assert unk < 0, "unknown curve is -1" +}