From 9a13547fe2918c23ff6984d6f0d1626ea00dd148 Mon Sep 17 00:00:00 2001 From: bigmerge Date: Sun, 16 Aug 2026 16:48:04 -0500 Subject: [PATCH 1/2] runtime: put el_runtime.c on a ratchet, and actually run the guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/check-single-runtime.sh guards against el_runtime.c being COPIED — it was written after a lagging fork shipped to prod and dropped learned hebb edges. Nothing guarded against it GROWING. So it grew: 10,607 -> 20,527 lines, 94% in 3.5 months, the whole time under an explicit commit-message promise that it was a temporary shim about to be deleted. Worse, the copy guard was never wired in. Its own footer described the CI wire-in as a TODO, and the TODO had never been done — the script existed but ran nowhere, in no workflow and in no hook, so it had caught nothing for as long as it has been in the tree. A guard that does not run is a comment. This adds the missing guard and runs both. * lang/runtime/BUDGET — a RATCHET, not a limit. max_lines is set at the current 20,527 with NO headroom: the file cannot grow by one line. A second cap, max_engram_fns (279), counts top-level engram_/eg_/cog_ definitions in it — ~47.5% of the file is engram code and engram already owns six sibling .c files, so this is the scoreboard for moving it out. Both may only go DOWN. * scripts/check-runtime-growth.sh — enforces the ratchet, and three invariants that keep the multi-file runtime honest: every .c in lang/runtime/ is either in SOURCES or explicitly platform-optional (an unaccounted .c is compiled by nothing and is silently dead); install.sh's hardcoded download list matches SOURCES (it cannot call the helper — it runs where there is no checkout — so that copy is checked, not trusted); and an advisory nudge to lower the budget when you have earned it. * Both guards now run as early steps in ci-dev.yaml, ci-stage.yaml and sdk-release.yaml, and in .githooks/pre-commit. The failure message is the point. The guard that existed said what was wrong but not where the code should go, which makes it easy to "fix" by arguing with the guard. This one names the destination: the concern-owning .c, or a new .c plus one line in SOURCES, or c_source in a program's manifest.el — and it prints the `nm` command that proves placement is link-time and that the shipped compiler already links from ten translation units. Every runtime file except el_runtime.c is deliberately uncapped, because that is where code is supposed to go. Proven with negative controls, per lang/AGENTS.md step 5 — each shown FAILING: * +1 line to el_runtime.c -> FAIL (20528/20527) * +1 engram fn, net-zero lines -> FAIL (280/279) * a new unaccounted lang/runtime/*.c -> FAIL * engram_store.c removed from install.sh -> FAIL, names the missing file * el_runtime.c truncated to 20,000 lines -> PASS + "lower max_lines to 20000" * baseline, tree unmodified -> OK, and both guards green el_runtime.c is byte-identical after the controls; this commit changes zero lines of it. --- .gitea/workflows/ci-dev.yaml | 10 ++ .gitea/workflows/ci-stage.yaml | 10 ++ .gitea/workflows/sdk-release.yaml | 10 ++ .githooks/pre-commit | 9 ++ lang/AGENTS.md | 2 + lang/runtime/BUDGET | 39 ++++++++ scripts/check-runtime-growth.sh | 161 ++++++++++++++++++++++++++++++ scripts/check-single-runtime.sh | 18 ++-- 8 files changed, 250 insertions(+), 9 deletions(-) create mode 100644 lang/runtime/BUDGET create mode 100755 scripts/check-runtime-growth.sh diff --git a/.gitea/workflows/ci-dev.yaml b/.gitea/workflows/ci-dev.yaml index a0af2c5..373258e 100644 --- a/.gitea/workflows/ci-dev.yaml +++ b/.gitea/workflows/ci-dev.yaml @@ -19,6 +19,16 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # Guards must run from the REPO ROOT — override the job's + # defaults.run.working-directory: lang + - name: Guard - single canonical runtime source + working-directory: ${{ github.workspace }} + run: bash scripts/check-single-runtime.sh + + - name: Guard - el_runtime.c growth budget + working-directory: ${{ github.workspace }} + run: bash scripts/check-runtime-growth.sh + - name: Install build dependencies run: | apt-get update -qq diff --git a/.gitea/workflows/ci-stage.yaml b/.gitea/workflows/ci-stage.yaml index ad82268..f87d731 100644 --- a/.gitea/workflows/ci-stage.yaml +++ b/.gitea/workflows/ci-stage.yaml @@ -29,6 +29,16 @@ jobs: fi echo "Source branch check passed: ${SOURCE} -> stage" + # Guards must run from the REPO ROOT — override the job's + # defaults.run.working-directory: lang + - name: Guard - single canonical runtime source + working-directory: ${{ github.workspace }} + run: bash scripts/check-single-runtime.sh + + - name: Guard - el_runtime.c growth budget + working-directory: ${{ github.workspace }} + run: bash scripts/check-runtime-growth.sh + - name: Install build dependencies run: | apt-get update -qq diff --git a/.gitea/workflows/sdk-release.yaml b/.gitea/workflows/sdk-release.yaml index f79347e..adaa353 100644 --- a/.gitea/workflows/sdk-release.yaml +++ b/.gitea/workflows/sdk-release.yaml @@ -29,6 +29,16 @@ jobs: fi echo "Source branch check passed: ${SOURCE} -> main" + # Guards must run from the REPO ROOT — override the job's + # defaults.run.working-directory: lang + - name: Guard - single canonical runtime source + working-directory: ${{ github.workspace }} + run: bash scripts/check-single-runtime.sh + + - name: Guard - el_runtime.c growth budget + working-directory: ${{ github.workspace }} + run: bash scripts/check-runtime-growth.sh + - name: Install build dependencies run: | apt-get update -qq diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 2ce3237..c168617 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -9,6 +9,15 @@ LANG_DIR="$ROOT/lang" RUNTIME="$LANG_DIR/runtime" ELC="$LANG_DIR/dist/platform/elc" +# Runtime guards — catch drift and growth before they are committed, not in CI. +# check-single-runtime.sh : el_runtime.c must not be FORKED (a lagging copy +# shipped to prod and dropped learned hebb edges). +# check-runtime-growth.sh : el_runtime.c must not GROW (it is a 2026-05-03 +# build shim that was never retired; see BUDGET). +echo "→ Runtime guards..." +bash "$ROOT/scripts/check-single-runtime.sh" +bash "$ROOT/scripts/check-runtime-growth.sh" + # If elc isn't built yet, skip with a warning rather than blocking if [ ! -x "$ELC" ]; then echo "⚠ elc not found at lang/dist/platform/elc — skipping pre-commit tests" diff --git a/lang/AGENTS.md b/lang/AGENTS.md index 4d9c0fb..0bbee99 100644 --- a/lang/AGENTS.md +++ b/lang/AGENTS.md @@ -97,6 +97,8 @@ The runtime is native El (`runtime/*.el`) over a C OS-boundary. **Status (verifi Choose the file by concern: engram store ops → `engram_store.c`; index → `engram_vindex.c`; geometry/priming → `engram_geometry.c`; reasoning → `engram_reason.c`; grounding/consistency → `engram_verify.c`; think/stance → `engram_cognition.c`. **If no existing file owns it, create one** — add the `.c` to `runtime/SOURCES` (one line) and every build path picks it up. For a builtin that belongs to a downstream program rather than the runtime, declare `c_source "path/to/file.c"` in that program's `manifest.el`; `elb` already links it (`parse_manifest_c_sources`, `lang/elb.el:82`). +> **`el_runtime.c` is on a ratchet and will reject your commit.** `runtime/BUDGET` caps it at its current line count *with no headroom*, and separately caps the number of `engram_*`/`eg_*`/`cog_*` function definitions in it. `scripts/check-runtime-growth.sh` enforces both in CI and in `.githooks/pre-commit`. **The numbers may only ever go down — do not raise them.** Every other runtime file is deliberately uncapped, because that is where the code is supposed to go. When you move code *out*, lower the numbers in the same commit; the guard tells you the new values. + When you add a C builtin (verbatim-emit recipe — the El name is emitted as the exact C symbol; `builtin_arity` is an arity guard only, not a dispatch table): 1. Implement the C function in the **concern-owning `.c`** (and declare it in that file's `.h`). Add the file to `runtime/SOURCES` if it is new. Only put it in `el_runtime.c` if it is genuinely EL core (val/str/map/list/arena) — that is ~8% of what is in there today. 2. Add a `__`-prefixed thin wrapper in `el_seed.c` and declare it in `el_seed.h`. diff --git a/lang/runtime/BUDGET b/lang/runtime/BUDGET new file mode 100644 index 0000000..0113d5b --- /dev/null +++ b/lang/runtime/BUDGET @@ -0,0 +1,39 @@ +# BUDGET — a RATCHET on lang/runtime/el_runtime.c. Enforced by +# scripts/check-runtime-growth.sh. These numbers may only ever go DOWN. +# +# WHY THIS FILE EXISTS +# -------------------- +# scripts/check-single-runtime.sh guards against el_runtime.c being COPIED. +# Nothing guarded against it GROWING. It grew from 10,607 lines to 20,527 — +# 94% — in 3.5 months, while under an explicit commit-message promise that it +# was a temporary shim about to be deleted. +# +# It grew because lang/AGENTS.md told every agent to grow it: it claimed +# el_runtime.c was "the authoritative single-file link target" and that a new +# C builtin "must live there to be linkable". That is false — placement is a +# link-time concern, `builtin_arity` is an arity guard not a dispatch table, +# and the shipped elc already links from ten translation units. The claim is +# corrected, and this file is the mechanism that keeps it corrected. +# +# THIS IS A RATCHET, NOT A LIMIT +# ------------------------------ +# The budget is set at the CURRENT size. There is no headroom, deliberately. +# The file cannot grow by even one line. Any new code goes in the .c that owns +# the concern — that is the whole point, and every other runtime file is +# deliberately UNCAPPED. +# +# When you move code OUT, lower the number in the same commit. The guard tells +# you to when you have earned it. +# +# FORMAT: — `#` comments and blank lines ignored. + +# Maximum lines in lang/runtime/el_runtime.c. +# 2026-08-16: 20,527 — the high-water mark, ratcheted from here. +max_lines 20527 + +# Maximum top-level engram/eg_/cog_ function definitions in el_runtime.c. +# ~47.5% of the file is engram code, and engram already owns six dedicated +# sibling files (engram_{store,vindex,geometry,reason,verify,cognition}.c). +# Every one of these belongs in one of them. This is the Stage 3 scoreboard. +# 2026-08-16: 279. +max_engram_fns 279 diff --git a/scripts/check-runtime-growth.sh b/scripts/check-runtime-growth.sh new file mode 100755 index 0000000..eb140f8 --- /dev/null +++ b/scripts/check-runtime-growth.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# check-runtime-growth.sh — GROWTH guard for lang/runtime/el_runtime.c. +# +# Sibling to scripts/check-single-runtime.sh. That one guards against the file +# being COPIED (a lagging fork shipped to prod and dropped learned hebb edges). +# Nothing guarded against it GROWING — so it grew from 10,607 to 20,527 lines in +# 3.5 months, while under an explicit commit-message promise that it was a +# temporary shim about to be deleted. +# +# This enforces the RATCHET in lang/runtime/BUDGET: the numbers may only go down. +# +# It also checks two invariants that keep the multi-file runtime honest: +# * every .c in lang/runtime/ is either in SOURCES or explicitly optional +# * lang/install.sh's hardcoded download list matches SOURCES +# +# Exits non-zero on any violation. Run from anywhere; resolves the repo root. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +RUNTIME_DIR="lang/runtime" +TARGET="$RUNTIME_DIR/el_runtime.c" +BUDGET_FILE="$RUNTIME_DIR/BUDGET" +SOURCES_FILE="$RUNTIME_DIR/SOURCES" + +FAIL=0 + +for f in "$TARGET" "$BUDGET_FILE" "$SOURCES_FILE"; do + if [ ! -f "$f" ]; then + echo "FATAL: required file missing: $f" >&2 + exit 1 + fi +done + +budget() { + local key="$1" + sed -e 's/#.*//' "$BUDGET_FILE" | awk -v k="$key" '$1==k {print $2; found=1} END{if(!found) exit 1}' +} + +MAX_LINES="$(budget max_lines)" || { echo "FATAL: no 'max_lines' in $BUDGET_FILE" >&2; exit 1; } +MAX_ENGRAM="$(budget max_engram_fns)" || { echo "FATAL: no 'max_engram_fns' in $BUDGET_FILE" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# The message every failure prints. The guard that existed before this one told +# you what was wrong but not where the code should go — so it was easy to +# "fix" by arguing with the guard. This one names the destination. +# --------------------------------------------------------------------------- +where_it_goes() { + cat >&2 <<'MSG' + + WHERE THE CODE ACTUALLY GOES + ---------------------------- + Placement is a LINK-TIME concern. The compiler cannot tell which .c a symbol + came from: `builtin_arity` in el-compiler/src/codegen.el maps NAME -> ARITY + INT only, the El name is emitted as the exact C symbol, and `ld` resolves it. + The SHIPPED compiler already links from ten translation units — check it: + + nm lang/dist/platform/elc | grep -E 'T _(engram_think|vindex_insert)' + + So a builtin defined in a sibling .c is EXACTLY as linkable as one defined in + el_runtime.c. Pick the file that owns the concern: + + engram store ops ......... lang/runtime/engram_store.c + ANN / vector index ....... lang/runtime/engram_vindex.c + geometry, priming ........ lang/runtime/engram_geometry.c + reasoning operators ...... lang/runtime/engram_reason.c + grounding, consistency ... lang/runtime/engram_verify.c + think, stance ............ lang/runtime/engram_cognition.c + + No existing file owns it? Create one, add ONE line to lang/runtime/SOURCES, + and every build path picks it up. Every runtime file EXCEPT el_runtime.c is + deliberately uncapped. + + Belongs to a downstream program, not the runtime? Declare + `c_source "path/to/file.c"` in that program's manifest.el — elb already links + it (parse_manifest_c_sources, lang/elb.el:82). + + See lang/AGENTS.md "Where a new C builtin goes". +MSG +} + +# --- 1. Line-count ratchet --------------------------------------------------- +LINES="$(wc -l < "$TARGET" | tr -d ' ')" +if [ "$LINES" -gt "$MAX_LINES" ]; then + echo "FAIL: $TARGET grew past its budget." >&2 + echo " now: $LINES lines" >&2 + echo " budget: $MAX_LINES lines (lang/runtime/BUDGET: max_lines)" >&2 + echo " over by: $((LINES - MAX_LINES))" >&2 + echo "" >&2 + echo "This file is a 2026-05-03 build shim that was scheduled for deletion and" >&2 + echo "never retired. It does not get to grow. Do NOT raise the budget." >&2 + where_it_goes + FAIL=1 +fi + +# --- 2. Engram-concern ratchet ---------------------------------------------- +# ~47.5% of el_runtime.c is engram code, and engram already owns six sibling +# files. This count is the Stage 3 scoreboard: it may only go down. +ENGRAM_FNS="$(grep -cE '^(static +)?[A-Za-z_][A-Za-z0-9_ *]*\b(engram|eg|cog)_[a-z0-9_]+\(' "$TARGET" || true)" +if [ "$ENGRAM_FNS" -gt "$MAX_ENGRAM" ]; then + echo "FAIL: new engram/eg_/cog_ function(s) added to $TARGET." >&2 + echo " now: $ENGRAM_FNS definitions" >&2 + echo " budget: $MAX_ENGRAM (lang/runtime/BUDGET: max_engram_fns)" >&2 + echo "" >&2 + echo "Engram code belongs in the six engram_*.c files that already exist." >&2 + where_it_goes + FAIL=1 +fi + +# --- 3. Ratchet-down nudge (advisory, never fails) --------------------------- +if [ "$LINES" -lt "$MAX_LINES" ]; then + echo "NOTE: $TARGET is $((MAX_LINES - LINES)) lines under budget — lower" >&2 + echo " 'max_lines' to $LINES in $BUDGET_FILE in this same commit, so the" >&2 + echo " ground you gained cannot be quietly given back." >&2 +fi +if [ "$ENGRAM_FNS" -lt "$MAX_ENGRAM" ]; then + echo "NOTE: $((MAX_ENGRAM - ENGRAM_FNS)) engram fn(s) moved out — lower" >&2 + echo " 'max_engram_fns' to $ENGRAM_FNS in $BUDGET_FILE in this same commit." >&2 +fi + +# --- 4. Every runtime .c is accounted for ------------------------------------ +# A new .c that is in neither SOURCES nor the optional list will not be +# compiled by any build path — it would be silently dead. Catch that here. +OPTIONAL_RE='^(el_android|el_gtk4|el_lvgl|el_sdl2|el_win32|el_runtime_win32|eg_cosine_batch_strategy_ggml|vindex_bench)\.c$' +mapfile -t IN_SOURCES < <(scripts/el-runtime-sources.sh) +for path in "$RUNTIME_DIR"/*.c; do + base="$(basename "$path")" + if printf '%s\n' "${IN_SOURCES[@]}" | grep -qxF "$base"; then continue; fi + if [[ "$base" =~ $OPTIONAL_RE ]]; then continue; fi + echo "FAIL: $path is in neither lang/runtime/SOURCES nor the platform-optional" >&2 + echo " list in this guard. It will not be compiled by any build path." >&2 + echo " Add it to SOURCES (one line), or add it to OPTIONAL_RE here if it" >&2 + echo " is a platform/strategy variant that is linked in deliberately." >&2 + FAIL=1 +done + +# --- 5. install.sh must not drift from SOURCES ------------------------------- +# install.sh runs on machines with no repo checkout, so it cannot call +# el-runtime-sources.sh and has to hardcode the list. That copy is exactly the +# kind of duplicate that silently drifted before — so it is checked, not trusted. +INSTALL_SH="lang/install.sh" +if [ -f "$INSTALL_SH" ]; then + EXPECTED="$(scripts/el-runtime-sources.sh | sort)" + ACTUAL="$(sed -n '/^RUNTIME_SOURCES=(/,/^)/p' "$INSTALL_SH" \ + | grep -oE '[a-z_0-9]+\.c' | sort)" + if [ "$EXPECTED" != "$ACTUAL" ]; then + echo "FAIL: $INSTALL_SH RUNTIME_SOURCES has drifted from $SOURCES_FILE." >&2 + echo " Only in SOURCES: $(comm -23 <(echo "$EXPECTED") <(echo "$ACTUAL") | tr '\n' ' ')" >&2 + echo " Only in install.sh: $(comm -13 <(echo "$EXPECTED") <(echo "$ACTUAL") | tr '\n' ' ')" >&2 + echo " An SDK that ships the wrong set produces a lib/ that cannot link." >&2 + FAIL=1 + fi +fi + +if [ "$FAIL" -ne 0 ]; then + exit 1 +fi + +echo "OK: el_runtime.c within budget ($LINES/$MAX_LINES lines, $ENGRAM_FNS/$MAX_ENGRAM engram fns);" +echo " runtime sources accounted for; install.sh in step with SOURCES." diff --git a/scripts/check-single-runtime.sh b/scripts/check-single-runtime.sh index 6330ece..de14350 100755 --- a/scripts/check-single-runtime.sh +++ b/scripts/check-single-runtime.sh @@ -81,14 +81,14 @@ fi echo "OK: single canonical runtime source — $CANONICAL (no un-allowlisted forks)." # --------------------------------------------------------------------------- -# CI wire-in: -# foundation/el .gitea/workflows/ci-dev.yaml, ci-stage.yaml, sdk-release.yaml -# Add an early step (before the build/publish steps). It must run from the -# REPO ROOT, so override the job's `defaults.run.working-directory: lang`: +# CI wire-in — DONE (2026-08-16). This block used to describe the wire-in as a +# TODO, and it had never been done: the guard existed but ran nowhere, so it +# caught nothing for as long as it has been in the tree. It is now an early step +# in ci-dev.yaml, ci-stage.yaml and sdk-release.yaml (each with +# `working-directory: ${{ github.workspace }}`, since the jobs default to lang/), +# and it runs in .githooks/pre-commit. # -# - name: Guard - single canonical runtime source -# working-directory: ${{ github.workspace }} -# run: bash scripts/check-single-runtime.sh -# -# Also add to .githooks/pre-commit so drift is caught before it is committed. +# Its sibling scripts/check-runtime-growth.sh is wired in at the same points and +# guards the other half of the problem: this script stops el_runtime.c being +# COPIED, that one stops it GROWING. # --------------------------------------------------------------------------- From addd51209f3ad88012f2224c09058845c68e1b75 Mon Sep 17 00:00:00 2001 From: bigmerge Date: Sun, 16 Aug 2026 16:58:18 -0500 Subject: [PATCH 2/2] runtime: extract engram_text.c, and repair 10 harnesses that could not link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First concern moved out of el_runtime.c under the ratchet, and the move is deliberately small: it exists to prove the mechanism end to end before anything large depends on it. engram_text.{c,h} — query tokenization, candidate-token hygiene, word-boundary matching, and the text-damage signature. Four functions, moved verbatim; only `static` was dropped and each doc comment travelled with the code. They touch no EL value type and no engram store type: plain C over / over char buffers. They were never el_runtime.c's business. el_runtime.c 20,527 -> 20,427 lines (BUDGET max_lines ratcheted down) engram fns 279 -> 275 (BUDGET max_engram_fns ratcheted down) The Stage 1 extension point worked as designed: adding the file to lang/runtime/SOURCES was one line, and every build path picked it up. The Stage 2 drift guard then caught that I had NOT added it to install.sh's standalone list — the exact class of drift it was written for, on its first real change, before the commit rather than after a broken SDK shipped. WHY ONLY 100 LINES, AND WHAT ACTUALLY BLOCKS THE REST Measured, not estimated: of 273 engram-domain functions in el_runtime.c (~9,700 lines), only 75 (~1,058 lines) can move today, and they are scattered rather than clustered. The blocker is a single fact: EngramNode, EngramEdge, EngramStore, EngramLayer, EngramWal and EngramIdSlot are typedef'd INSIDE el_runtime.c. No sibling can see them. engram_store.h defines a SEPARATE serializable "node view" struct and maps between the two. So every engram function that takes an EngramNode* — which is most of them, 109 of 273 by direct type reference — cannot compile in engram_store.c until those types move to a shared header. That extraction is the real Stage 3 enabler and it deserves its own change: it touches the most load-bearing struct in the system, and doing it in the same commit as a code move would make a regression impossible to bisect. REPAIRED: 10 engram harnesses that had silently stopped linking Not new breakage from this move — verified against unmodified dev, where el_runtime.c + engram_store.c alone already failed with undefined symbols. They had been dead for as long as el_runtime.c has been calling into the siblings, and nothing noticed because nothing ran them. run_m3_parity, run_m7_traversal, run_m35_hebb_persist, run_interoception_p0..p5 — now build from $(scripts/el-runtime-sources.sh) run_wal_tests — its two TUs #include "el_runtime.c" directly, so it links the SIBLINGS ONLY; adding el_runtime.c to that link line would define every symbol twice (That #include'd .c is worth recording: the runtime does have one, in engram/test/test_wal.c and the generated test_failloud.c.) Verified locally — every one of these was run, not assumed: * m3_parity ............ PASS, incl. ASan+UBSan clean across seed/on/reboot * m7_traversal ......... PASS * m35_hebb_persist ..... PASS (the gate over the original prod hebb bug) * interoception p0..p5 . PASS (all six) * wal_tests ............ 66 passed, 0 failed, + fail-loud exit check * self-host fixpoint ... byte-identical, AND the emitted C is byte-identical to the pre-move compiler output — the move changes nothing the compiler produces * engram/src/server.el . compiles and links * native suites ........ 8 of 13, unchanged from before the move; the same 5 pre-existing failures, no regression * both runtime guards .. green at the new, lower budget Also fixes a block comment left unterminated by the extraction (the deleted range carried its closing */), restoring the compile to its single pre-existing -Wcomment warning. --- engram/test/run_interoception_p0.sh | 20 +++-- engram/test/run_interoception_p1.sh | 20 +++-- engram/test/run_interoception_p2.sh | 20 +++-- engram/test/run_interoception_p3.sh | 20 +++-- engram/test/run_interoception_p4.sh | 20 +++-- engram/test/run_interoception_p5.sh | 20 +++-- engram/test/run_m35_hebb_persist.sh | 14 +++- engram/test/run_m3_parity.sh | 14 +++- engram/test/run_m7_traversal.sh | 14 +++- engram/test/run_wal_tests.sh | 13 ++- lang/install.sh | 2 + lang/runtime/BUDGET | 10 ++- lang/runtime/SOURCES | 5 ++ lang/runtime/el_runtime.c | 110 ++----------------------- lang/runtime/engram_text.c | 122 ++++++++++++++++++++++++++++ lang/runtime/engram_text.h | 66 +++++++++++++++ 16 files changed, 319 insertions(+), 171 deletions(-) create mode 100644 lang/runtime/engram_text.c create mode 100644 lang/runtime/engram_text.h diff --git a/engram/test/run_interoception_p0.sh b/engram/test/run_interoception_p0.sh index 99507ab..9ee3ed8 100755 --- a/engram/test/run_interoception_p0.sh +++ b/engram/test/run_interoception_p0.sh @@ -3,10 +3,14 @@ # Throwaway HOME + /tmp only. Never touches ~/.neuron or :8742. set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" -GEO="$HERE/../../lang/runtime/engram_geometry.c" -VIDX="$HERE/../../lang/runtime/engram_vindex.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-p0-XXXXXX)" export HOME="$WORK/home"; mkdir -p "$HOME" @@ -14,8 +18,8 @@ unset ENGRAM_STORE fail=0 echo "== compile (plain) ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p0_emb.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p0" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } +gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p0_emb.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p0" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } D="$WORK/d"; mkdir -p "$D" "$WORK/p0" "$D" || { echo "FAIL: run"; fail=1; } @@ -69,8 +73,8 @@ PY echo echo "== ASan+UBSan ==" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_interoception_p0_emb.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p0.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -20 "$WORK/san_cc.log"; fail=1; } + -I "$INC" "$HERE/test_interoception_p0_emb.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p0.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -20 "$WORK/san_cc.log"; fail=1; } if [ -x "$WORK/p0.san" ]; then export ASAN_OPTIONS=detect_leaks=0 DS="$WORK/ds"; mkdir -p "$DS" diff --git a/engram/test/run_interoception_p1.sh b/engram/test/run_interoception_p1.sh index 41e5dbe..82591a1 100755 --- a/engram/test/run_interoception_p1.sh +++ b/engram/test/run_interoception_p1.sh @@ -3,10 +3,14 @@ # Throwaway HOME + /tmp only. Never touches ~/.neuron or :8742. set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" -GEO="$HERE/../../lang/runtime/engram_geometry.c" -VIDX="$HERE/../../lang/runtime/engram_vindex.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-p1-XXXXXX)" export HOME="$WORK/home"; mkdir -p "$HOME" @@ -14,8 +18,8 @@ unset ENGRAM_STORE ENGRAM_CONSOLIDATION ENGRAM_CONSOL_CONN_MIN ENGRAM_CONSOL_PER fail=0 echo "== compile ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p1_consol.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p1" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } +gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p1_consol.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p1" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } echo echo "== (a) HEADLINE: hebb accrual curve over N co-activations (flag OFF, pure trunk) ==" @@ -129,8 +133,8 @@ cat "$WORK/off.txt" | sed 's/^/ /' echo echo "== ASan+UBSan (connect + perm + accrual-short) ==" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_interoception_p1_consol.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p1.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } + -I "$INC" "$HERE/test_interoception_p1_consol.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p1.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } if [ -x "$WORK/p1.san" ]; then export ASAN_OPTIONS=detect_leaks=0 DS="$WORK/san"; mkdir -p "$DS" diff --git a/engram/test/run_interoception_p2.sh b/engram/test/run_interoception_p2.sh index 3e209b5..0d2b7a9 100755 --- a/engram/test/run_interoception_p2.sh +++ b/engram/test/run_interoception_p2.sh @@ -3,10 +3,14 @@ # Throwaway HOME + /tmp only. TC defaults to 3600s; we pin it for the math. set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" -GEO="$HERE/../../lang/runtime/engram_geometry.c" -VIDX="$HERE/../../lang/runtime/engram_vindex.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-p2-XXXXXX)" export HOME="$WORK/home"; mkdir -p "$HOME" @@ -15,8 +19,8 @@ unset ENGRAM_STORE fail=0 echo "== compile ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p2_chrono.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p2" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } +gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p2_chrono.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p2" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } sum_wm(){ python3 -c "import json,sys; g=json.load(open('$1')); print(sum(n.get('working_memory_weight',0) for n in g['nodes']))"; } @@ -78,8 +82,8 @@ python3 -c "import sys; sys.exit(0 if abs($OFFWM-1.2)<1e-9 else 1)" \ echo echo "== ASan+UBSan ==" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_interoception_p2_chrono.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p2.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } + -I "$INC" "$HERE/test_interoception_p2_chrono.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p2.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } if [ -x "$WORK/p2.san" ]; then export ASAN_OPTIONS=detect_leaks=0 DS="$WORK/san"; mkdir -p "$DS" diff --git a/engram/test/run_interoception_p3.sh b/engram/test/run_interoception_p3.sh index 26d2fd6..f56596e 100755 --- a/engram/test/run_interoception_p3.sh +++ b/engram/test/run_interoception_p3.sh @@ -3,18 +3,22 @@ # Read-only pure primitive; no store, no flag. Throwaway /tmp only. set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" -GEO="$HERE/../../lang/runtime/engram_geometry.c" -VIDX="$HERE/../../lang/runtime/engram_vindex.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-p3-XXXXXX)" export HOME="$WORK/home"; mkdir -p "$HOME" fail=0 echo "== compile ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p3_drift.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p3" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } +gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p3_drift.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p3" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } "$WORK/p3" > "$WORK/out.txt" 2>&1 || { echo "FAIL run"; cat "$WORK/out.txt"; fail=1; } cat "$WORK/out.txt" | sed 's/^/ /' @@ -52,8 +56,8 @@ PY echo echo "== ASan+UBSan ==" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_interoception_p3_drift.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p3.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } + -I "$INC" "$HERE/test_interoception_p3_drift.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p3.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } if [ -x "$WORK/p3.san" ]; then export ASAN_OPTIONS=detect_leaks=0 "$WORK/p3.san" >/dev/null 2>"$WORK/san.log" diff --git a/engram/test/run_interoception_p4.sh b/engram/test/run_interoception_p4.sh index 9d81bc4..656c646 100755 --- a/engram/test/run_interoception_p4.sh +++ b/engram/test/run_interoception_p4.sh @@ -2,10 +2,14 @@ # M-INTEROCEPTION P4 gate: afferent input counters in act-stats (additive). set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" -GEO="$HERE/../../lang/runtime/engram_geometry.c" -VIDX="$HERE/../../lang/runtime/engram_vindex.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-p4-XXXXXX)" export HOME="$WORK/home"; mkdir -p "$HOME" @@ -13,8 +17,8 @@ unset ENGRAM_STORE fail=0 echo "== compile ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p4_afferent.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p4" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } +gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p4_afferent.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p4" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } "$WORK/p4" > "$WORK/out.txt" 2>&1 || { echo "FAIL run"; cat "$WORK/out.txt"; fail=1; } grep -oE 'aff_[a-z_]+":[0-9]+' "$WORK/out.txt" | sed 's/^/ /' | head -30 @@ -53,8 +57,8 @@ PY echo echo "== ASan+UBSan ==" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_interoception_p4_afferent.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p4.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } + -I "$INC" "$HERE/test_interoception_p4_afferent.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p4.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } if [ -x "$WORK/p4.san" ]; then export ASAN_OPTIONS=detect_leaks=0 "$WORK/p4.san" >/dev/null 2>"$WORK/san.log" diff --git a/engram/test/run_interoception_p5.sh b/engram/test/run_interoception_p5.sh index 90e6c81..a21e85e 100755 --- a/engram/test/run_interoception_p5.sh +++ b/engram/test/run_interoception_p5.sh @@ -2,10 +2,14 @@ # M-INTEROCEPTION P5 gate: dream-recall builtin engram_dreams_json (honesty rail). set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" -GEO="$HERE/../../lang/runtime/engram_geometry.c" -VIDX="$HERE/../../lang/runtime/engram_vindex.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-p5-XXXXXX)" export HOME="$WORK/home"; mkdir -p "$HOME" @@ -13,8 +17,8 @@ unset ENGRAM_STORE fail=0 echo "== compile ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p5_dreams.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p5" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } +gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p5_dreams.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p5" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } D="$WORK/d"; mkdir -p "$D" "$WORK/p5" "$D" > "$WORK/out.txt" 2>&1 || { echo "FAIL run"; cat "$WORK/out.txt"; fail=1; } @@ -57,8 +61,8 @@ PY echo echo "== ASan+UBSan ==" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_interoception_p5_dreams.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p5.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } + -I "$INC" "$HERE/test_interoception_p5_dreams.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p5.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } if [ -x "$WORK/p5.san" ]; then export ASAN_OPTIONS=detect_leaks=0 DS="$WORK/ds"; mkdir -p "$DS" diff --git a/engram/test/run_m35_hebb_persist.sh b/engram/test/run_m35_hebb_persist.sh index 1ce7dd3..1691aae 100755 --- a/engram/test/run_m35_hebb_persist.sh +++ b/engram/test/run_m35_hebb_persist.sh @@ -6,8 +6,14 @@ # Writes ONLY under a throwaway /tmp dir with a throwaway HOME. set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-m35-XXXXXX)" BIN="$WORK/m35" @@ -17,7 +23,7 @@ unset ENGRAM_STORE fail=0 echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m35_hebb_persist.c) ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_m35_hebb_persist.c" "$RT" "$ST" -lcurl -o "$BIN" 2>"$WORK/cc.log" +gcc -O1 -std=c11 -I "$INC" "$HERE/test_m35_hebb_persist.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$BIN" 2>"$WORK/cc.log" if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi echo @@ -139,7 +145,7 @@ echo echo "== 5) ASan+UBSan build, exercise the full persist+reboot flow (leaks off — harness intentionally leaks el_strdup) ==" SANBIN="$WORK/m35.san" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_m35_hebb_persist.c" "$RT" "$ST" -lcurl -o "$SANBIN" 2>"$WORK/san_cc.log" + -I "$INC" "$HERE/test_m35_hebb_persist.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$SANBIN" 2>"$WORK/san_cc.log" if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else export ASAN_OPTIONS=detect_leaks=0 DSAN="$WORK/san"; mkdir -p "$DSAN" diff --git a/engram/test/run_m3_parity.sh b/engram/test/run_m3_parity.sh index 6eadaee..c7d533a 100755 --- a/engram/test/run_m3_parity.sh +++ b/engram/test/run_m3_parity.sh @@ -4,8 +4,14 @@ # Writes ONLY under a throwaway /tmp dir with a throwaway HOME + ENGRAM_DATA_DIR. set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-m3-XXXXXX)" DATA="$WORK/data"; mkdir -p "$DATA" @@ -17,7 +23,7 @@ unset ENGRAM_STORE fail=0 echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m3_parity.c) ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_m3_parity.c" "$RT" "$ST" -lcurl -o "$BIN" 2>"$WORK/cc.log" +gcc -O1 -std=c11 -I "$INC" "$HERE/test_m3_parity.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$BIN" 2>"$WORK/cc.log" if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi grep -i warning "$WORK/cc.log" | grep -iE 'engram_store|eg_store|eg_load|scan_nodes|scan_edges' && echo "(warnings in M3 code above)" || true @@ -106,7 +112,7 @@ echo echo "== 5) ASan+UBSan build, exercise M3 scan/boot/hooks (leaks off — harness intentionally leaks el_strdup) ==" SANBIN="$WORK/m3.san" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_m3_parity.c" "$RT" "$ST" -lcurl -o "$SANBIN" 2>"$WORK/san_cc.log" + -I "$INC" "$HERE/test_m3_parity.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$SANBIN" 2>"$WORK/san_cc.log" if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else export ASAN_OPTIONS=detect_leaks=0 DATA2="$WORK/data2"; mkdir -p "$DATA2" diff --git a/engram/test/run_m7_traversal.sh b/engram/test/run_m7_traversal.sh index 693a9fe..1340704 100755 --- a/engram/test/run_m7_traversal.sh +++ b/engram/test/run_m7_traversal.sh @@ -6,8 +6,14 @@ # Writes ONLY under a throwaway /tmp dir with a throwaway HOME. set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-m7-XXXXXX)" DATA="$WORK/data"; mkdir -p "$DATA" @@ -22,7 +28,7 @@ unset ENGRAM_STORE fail=0 echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m7_traversal.c) ==" -gcc -O2 -std=c11 -I "$INC" "$HERE/test_m7_traversal.c" "$RT" "$ST" -lcurl -lm -o "$BIN" 2>"$WORK/cc.log" +gcc -O2 -std=c11 -I "$INC" "$HERE/test_m7_traversal.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$BIN" 2>"$WORK/cc.log" if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi echo " ok: compiled" @@ -115,7 +121,7 @@ echo echo "== 3) ASan+UBSan clean across parity + a small perf loop (leaks off — harness intentionally leaks el_strdup) ==" SANBIN="$WORK/m7.san" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_m7_traversal.c" "$RT" "$ST" -lcurl -lm -o "$SANBIN" 2>"$WORK/san_cc.log" + -I "$INC" "$HERE/test_m7_traversal.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$SANBIN" 2>"$WORK/san_cc.log" if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else export ASAN_OPTIONS=detect_leaks=0 D2="$WORK/data2"; mkdir -p "$D2" diff --git a/engram/test/run_wal_tests.sh b/engram/test/run_wal_tests.sh index eddd791..ae3b0cb 100755 --- a/engram/test/run_wal_tests.sh +++ b/engram/test/run_wal_tests.sh @@ -3,8 +3,17 @@ set -e HERE="$(cd "$(dirname "$0")" && pwd)" REL="$HERE/../../lang/runtime" +# test_wal.c and test_failloud.c #include "el_runtime.c" directly, so el_runtime.c +# is already IN the translation unit — link the SIBLINGS only, or every symbol in +# it is defined twice. The siblings are still required: el_runtime.c calls into +# all six engram TUs. (lang/runtime/SOURCES is the source of truth.) +RTSIB="$("$HERE/../../scripts/el-runtime-sources.sh" "$REL" | grep -v '/el_runtime\.c$')" +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi cc -O2 -fbracket-depth=1024 -Wno-parentheses-equality -I"$REL" \ - "$HERE/test_wal.c" -lcurl -lpthread -o /tmp/test_wal + "$HERE/test_wal.c" $RTSIB $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/test_wal HOME=/tmp/engram-throwaway-home /tmp/test_wal # Fail-loud data-dir check (must exit 1 with a FATAL line): cat > /tmp/test_failloud.c <<'C' @@ -12,5 +21,5 @@ cat > /tmp/test_failloud.c <<'C' int main(void){ unsetenv("ENGRAM_DATA_DIR"); unsetenv("HOME"); engram_resolve_data_dir(); printf("REACHED\n"); return 0; } C -cc -O2 -fbracket-depth=1024 -Wno-parentheses-equality -I"$REL" /tmp/test_failloud.c -lcurl -lpthread -o /tmp/test_failloud +cc -O2 -fbracket-depth=1024 -Wno-parentheses-equality -I"$REL" /tmp/test_failloud.c $RTSIB $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/test_failloud if env -u HOME -u ENGRAM_DATA_DIR /tmp/test_failloud; then echo "FAIL: should have exited"; exit 1; else echo "[PASS] fail-loud exit on unresolvable HOME"; fi diff --git a/lang/install.sh b/lang/install.sh index 9317853..ac895b6 100644 --- a/lang/install.sh +++ b/lang/install.sh @@ -60,12 +60,14 @@ RUNTIME_SOURCES=( el_runtime.c el_seed.c engram_store.c engram_vindex.c engram_geometry.c engram_reason.c engram_verify.c engram_cognition.c + engram_text.c eg_cosine_batch.c eg_cosine_batch_strategy_cpu.c ) RUNTIME_HEADERS=( el_runtime.h el_seed.h engram_store.h engram_vindex.h engram_geometry.h engram_reason.h engram_verify.h engram_cognition.h + engram_text.h eg_cosine_batch.h eg_cosine_batch_strategy.h ) diff --git a/lang/runtime/BUDGET b/lang/runtime/BUDGET index 0113d5b..6706a5d 100644 --- a/lang/runtime/BUDGET +++ b/lang/runtime/BUDGET @@ -28,12 +28,14 @@ # FORMAT: — `#` comments and blank lines ignored. # Maximum lines in lang/runtime/el_runtime.c. -# 2026-08-16: 20,527 — the high-water mark, ratcheted from here. -max_lines 20527 +# 2026-08-16: 20,527 — the high-water mark. +# 2026-08-16: 20,427 — engram_text.c extracted (tokenize, token hygiene, +# word-boundary match, damage signature). Ratcheted down. +max_lines 20427 # Maximum top-level engram/eg_/cog_ function definitions in el_runtime.c. # ~47.5% of the file is engram code, and engram already owns six dedicated # sibling files (engram_{store,vindex,geometry,reason,verify,cognition}.c). # Every one of these belongs in one of them. This is the Stage 3 scoreboard. -# 2026-08-16: 279. -max_engram_fns 279 +# 2026-08-16: 279 -> 275 (4 moved to engram_text.c). +max_engram_fns 275 diff --git a/lang/runtime/SOURCES b/lang/runtime/SOURCES index e44c152..d825445 100644 --- a/lang/runtime/SOURCES +++ b/lang/runtime/SOURCES @@ -44,6 +44,11 @@ engram_reason.c engram_verify.c engram_cognition.c +# --- Text: tokenization, token hygiene, damage signature --------------------- +# Extracted from el_runtime.c 2026-08-16. Plain C over / — +# touches no EL value type and no engram store type. New text helpers go HERE. +engram_text.c + # --- Vector math: batch cosine + its CPU strategy ---------------------------- # The ggml strategy (eg_cosine_batch_strategy_ggml.c) is an OPTIONAL swap-in and # is deliberately NOT in the default set — it needs ggml headers. Link it in diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index 0bd532c..441be58 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -8638,6 +8638,7 @@ static char* engram_first_n_chars(const char* s, size_t n) { * mutation (node/edge create, forget) is mirrored through the store's * WAL-logged API so neuron.egm/neuron.wal stay authoritative. * ══════════════════════════════════════════════════════════════════════════ */ +#include "engram_text.h" /* text: tokenize, token hygiene, loss signature */ #include "engram_store.h" #include "engram_vindex.h" /* M8: ANN (HNSW) index for activation seed selection */ #include "engram_geometry.h" /* M9: centered relational-neighborhood geometry (priming) */ @@ -9061,31 +9062,9 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) { * RIGHT NOW" — which is the regression question, and the one that * would have caught this in a day instead of two months. * - * SIGNATURE. Conservative on purpose — a false alarm that cries corruption - * over ordinary punctuation is worse than useless. Two patterns, both of - * which are essentially absent from well-formed English prose: - * (a) alnum '?' alnum — "na?ve", "caf?s", "don?t". A real question mark - * never sits between two word characters. - * (b) ' ? ' followed by a lowercase letter — a lost em/en dash. A real - * question mark is not preceded by a space, and - * what follows one starts a new sentence. - * Deliberately NOT flagged: a trailing '?' after a word, '? ' before a - * capital, or '?' at end of string — all legitimate. This under-counts (it - * cannot see a mangled 'café ' where the '?' landed before a space), so the - * census is a floor on the damage, never an exaggeration of it. */ -static int eg_text_loss_signature(const char* s) { - if (!s) return 0; - for (const char* p = s; *p; p++) { - if (*p != '?') continue; - unsigned char prev = (p == s) ? 0 : (unsigned char)p[-1]; - unsigned char next = (unsigned char)p[1]; - /* (a) sandwiched between word characters. */ - if (isalnum(prev) && isalnum(next)) return 1; - /* (b) spaced, with lowercase continuation — a lost dash. */ - if (prev == ' ' && next == ' ' && islower((unsigned char)p[2])) return 1; - } - return 0; -} + * The SIGNATURE itself (eg_text_loss_signature) moved to engram_text.c on + * 2026-08-16 — it is plain C over and touches nothing in here. The + * stock/flow gauges below stay, because they touch store and EL value types. */ /* Damaged-node creations since process start. See the block comment above. */ static int64_t _eg_txt_write_damaged = 0; @@ -9697,37 +9676,7 @@ static int istr_contains(const char* hay, const char* needle) { * fix landed but never reached this release runtime — the copy the engram * binary actually builds against.) */ #define ENGRAM_MAX_QTOKENS 32 -#define ENGRAM_QTOK_LEN 256 - -/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct - * (case-insensitive) tokens. Returns the token count. Over-long tokens are - * truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */ -static int engram_tokenize_query(const char* q, - char toks[][ENGRAM_QTOK_LEN], int maxtok) { - int n = 0; - if (!q) return 0; - const char* p = q; - while (*p && n < maxtok) { - while (*p && isspace((unsigned char)*p)) p++; - if (!*p) break; - char buf[ENGRAM_QTOK_LEN]; - size_t tl = 0; - while (*p && !isspace((unsigned char)*p)) { - if (tl < sizeof(buf) - 1) buf[tl++] = *p; - p++; - } - buf[tl] = '\0'; - if (tl == 0) continue; - int dup = 0; - for (int s = 0; s < n; s++) { - if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; } - } - if (dup) continue; - memcpy(toks[n], buf, tl + 1); - n++; - } - return n; -} +/* ENGRAM_QTOK_LEN and engram_tokenize_query moved to engram_text.h/.c. */ /* Count how many of the ntok distinct query tokens appear (case-insensitive) * in the node's content, label, or tags. 0 == no match. */ @@ -16389,29 +16338,6 @@ el_val_t engram_label_df(el_val_t term) { #define ENGRAM_ST_TOKLEN 64 #define ENGRAM_ST_SCANCHARS 400 -/* Trim leading/trailing non-alphanumerics, then accept only tokens whose core - * is alphanumeric plus '-' and '_' with at least 3 letters. This subsumes the - * quoted-title guard (2026-07-25) and the "