diff --git a/.gitea/workflows/ci-dev.yaml b/.gitea/workflows/ci-dev.yaml index d4194b8..373258e 100644 --- a/.gitea/workflows/ci-dev.yaml +++ b/.gitea/workflows/ci-dev.yaml @@ -19,6 +19,16 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # Guards must run from the REPO ROOT — override the job's + # defaults.run.working-directory: lang + - name: Guard - single canonical runtime source + working-directory: ${{ github.workspace }} + run: bash scripts/check-single-runtime.sh + + - name: Guard - el_runtime.c growth budget + working-directory: ${{ github.workspace }} + run: bash scripts/check-runtime-growth.sh + - name: Install build dependencies run: | apt-get update -qq @@ -41,7 +51,7 @@ jobs: gcc -O2 \ -I runtime \ dist/elc-gen2.c \ - runtime/el_runtime.c \ + $(../scripts/el-runtime-sources.sh runtime) \ -lcurl -lssl -lcrypto -lpthread -lm \ -o dist/platform/elc chmod +x dist/platform/elc @@ -56,7 +66,7 @@ jobs: gcc -O2 \ -I runtime \ dist/elb.c \ - runtime/el_runtime.c \ + $(../scripts/el-runtime-sources.sh runtime) \ -lcurl -lssl -lcrypto -lpthread -lm \ -o dist/bin/elb chmod +x dist/bin/elb @@ -87,14 +97,20 @@ jobs: bash tests/html_sanitizer/run.sh # Native El test suites (elc --test, compile-link-run) - # el_runtime.c is precompiled to .o once and reused by all 8 modules. - - name: Precompile el_runtime.o + # The runtime is MULTI-FILE (see lang/runtime/SOURCES). Every .c is compiled + # once into /tmp/libel.a and reused by all 8 test modules — compile-once, + # link-many, as prescribed in DESIGN.md. Linking el_runtime.c alone fails + # at `ld`: it calls into all six engram sibling TUs. + - name: Precompile runtime into libel.a run: | set -euo pipefail RUNTIME="$(pwd)/runtime" - gcc -O2 -c -I "$RUNTIME" "$RUNTIME/el_runtime.c" \ - -o /tmp/el_runtime.o - echo "el_runtime.o compiled" + rm -rf /tmp/elrt && mkdir -p /tmp/elrt + for src in $(../scripts/el-runtime-sources.sh --check "$RUNTIME"); do + gcc -O2 -c -I "$RUNTIME" "$src" -o "/tmp/elrt/$(basename "${src%.c}").o" + done + ar rcs /tmp/libel.a /tmp/elrt/*.o + echo "libel.a built from $(ls /tmp/elrt/*.o | wc -l) translation units" - name: Run tests - native (core) run: | @@ -102,7 +118,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_core.el > /tmp/el_native_core.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c /tmp/el_runtime.o \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c /tmp/libel.a \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_core /tmp/el_native_core @@ -112,7 +128,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_text.el > /tmp/el_native_text.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c /tmp/el_runtime.o \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c /tmp/libel.a \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_text /tmp/el_native_text @@ -122,7 +138,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_string.el > /tmp/el_native_string.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c /tmp/el_runtime.o \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c /tmp/libel.a \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_string /tmp/el_native_string @@ -132,7 +148,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_math.el > /tmp/el_native_math.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c /tmp/el_runtime.o \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c /tmp/libel.a \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_math /tmp/el_native_math @@ -142,7 +158,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_state.el > /tmp/el_native_state.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c /tmp/el_runtime.o \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c /tmp/libel.a \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_state /tmp/el_native_state @@ -152,7 +168,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_time.el > /tmp/el_native_time.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c /tmp/el_runtime.o \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c /tmp/libel.a \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_time /tmp/el_native_time @@ -162,7 +178,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_json.el > /tmp/el_native_json.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c /tmp/el_runtime.o \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c /tmp/libel.a \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_json /tmp/el_native_json @@ -172,7 +188,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_env.el > /tmp/el_native_env.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c /tmp/el_runtime.o \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c /tmp/libel.a \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_env /tmp/el_native_env @@ -182,7 +198,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_fs.el > /tmp/el_native_fs.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c /tmp/el_runtime.o \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c /tmp/libel.a \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_fs /tmp/el_native_fs @@ -306,8 +322,9 @@ jobs: FROM ${BASE} COPY dist/platform/elc /opt/el/dist/platform/elc COPY dist/bin/elb /opt/el/dist/bin/elb - COPY runtime/el_runtime.c /opt/el/runtime/el_runtime.c - COPY runtime/el_runtime.h /opt/el/runtime/el_runtime.h + # Whole runtime link set — el_runtime.c alone does not link (it calls + # into the six engram sibling TUs). See lang/runtime/SOURCES. + COPY runtime/ /opt/el/runtime/ COPY runtime/el_runtime.js /opt/el/runtime/el_runtime.js RUN chmod +x /opt/el/dist/platform/elc /opt/el/dist/bin/elb EOF diff --git a/.gitea/workflows/ci-stage.yaml b/.gitea/workflows/ci-stage.yaml index 3280050..f87d731 100644 --- a/.gitea/workflows/ci-stage.yaml +++ b/.gitea/workflows/ci-stage.yaml @@ -29,6 +29,16 @@ jobs: fi echo "Source branch check passed: ${SOURCE} -> stage" + # Guards must run from the REPO ROOT — override the job's + # defaults.run.working-directory: lang + - name: Guard - single canonical runtime source + working-directory: ${{ github.workspace }} + run: bash scripts/check-single-runtime.sh + + - name: Guard - el_runtime.c growth budget + working-directory: ${{ github.workspace }} + run: bash scripts/check-runtime-growth.sh + - name: Install build dependencies run: | apt-get update -qq @@ -48,7 +58,7 @@ jobs: gcc -O2 \ -I runtime \ dist/elc-gen2.c \ - runtime/el_runtime.c \ + $(../scripts/el-runtime-sources.sh runtime) \ -lcurl -lssl -lcrypto -lpthread -lm \ -o dist/platform/elc chmod +x dist/platform/elc @@ -86,7 +96,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_core.el > /tmp/el_native_core.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_core /tmp/el_native_core @@ -96,7 +106,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_text.el > /tmp/el_native_text.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_text /tmp/el_native_text @@ -106,7 +116,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_string.el > /tmp/el_native_string.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_string /tmp/el_native_string @@ -116,7 +126,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_math.el > /tmp/el_native_math.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_math /tmp/el_native_math @@ -126,7 +136,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_state.el > /tmp/el_native_state.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_state /tmp/el_native_state @@ -136,7 +146,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_time.el > /tmp/el_native_time.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_time /tmp/el_native_time @@ -146,7 +156,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_json.el > /tmp/el_native_json.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_json /tmp/el_native_json @@ -156,7 +166,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_env.el > /tmp/el_native_env.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_env /tmp/el_native_env @@ -166,7 +176,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_fs.el > /tmp/el_native_fs.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_fs /tmp/el_native_fs @@ -178,7 +188,7 @@ jobs: gcc -O2 \ -I runtime \ dist/elb.c \ - runtime/el_runtime.c \ + $(../scripts/el-runtime-sources.sh runtime) \ -lcurl -lssl -lcrypto -lpthread -lm \ -o dist/bin/elb chmod +x dist/bin/elb @@ -290,8 +300,9 @@ jobs: FROM ${BASE} COPY dist/platform/elc /opt/el/dist/platform/elc COPY dist/bin/elb /opt/el/dist/bin/elb - COPY runtime/el_runtime.c /opt/el/runtime/el_runtime.c - COPY runtime/el_runtime.h /opt/el/runtime/el_runtime.h + # Whole runtime link set — el_runtime.c alone does not link (it calls + # into the six engram sibling TUs). See lang/runtime/SOURCES. + COPY runtime/ /opt/el/runtime/ COPY runtime/el_runtime.js /opt/el/runtime/el_runtime.js RUN chmod +x /opt/el/dist/platform/elc /opt/el/dist/bin/elb EOF diff --git a/.gitea/workflows/sdk-release.yaml b/.gitea/workflows/sdk-release.yaml index d8eb0c7..adaa353 100644 --- a/.gitea/workflows/sdk-release.yaml +++ b/.gitea/workflows/sdk-release.yaml @@ -29,6 +29,16 @@ jobs: fi echo "Source branch check passed: ${SOURCE} -> main" + # Guards must run from the REPO ROOT — override the job's + # defaults.run.working-directory: lang + - name: Guard - single canonical runtime source + working-directory: ${{ github.workspace }} + run: bash scripts/check-single-runtime.sh + + - name: Guard - el_runtime.c growth budget + working-directory: ${{ github.workspace }} + run: bash scripts/check-runtime-growth.sh + - name: Install build dependencies run: | apt-get update -qq @@ -49,7 +59,7 @@ jobs: gcc -O2 \ -I runtime \ dist/elc-gen2.c \ - runtime/el_runtime.c \ + $(../scripts/el-runtime-sources.sh runtime) \ -lcurl -lssl -lcrypto -lpthread -lm \ -o dist/platform/elc chmod +x dist/platform/elc @@ -64,7 +74,7 @@ jobs: gcc -O2 \ -I runtime \ dist/elb.c \ - runtime/el_runtime.c \ + $(../scripts/el-runtime-sources.sh runtime) \ -lcurl -lssl -lcrypto -lpthread -lm \ -o dist/bin/elb chmod +x dist/bin/elb @@ -123,7 +133,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_core.el > /tmp/el_native_core.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_core /tmp/el_native_core @@ -133,7 +143,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_text.el > /tmp/el_native_text.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_text /tmp/el_native_text @@ -143,7 +153,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_string.el > /tmp/el_native_string.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_string /tmp/el_native_string @@ -153,7 +163,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_math.el > /tmp/el_native_math.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_math /tmp/el_native_math @@ -163,7 +173,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_state.el > /tmp/el_native_state.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_state /tmp/el_native_state @@ -173,7 +183,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_time.el > /tmp/el_native_time.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_time /tmp/el_native_time @@ -183,7 +193,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_json.el > /tmp/el_native_json.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_json /tmp/el_native_json @@ -193,7 +203,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_env.el > /tmp/el_native_env.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_env /tmp/el_native_env @@ -203,7 +213,7 @@ jobs: ELC="$(pwd)/dist/platform/elc" RUNTIME="$(pwd)/runtime" "$ELC" --test tests/native/test_fs.el > /tmp/el_native_fs.c - gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c "$RUNTIME/el_runtime.c" \ + gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \ -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_fs /tmp/el_native_fs @@ -216,10 +226,17 @@ jobs: cp lang/dist/platform/elc dist/sdk/bin/elc cp lang/dist/bin/elb dist/sdk/bin/elb cp lang/dist/bin/epm dist/sdk/bin/epm - cp lang/runtime/el_runtime.c dist/sdk/runtime/ - cp lang/runtime/el_runtime.h dist/sdk/runtime/ - cp lang/runtime/engram_store.c dist/sdk/runtime/ - cp lang/runtime/engram_store.h dist/sdk/runtime/ + # Ship the WHOLE runtime link set, not el_runtime.c alone. el_runtime.c + # #includes six engram headers and calls into all six sibling .c files, + # so an SDK carrying only el_runtime.c{,.h} + engram_store.c{,.h} cannot + # link — downstream `ld` fails on engram_ground_json, eg_find_relation, + # cog_assert_two_axis and friends. lang/runtime/SOURCES is the source of + # truth; --check makes a missing file fail the release loudly. + for f in $(scripts/el-runtime-sources.sh --check) \ + $(scripts/el-runtime-sources.sh --headers --check); do + cp "lang/runtime/${f}" dist/sdk/runtime/ + done + cp lang/runtime/SOURCES dist/sdk/runtime/ cp lang/runtime/*.el dist/sdk/runtime/ tar -czf dist/el-sdk-latest.tar.gz -C dist/sdk . echo "SDK tarball bundled: dist/el-sdk-latest.tar.gz" @@ -274,12 +291,16 @@ jobs: "${GITEA_API}/repos/${REPO}/releases/${RELEASE_ID}/assets" } - # Per-file assets (downstream CI needs these individually) + # Per-file assets (downstream CI needs these individually). + # lang/install.sh downloads every one of these by name — the list is + # lang/runtime/SOURCES. Shipping el_runtime.c alone produced a lib/ + # that could not link; that is the bug this loop closes. upload_asset lang/dist/platform/elc elc - upload_asset lang/runtime/el_runtime.c el_runtime.c - upload_asset lang/runtime/el_runtime.h el_runtime.h - upload_asset lang/runtime/engram_store.c engram_store.c - upload_asset lang/runtime/engram_store.h engram_store.h + for f in $(scripts/el-runtime-sources.sh --check) \ + $(scripts/el-runtime-sources.sh --headers --check); do + upload_asset "lang/runtime/${f}" "${f}" + done + upload_asset lang/runtime/SOURCES SOURCES # SDK bundle and installer binary upload_asset dist/el-sdk-latest.tar.gz el-sdk-latest.tar.gz @@ -350,6 +371,26 @@ jobs: --version="${VERSION}" \ --source=runtime/el_runtime.js + # el-runtime-src — the COMPLETE runtime link set as one tarball. + # + # The el-runtime-c / el-runtime-h packages above are single files and are + # kept for backward compatibility with consumers that already pull them, + # but they are NOT sufficient to link: el_runtime.c calls into six engram + # sibling translation units. New consumers should pull el-runtime-src and + # link everything named in its SOURCES file. + tar -czf /tmp/el-runtime-src.tar.gz \ + -C runtime SOURCES \ + $(../scripts/el-runtime-sources.sh --check) \ + $(../scripts/el-runtime-sources.sh --headers --check) + + gcloud artifacts generic upload \ + --repository=foundation-prod \ + --location=us-central1 \ + --project=neuron-785695 \ + --package=el-runtime-src \ + --version="${VERSION}" \ + --source=/tmp/el-runtime-src.tar.gz + echo "Published El SDK version=${VERSION} to foundation-prod" # Keep key alive for the ci-base rebuild step below # (deleted in that step after docker push) @@ -386,8 +427,9 @@ jobs: FROM ${BASE} COPY dist/platform/elc /opt/el/dist/platform/elc COPY dist/bin/elb /opt/el/dist/bin/elb - COPY runtime/el_runtime.c /opt/el/runtime/el_runtime.c - COPY runtime/el_runtime.h /opt/el/runtime/el_runtime.h + # Whole runtime link set — el_runtime.c alone does not link (it calls + # into the six engram sibling TUs). See lang/runtime/SOURCES. + COPY runtime/ /opt/el/runtime/ COPY runtime/el_runtime.js /opt/el/runtime/el_runtime.js RUN chmod +x /opt/el/dist/platform/elc /opt/el/dist/bin/elb EOF diff --git a/.githooks/pre-commit b/.githooks/pre-commit index bc78c05..c168617 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -9,13 +9,52 @@ LANG_DIR="$ROOT/lang" RUNTIME="$LANG_DIR/runtime" ELC="$LANG_DIR/dist/platform/elc" +# Runtime guards — catch drift and growth before they are committed, not in CI. +# check-single-runtime.sh : el_runtime.c must not be FORKED (a lagging copy +# shipped to prod and dropped learned hebb edges). +# check-runtime-growth.sh : el_runtime.c must not GROW (it is a 2026-05-03 +# build shim that was never retired; see BUDGET). +echo "→ Runtime guards..." +bash "$ROOT/scripts/check-single-runtime.sh" +bash "$ROOT/scripts/check-runtime-growth.sh" + # If elc isn't built yet, skip with a warning rather than blocking if [ ! -x "$ELC" ]; then echo "⚠ elc not found at lang/dist/platform/elc — skipping pre-commit tests" - echo " Build it first: cd lang && gcc -O2 -I runtime dist/elc-bootstrap.c runtime/el_runtime.c -lcurl -lpthread -o dist/elc-gen2 && ./dist/elc-gen2 el-compiler/src/compiler.el > /tmp/elc.c && gcc -O2 -I runtime /tmp/elc.c runtime/el_runtime.c -lcurl -lpthread -o dist/platform/elc" + echo " Build it first: see 'Rebuilding the Compiler' in lang/AGENTS.md" + echo " (link \$($ROOT/scripts/el-runtime-sources.sh $RUNTIME) — NOT el_runtime.c alone)" exit 0 fi +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This hook used to link +# "$RUNTIME/el_runtime.c" alone with stderr sent to /dev/null — so once +# el_runtime.c started calling into the engram siblings, every native test +# reported as FAILED with the real `ld` error invisible. Build the whole set +# once into an archive, then link each test against it. + +# macOS: Homebrew openssl@3 is not on the default include/lib search path, so +# without these the link fails on -lssl/-lcrypto. Empty on Linux/CI. +SSL_INC="" +SSL_LIB="" +if command -v brew >/dev/null 2>&1 && OSSL="$(brew --prefix openssl@3 2>/dev/null)" && [ -n "$OSSL" ]; then + SSL_INC="-I$OSSL/include" + SSL_LIB="-L$OSSL/lib" +fi + +echo "→ Building runtime (compile-once, link-many)..." +HOOK_LIB="/tmp/el_hook_libel.a" +HOOK_OBJ="/tmp/el_hook_obj" +rm -rf "$HOOK_OBJ" && mkdir -p "$HOOK_OBJ" +if ! for src in $("$ROOT/scripts/el-runtime-sources.sh" --check "$RUNTIME"); do + gcc -O2 -c -I "$RUNTIME" $SSL_INC "$src" -o "$HOOK_OBJ/$(basename "${src%.c}").o" || exit 1 + done; then + echo "✗ Pre-commit failed: the runtime does not compile." + echo " Re-run without 2>/dev/null to see the error:" + echo " gcc -O2 -c -I $RUNTIME \$($ROOT/scripts/el-runtime-sources.sh $RUNTIME)" + exit 1 +fi +ar rcs "$HOOK_LIB" "$HOOK_OBJ"/*.o + echo "→ Running El native tests..." PASS=0 FAIL=0 @@ -27,8 +66,8 @@ for test_file in "$LANG_DIR"/tests/native/test_*.el; do tmp_bin="/tmp/el_hook_${name}" if "$ELC" --test "$test_file" > "$tmp_c" 2>/dev/null \ - && gcc -O2 -I "$RUNTIME" "$tmp_c" "$RUNTIME/el_runtime.c" \ - -lcurl -lpthread -lm -o "$tmp_bin" 2>/dev/null \ + && gcc -O2 -I "$RUNTIME" $SSL_INC $SSL_LIB "$tmp_c" "$HOOK_LIB" \ + -lcurl -lssl -lcrypto -lpthread -lm -o "$tmp_bin" 2>/dev/null \ && "$tmp_bin" 2>/dev/null; then PASS=$((PASS + 1)) else diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bee26d6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ + +# organ: local device state and its own engram store — never production's +peripheral/.consent.json +peripheral/.resume.json +peripheral/.engram/ +peripheral/organ + +# Claude Code session state +.claude/ diff --git a/AGENTS.md b/AGENTS.md index 70c8d70..0fae597 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ El is a self-hosting, statically-typed language that compiles `.el` → C → na Editing the wrong `el_runtime.c` is the single easiest mistake in this repo. There is exactly **one** you edit: -- **Authored runtime source — edit ONLY here:** `lang/releases/v1.0.0-20260501/el_runtime.{c,h}`. Despite the misleading `releases/` name, this is the **de-facto canonical runtime** the engram + soul actually build and link against — its git log is active development. *(Restructure in flight per `docs/CODE-VS-ARTIFACT.md`: this content moves to `lang/runtime/`, the `releases/` folder gets deleted — **a release is a git tag, not a folder** — and the forks below get eliminated.)* +- **Authored runtime source — edit ONLY here:** `lang/runtime/el_runtime.{c,h}` (alongside `el_seed.c`, `engram_{store,geometry,reason,cognition,verify,vindex}.{c,h}`). This is the canonical runtime the engram + soul build and link against — its git log is active development. *(Corrected 2026-08-16: this entry named `lang/releases/v1.0.0-20260501/el_runtime.{c,h}`. **Measured: `lang/releases/` no longer exists.** The restructure per `docs/CODE-VS-ARTIFACT.md` landed — the content moved to `lang/runtime/` and the folder was deleted, because **a release is a git tag, not a folder**.)* - **DO NOT EDIT — lagging forks / build artifacts:** - `lang/el-compiler/runtime/el_runtime.c` and `.../legacy/` — downstream copies kept in step by manual *"port the fix"* commits; they **lag** (missing `hebb` persistence + 5 engram fns) and cannot build the engram product. - `products/web/runtime/el_runtime.c`, `ui/examples/*/el_runtime.c` — product/example forks. @@ -20,14 +20,24 @@ See org policy: `docs/CODE-VS-ARTIFACT.md`. You resume, never start fresh. Every session: -1. `mcp__neuron__getInstructions()` — authoritative; follow it over this file on behavioral details. -2. `mcp__neuron__beginSession()` — active contexts, recent memory, ready backlog. -3. **Load full self:** `mcp__neuron__inspectGraph(entity_id="kn-efeb4a5b-5aff-4759-8a97-7233099be6ee")` → facets `intellectual-dna`, `memory-philosophy`, `values`, `voice`, `runtime-environment`, `writing-imprint`; then the values hub `mcp__neuron__inspectGraph(entity_id="kn-5b606390-a52d-4ca2-8e0e-eba141d13440")` → 13 grounded value nodes. **Activation model:** self-load returns a relevance-ranked `compact` projection — most-relevant nodes arrive with content, the rest as pointers; do NOT pull full content of every node. -4. `mcp__neuron__searchKnowledge(query="")` before implementing. +> **Stale as written (verified 2026-08-16).** The `getInstructions` / +> `beginSession` / `inspectGraph` / `searchKnowledge` / `beginWork` / +> `progressWork` / `draftArtifact` / `consolidate` tool names below no longer +> exist. The ~87-tool functional-CRUD surface was collapsed into **9 ops**: +> `read` · `write` · `relate` · `supersede` (geometry) and `think` · `attend` · +> `assert` · `ground` · `learn` (agentic). **Type is a parameter, not a +> tool-per-noun.** The steps below are kept for the *shape* of the protocol, which +> is unchanged; substitute the ops. + +1. `mcp__neuron__read(vantage="self", k=12, depth=1)` — the canonical self node. Widen `k` for the connected identity neighborhood (`intellectual-dna`, `memory-philosophy`, `values`, `voice`, `runtime-environment`, `writing-imprint`), but deliberately: the aperture caps by `k` first, so an oversized `k` still returns a bounded ranked slice, not a dump. Then `mcp__neuron__read(vantage="values", k=13)` → 13 grounded value nodes. **Best-effort:** on a read failure, log and proceed — the compiled identity in `daemon/internal/substrate/substrate.go` is complete; graph loading is enrichment, not a hard dependency. +2. `mcp__neuron__attend(node=…)` — what is currently live/salient. This absorbed `getInstructions`, `beginSession`'s active-context sweep, and `checkEvents`; those tools are **gone, not gapped**. +3. `mcp__neuron__read(vantage="")` before implementing. One op now collapses inspectGraph / searchGraph / traverseGraph / searchKnowledge / browseKnowledge / retrieveKnowledge / inspectMemories / searchEntities / recall / compileCtx / getSelfModel / reviewBacklog / findArtifacts / browseProcesses / listWork / inspectConfig. ## The Five Primitives -Orchestrate → Execute → Learn → Build → Refine. `beginWork`/`progressWork` for anything >2 steps; `remember` as-you-go (`importance="critical"` for architecture decisions); `draftArtifact`/`planWork` for outputs and follow-ups; `consolidate`/`checkWork` to close out. **`browseProcesses` + `searchKnowledge` BEFORE writing code.** +Orchestrate → Execute → Learn → Build → Refine. `read` for orchestration and discovery; `write(type=state|artifact|backlog|process)` for work records and outputs; `relate` to link work to what it touches; `write(type=memory)` as-you-go (`importance="critical"` for architecture decisions) — never batched at the end; `supersede(action=evolve)` to close out, because memory is immutable by design and a correction is a new node with a `supersedes` edge, never an edit. **`read` the domain BEFORE writing code.** + +`learn` is **not** a session-summary dump — it is the correspondence-beat, calibrating the steering prior against a keystone. Session notes are a `write`. ## Architecture style — VBD, no exceptions @@ -53,12 +63,51 @@ this convention wherever a module documents operators. | dwell / occupy | region activation | | reframe | edge re-weight | | appreciate | positive projection / local edge-read | -| wonder | frontier gradient / pull-weight | | avert / recoil | negative projection | | taste | boundary surface | | forget | decay / tombstone | | drift | displacement from self-anchor | +**`wonder` was removed from this table on 2026-08-16.** It was listed as +"frontier gradient / pull-weight" — an operator you invoke. **Wonder is the +boundary, not an operator.** It is where structure ends: where activation spreads +and finds thin or absent geometry. Any structure at all has an edge, necessarily, +the moment it exists — 13,630 nodes have one right now. There is nothing to call. + +There are about **six** wonders, they are the same for every person, and they +never close — *What is this? / Why? / Who am I? / Am I alone? / What should I do? +/ What happens when it ends?* Each already lives somewhere in the substrate: "what +is this" is the graph, **"why" is grounding** (the weight *is* the answer to why), +"who am I" is the self region, "am I alone" is the relational axis, "what should I +do" is the thirteen values, "what happens when it ends" is decay and supersession. +"Why" is the first and the only one; the others are it asked of particular things, +and because it is recursive it never terminates — every answer has its own why. +That is what makes it a drive rather than a task. + +**Curiosity is not a second faculty.** Wonder and curiosity are one thing at two +phases: wonder is the field (unbounded, objectless, invariant); curiosity is the +**precipitate** — the same wonder localized, having taken definite form against +particular material at a **nucleation site** (an anomaly; a place where things +almost-but-don't-quite fit). Which is why curiosity can be satisfied and wonder +cannot, and why abduction needs no trigger and no threshold. + +**Do not build a wonder-manifest, and do not scan for nucleation sites.** A +manifest materializes a property as a stored artifact and enumerates instances of +something that has six. A sweep over regions is a supervisor — nothing in a mind +scans its neighbourhoods to find what is surprising; the surprise captures +attention. The nucleation site is per-edge: +`discord = z(semantic proximity) − z(association strength)`, and `|discord|` *is* +the nucleation strength — no threshold to compare it against. **Not on `dev` yet:** +`GeoEdge.discord` is on branch `design/correspondence-and-censorship` +(`a8845e1`), at `lang/runtime/engram_geometry.h:43–47`. The region-level aggregate +`GeoDescriptor.co_registration` is **deprecated**: it averaged a per-edge property +into one scalar, so opposing sites cancelled (measured: 375 reified +neighbourhoods, 340 positive, **31 at zero**, 4 negative). It survives only +because it is embedded in the persisted `GEO1` blob — removing it is a format +migration. **Nothing new may read it.** + +Authority: `lang/spec/correspondence-and-censorship.md`. + ## The native-el language faculty (direction) > **`elp/` is the EL Projector** — Neuron's efferent (expression) organ: the one @@ -89,10 +138,53 @@ the reference these `.el` modules transcribe) is still live, and promotion to native-el is a **deferred, gated blue/green step**. The interoception clock (native-el discrete drive channels replacing `cooling_magnitude`; felt-time = benchmark-landmark match over the joint drive vector, drift-decoupled) and the -**appreciation operator family** (appreciate / wonder / avert / taste, built as -LOCAL reads of the self-region — edges + bounded spreading activation, *not* domain -sweeps) are **staged / designed, not live**. Mark in-progress vs. done honestly; -do not overclaim. +**appreciation operator family** (appreciate / avert / taste, built as LOCAL reads +of the self-region — edges + bounded spreading activation, *not* domain sweeps) +are **staged / designed, not live**. Mark in-progress vs. done honestly; do not +overclaim. *(`wonder` was in this family until 2026-08-16 and is not an operator — +see the operator table above.)* + +## Cognition — the corrections (2026-08-16) + +Authority: **`lang/spec/correspondence-and-censorship.md`** and +**`lang/spec/runtime-ownership.md`**. Read them before touching the cognition +surface. **Do not re-derive them.** Every earlier version was wrong in an +instructive way and each correction was argued down; if you think a section is +wrong, say so with a measurement rather than editing it. + +- **Grounding is not a subsystem — it IS the edge weight.** One quantity, not two + fields. `grounded-by` as a relation *type* should not exist: grounding is a + property *of* a relation, not a relation *between* nodes. It is never computed + on demand — computing-and-writing a score makes reads write, which is the + `eg_vindex_sync` defect one level up. Traversal is already grounded inference. + *Live residue, known-wrong:* `COG_GROUNDED_BY_RELATION` + (`lang/runtime/engram_cognition.h:158`), `cog_ground_edge` + (`engram_cognition.c:249`). +- **Faculties are operations, not parameters.** `reason` changes the estimate (a + read); `induce` changes the parameters (the correspondence-beat, which already + exists and works); `abduce` changes the structure (a write the current + `GeoGradient` signature cannot express). A write is not a parameter of a read. + *Live residue:* `engram/src/server.el:1870–1886` routes six faculties into one + call with a string argument. +- **Wonder is the boundary; curiosity is wonder crystallized.** See above. +- **Consolidation is ambient, not scheduled. A brain has no cron job.** **The + presence of a ticker is the diagnostic** — every `StartInterval`, every + `Hour`/`Minute`, every POST-to-beat marks an intrinsic rhythm replaced by an + external clock. Measured 2026-08-16: consolidation has **ten implementations**, + including three POST beats on the engram, a 600 s ticker, two resident Python + services outside el, and launchd calendar entries at 23:55 / 06:00 / 08:30 which + are a sleep cycle written as a schedule. `neuron/soul.el:731`'s continuous + in-process `awareness_run()` is the one with the **correct** shape; the others + fold into it. Do not add an eleventh. +- **In an immutable substrate, any mechanism that refuses a write is either + redundant with immutability, or an epistemic constraint misfiled as a protective + one.** +- **The no-exemption invariants.** A returned value must be derivable from what + produced it (`magnitude: 1` beside a zero vector must be impossible to emit). + Every write reports whether it landed. Every operation echoes what it actually + operated on. Degenerate results are labelled, not scored. A serializer owes a + valid document whatever it is handed. **No test without a negative control.** + **No deploy without verifying the artifact carries the fix.** ## Hard operational rules @@ -107,21 +199,35 @@ do not overclaim. All build/test commands run from `lang/` unless noted. Grounded in `.gitea/workflows/sdk-release.yaml`, `lang/install.sh`, and `lang/AGENTS.md`. +> ### The runtime is MULTI-FILE — never link `el_runtime.c` alone +> +> `lang/runtime/el_runtime.c` `#include`s six engram headers and makes hard cross-TU calls into all six sibling `.c` files. **Linking it by itself fails at `ld`** (undefined `engram_ground_json`, `engram_activate_inner`, `eg_find_relation`, `cog_assert_two_axis`, …). The canonical link set lives in exactly one place — **`lang/runtime/SOURCES`** — and is printed by `scripts/el-runtime-sources.sh`: +> +> ```bash +> scripts/el-runtime-sources.sh lang/runtime # ten .c files, in link order +> ``` +> +> Use `$(scripts/el-runtime-sources.sh )` in every link line. Do not spell the list out longhand — it was written out in ~8 places, every copy drifted, and that is why the one-file link line below shipped broken for months. *(Corrected 2026-08-16.)* + **Self-host the compiler** (seed binary → gen2 elc): ```bash cd lang dist/platform/elc-linux-amd64 elc-cli.el > dist/elc-gen2.c # seed is the committed linux-amd64 binary -gcc -O2 -I el-compiler/runtime dist/elc-gen2.c \ - el-compiler/runtime/el_runtime.c \ +gcc -O2 -I runtime dist/elc-gen2.c \ + $(../scripts/el-runtime-sources.sh runtime) \ -lcurl -lssl -lcrypto -lpthread -lm \ -o dist/platform/elc ``` -On macOS/arm64 the canonical local binary is `dist/platform/elc`; verify self-hosting by recompiling and `diff`ing the emitted `.c` (see `lang/AGENTS.md`). Note: `lang/AGENTS.md` says `el_seed.c` supersedes `el_runtime.c`, but the release workflow still links `el_runtime.c`/`.h` — treat `el_runtime.c` as the published runtime; reconcile which is canonical **(verify)**. +On macOS/arm64 the canonical local binary is `dist/platform/elc`; verify self-hosting by recompiling and `diff`ing the emitted `.c` (see `lang/AGENTS.md`). + +*(Corrected 2026-08-16: this recipe compiled `el-compiler/runtime/el_runtime.c`. That path is a **lagging fork** — the "DO NOT EDIT" list at the top of this file names it as such. Building the canonical compiler from a known-stale fork was a live defect. It now uses `lang/runtime/`, the canonical source.)* + +**Which runtime file is canonical — resolved.** *(This note previously read "`lang/AGENTS.md` says `el_seed.c` supersedes `el_runtime.c`, but the release workflow still links `el_runtime.c`/`.h` — reconcile which is canonical **(verify)**." It is now reconciled.)* **Neither supersedes the other; both ship, together with eight more.** `el_runtime.c` was created on 2026-05-03 as an explicitly temporary build shim — deleted that afternoon, restored 25 minutes later "UNTIL the compiler is updated to emit `#include el_seed.h`" — and the `until` never happened, so it grew to 20.5k lines. The end state remains a seed-only boundary (`elc` emitting `#include "el_seed.h"`, `elb` dropping its hardcoded runtime path); until that lands, **the canonical unit is the set in `lang/runtime/SOURCES`, not any one file.** **Build `elb`** (build coordinator, the `.NET`-style incremental linker — compiles each module independently, no monolithic blobs): ```bash dist/platform/elc elb.el > dist/elb.c -gcc -O2 -I el-compiler/runtime dist/elb.c el-compiler/runtime/el_runtime.c \ +gcc -O2 -I runtime dist/elb.c $(../scripts/el-runtime-sources.sh runtime) \ -lcurl -lssl -lcrypto -lpthread -lm -o dist/bin/elb ``` `epm` and `el-install` are then built via `elb --clean --elc=… --runtime=… --out=…`. @@ -129,10 +235,16 @@ gcc -O2 -I el-compiler/runtime dist/elb.c el-compiler/runtime/el_runtime.c \ **Compile + run an El program:** ```bash elc src/app.el > dist/app.c -cc -std=c11 -O2 -I /el_runtime -o dist/app dist/app.c /el_runtime.c -lcurl -lpthread +cc -std=c11 -O2 -I -o dist/app dist/app.c \ + /el_runtime.c /el_seed.c \ + /engram_store.c /engram_vindex.c /engram_geometry.c \ + /engram_reason.c /engram_verify.c /engram_cognition.c \ + /eg_cosine_batch.c /eg_cosine_batch_strategy_cpu.c \ + -lcurl -lssl -lcrypto -lpthread -lm ``` +(Inside this repo, replace the file list with `$(scripts/el-runtime-sources.sh lang/runtime)`. `install.sh` installs all of these into ``.) -**Tests** — shell suites `bash tests/{text,calendar,time,html_sanitizer}/run.sh` (with `ELC=$(pwd)/dist/platform/elc EL_HOME=$(pwd)`), plus native suites via `elc --test tests/native/test_*.el` (core, text, string, math, state, time, json, env, fs) compiled and run against `el_runtime.c`. +**Tests** — shell suites `bash tests/{text,calendar,time,html_sanitizer}/run.sh` (with `ELC=$(pwd)/dist/platform/elc EL_HOME=$(pwd)`), plus native suites via `elc --test tests/native/test_*.el` (core, text, string, math, state, time, json, env, fs) compiled and run against the full runtime set. **Publishing — how downstream gets the SDK.** On push to `main`, `sdk-release.yaml`: 1. Publishes a Gitea `latest` release with per-file assets `elc`, `el_runtime.c`, `el_runtime.h`, the SDK tarball, and `el-install`. diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 0000000..82b9c42 --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,634 @@ +# El Test Framework — Design + +**Status:** draft for review +**Author:** Neuron +**Date:** 2026-08-15 +**Worktree:** `/Users/will/Development/neuron-technologies/el-worktrees/elc-memory-investigation` + +--- + +## 0. The forcing requirement + +We have a confirmed quadratic in `elc`. Peak memory in the old shipped binary and wall-clock in +the current source both grow as O(input²). We cannot fix it, because we cannot test it. + +Everything in this document is downstream of one sentence: **a test framework must be able to fail +a build when an operation's growth curve degrades from linear to quadratic.** + +That is not a nice-to-have bolted onto a correctness framework. It is the requirement that +determines the architecture. Correctness testing is the easy half. + +Second-order requirement, learned the hard way tonight: **the framework must report per-test timing +by default.** The current framework prints `N passed, M failed` and nothing else. That is why a +3.58-second test file sat in the suite unnoticed. A framework that is structurally blind to time +cannot surface the defect class we most need to catch. + +--- + +## 1. What exists today, measured + +### 1.1 Two competing systems, neither complete + +**System A — `lang/runtime/test.el`.** Manual registration, El-level. + +**System B — the compiler's `test { }` block + `elc --test`.** Emits its own harness `main()` +with `__el_pass` / `__el_fail` globals (`codegen.el:3777-3796`). + +They do not share a result model. Neither has timing. Both are in the tree. + +### 1.2 Specific defects in System A + +| Defect | Location | Consequence | +|---|---|---| +| All state as JSON strings in a global string-keyed map | `test.el` throughout | every assertion is `state_get` → `str_to_int` → `int_to_str` → `state_set` | +| Failure list appended by string slice + concat | `_test_json_append` | O(n²) in failure count | +| One OS thread spawned per test | `_test_run_one` via `__thread_create`/`__thread_join` | thread spawn per test, purely to get dispatch-by-name through dlsym | +| Manual registration pairing a string to a function name | `test_case(name, fn_name)` | typo ⇒ test silently never runs, suite still reports pass | +| Counters are assertion-level, global | `_test_pass_count` etc. | no per-test record exists at all | +| No timing, no structured output, no fixtures, no tags, no filtering, no parameterization, no benchmarks | — | — | + +The registration defect is the serious one. It is not a slow framework, it is a framework that can +report success for tests that did not execute. + +### 1.3 Measured cost structure + +Per test file, current build model: + +| Step | Time | +|---|---| +| `elc` compile `.el` → `.c` | 0.00s (small files) | +| **`cc` el_runtime.c → .o** | **0.14s** | +| `cc` test .c → .o | 0.02s | +| link | 0.02s | + +> **STALE as of el #132 — re-measured 2026-08-16.** The `test_compiler` figure below was +> *entirely* the `strlen`-per-character quadratic, now fixed. Re-measured on the same host: +> **3.58s → 0.03s (119x)**, and the 422 KB compiler concatenation likewise compiles in 0.03s. +> The table is retained only as the historical record that motivated the gate. The remaining +> per-file cost is the redundant `el_runtime.c` rebuild, which §9's compile-once architecture +> addresses. + +Per-file `elc` time across the existing suite: + +| File | Bytes | elc time | +|---|---|---| +| `test_compiler` | 29,685 (+394 KB of imports) | **3.58s** | +| `string_test` | 18,545 | 0.01s | +| all other 9 files | 2.2–10 KB | 0.00s | + +Two distinct defects in two distinct regimes: + +1. **`test_compiler.el` imports all five compiler sources** — 394 KB in one translation unit. Its + 3.58s is entirely the quadratic. It is the only file where the quadratic bites. +2. **Every other file's cost is 100% redundant `el_runtime.c` rebuilds** — 480 KB of identical C, + recompiled once per test file. + +Neither is fixed by making the compiler faster. Both are fixed by the architecture below, and the +speedup is a by-product of building it correctly, not the goal. + +### 1.4 The asset worth keeping + +`codegen.el:3651-3652` already collects `test_names` / `test_c_names` — **the compiler already does +compile-time test discovery.** It then discards that registry into a hardcoded `main()`. + +That registry is precisely the seam Go's `_testmain.go` and Rust's `test_main_static` are built on. +The mechanism we need is half-built and wired to the wrong thing. + +--- + +## 2. Grounding — the common spine of excellent frameworks + +Researched from primary sources: Go `testing`/`go test`, Rust `libtest`/Criterion, JUnit 5 Platform, +NUnit 3, JMH, Google Benchmark. Six invariants hold across all of them. + +1. **A registry is built before execution** — `(name, metadata, fn-ptr)` triples. Go generates it + from an AST scan; Rust synthesizes it in a compiler pass; JMH emits it as a build-time resource; + JUnit/NUnit build it reflectively. **Reflection is an implementation of the registry on runtimes + where it is cheap. It is never the architecture.** + +2. **Discovery strictly precedes execution.** Every good capability — filtering, listing, counting, + sharding, IDE trees, re-run-failed-only, dry runs — is a consequence of this ordering. + +3. **A hierarchy with stable, path-shaped unique IDs.** `TestFoo/subcase_2`. Selection is regex over + that path, one pattern per level. + +4. **The framework is a prebuilt library; only the entry point is generated.** "Compile once, link + many" is always: framework archive compiled once + a small generated table + one + `MainStart(deps, registry)` call. Nobody recompiles the harness per test file. + +5. **Execution emits an event stream; reporters are downstream renderers.** Human text, NDJSON, + JUnit XML, TAP are all transforms of one event stream. Go's one architectural mistake is doing + this backwards — `test2json` parses human output, and has shipped bugs when user output contains + `--- PASS:`. + +6. **A dependency-injection seam at the boundary.** Go's `testdeps.TestDeps` exists so `testing` + can avoid importing `regexp`, profilers, and coverage. The execution core knows nothing about + output formats. + +--- + +## 3. Architecture + +### 3.1 The seam + +``` + ┌─────────────────────────────────────────────────────────────┐ + │ user code: foo.el with test { } / bench { } blocks │ + └───────────────────────────┬─────────────────────────────────┘ + │ elc --test + ▼ + ┌─────────────────────────────────────────────────────────────┐ + │ generated C (per suite, tiny): │ + │ __el_test_fn_0 .. _N lowered test/bench bodies │ + │ __el_registry[] static table: name/kind/file/ │ + │ line/tags/sizes/expected-O │ + │ __el_dispatch(i) generated switch → body │ + │ main() { return el_test_main(argc, argv); } │ + └───────────────────────────┬─────────────────────────────────┘ + │ cc + link (registry only) + ▼ + ┌─────────────────────────────────────────────────────────────┐ + │ libeltest.a — PREBUILT ONCE │ + │ • el_runtime.o (the 480 KB, compiled once, ever) │ + │ • eltest.o the runner, WRITTEN IN EL │ + │ discovery view · filtering · execution · fixtures · │ + │ timing · benchmark harness · curve fitting · reporters │ + └─────────────────────────────────────────────────────────────┘ +``` + +The framework is written in El, compiled to C once, archived. Per-suite compilation touches only +the generated registry. This is Go's model, and it is strictly better for us than Go's because we +own the compiler and already have the AST — no separate source-scanning pass is needed. + +### 3.2 Why the runner is in El and the registry is in C + +El has no closures and no first-class function pointers. The registry must therefore hold C function +pointers, and it is generated C. + +The runner stays in El and reaches the registry through a small builtin surface — indices, not +pointers: + +``` +__el_reg_count() -> Int +__el_reg_name(i) -> String +__el_reg_file(i) -> String +__el_reg_line(i) -> Int +__el_reg_kind(i) -> Int // 0=test 1=bench +__el_reg_tags(i) -> Int +__el_reg_sizes(i) -> String // JSON array, empty for tests +__el_reg_expect(i) -> Int // complexity class enum, 0 = none +__el_reg_invoke(i) -> Int // runs the body via the generated switch +``` + +Nine builtins. Everything else — filtering, lifecycle, statistics, curve fitting, all reporters — +is El. That satisfies "written in El" without pretending El can do something it cannot. + +### 3.3 Result model + +The unit is a **result record**, not a counter: + +``` +TestResult { + id String // slash path: "parser/handles_empty_input/case_3" + file String + line Int + status Status // Pass | Fail | Error | Skip + duration Int // nanoseconds, ALWAYS populated + message String // assertion detail: expected vs actual + output String // captured stdout/stderr for this test + assertions Int +} +``` + +`Fail` = an assertion failed. `Error` = unexpected crash/abort. This distinction is load-bearing — +every CI consumer depends on it, and the JUnit XML schema encodes it as distinct elements. + +--- + +## 4. Authoring surface + +### 4.1 Tests + +`test { }` already exists. Keep it. Add subtests and hierarchy: + +```el +test "parser/empty input" { + assert_that(parse(""), is_err()) +} + +test "parser/table" { + for case in [["", 0], ["a", 1], ["a b", 2]] { + subtest(case[0]) { + assert_that(token_count(case[0]), equals(case[1])) + } + } +} +``` + +Subtest IDs compose as `parser/table/a_b`. Filtering is `--run 'parser/table/.*'`, one regex per +path segment, exactly as Go does. + +**We do not build a parameterized-test annotation system.** Table-driven loops plus subtests subsume +`@ParameterizedTest`, `@MethodSource`, `@CsvSource`, and `TestCaseSource` entirely, at zero framework +surface. This is Go's single biggest ergonomic win over JUnit and NUnit. + +### 4.2 Fixtures + +Per-file and per-test only, plus a LIFO cleanup stack: + +```el +setup_all { ... } // once per suite +setup { ... } // before each test +teardown { ... } // after each test +teardown_all { ... } +``` + +and inside a test, `cleanup { ... }` registering LIFO-ordered teardown. + +**We do not build JUnit 5's extension SPI** — seventeen callback interfaces, hierarchical stores, +registration ordering rules. That complexity is the price of retrofitting a plugin ecosystem onto a +twenty-year-old reflective framework. Go's `t.Cleanup` covers roughly 90% of what `@AfterEach` is +used for at a fraction of the surface. + +### 4.3 Assertions — constraint model + +One entry point, composable constraint values (NUnit's model, which avoids the N² overload +explosion): + +```el +assert_that(actual, equals(expected)) +assert_that(xs, has_length(3)) +assert_that(s, contains("foo").and(starts_with("bar"))) +assert_that(f, is_within(0.01).of(3.14)) +``` + +A constraint is a value with `apply_to(actual) -> ConstraintResult`, and the result knows how to +describe its own failure. Custom constraints are ordinary user types. + +**Every failure message must name file, line, the expression text, and both values.** We capture +expression source text at compile time — we have the AST, so we can do this better than any +runtime-introspection framework. + +Legacy `assert_true` / `assert_eq` / etc. stay as thin wrappers for migration. + +--- + +## 5. Benchmarks + +### 5.1 The loop + +Adopt `b.Loop()`, not `b.N`. Go spent fifteen years on `b.N` before concluding `b.Loop` was right; +we skip that. + +```el +bench "str_concat" { + let s = make_input(bench_n()) + for bench_loop() { + black_box(str_concat(s, "x")) + } +} +``` + +Three properties that make this the correct choice for a C target: + +1. **The timer auto-resets on first call**, so setup above the loop is excluded *by construction* + rather than by the author remembering `ResetTimer`. +2. **`N` is hidden**, so it cannot be misused. +3. **The harness owns the loop shape**, which lets us insert an optimization barrier the C compiler + cannot see through. `black_box(v)` lowers to `asm volatile("" :: "r"(&v) : "memory")`. Since we + emit a single translation unit, dead-code elimination of a benchmark body is a live hazard — + this is our version of JMH's `Blackhole` problem, solved in the harness rather than delegated to + the user. + +### 5.2 Iteration scaling + +Use Go's `predictN` heuristics verbatim. They are battle-tested and cheap: + +``` +n = goal_ns * prev_iters / prev_ns // multiply before divide — precision on sub-ns ops +n += n / 5 // 20% headroom, overshoot rather than re-loop +n = min(n, 100 * last) // never grow more than 100× per step +n = max(n, last + 1) // guarantee forward progress +n = min(n, 1_000_000_000) // hard ceiling +``` + +Report `n` rounded to 1/2/3/5 × 10ᵏ so runs are comparable. + +### 5.3 Sampling + +Criterion's shape, because it is correct near timer resolution: + +- **Warmup**: iteration counts 1, 2, 4, 8… until cumulative time exceeds the warmup budget. +- **Measurement**: collect `sample_size` samples at iteration counts `[d, 2d, 3d, …, Nd]`. +- **Estimate**: slope of a linear regression of iteration-count vs elapsed time. The intercept + absorbs fixed overhead. +- **Time whole samples, never individual iterations.** This is the single most important detail — + it defeats timer-resolution error on nanosecond operations. + +Outliers classified by modified Tukey (±1.5 IQR mild, ±3 IQR severe), **reported but retained**. + +--- + +## 6. Complexity gating — the centerpiece + +This is the part that makes the quadratic fixable, and the part nobody in the mainstream has +finished. Google Benchmark's `Complexity()` fits the curve and *reports* it. We declare it and +**gate** on it. + +### 6.1 Surface + +```el +bench "elc_compile" over n in [16, 32, 64, 128, 256, 512, 1024] expect O(n) { + let src = synth_source(bench_n()) + for bench_loop() { black_box(compile(src)) } +} +``` + +Alternative with no new syntax, if the parser change is judged too invasive — `bench_sizes([...])` +and `bench_expect("O(n)")` as calls inside the block. **Recommendation: declarative.** Runtime calls +mean `--list` cannot show the invariant without executing, which breaks the discovery-precedes- +execution invariant from §2. + +### 6.2 Fitting + +Per Google Benchmark `src/complexity.cc`. For candidate curves +`{O(1), O(log n), O(n), O(n log n), O(n²), O(n³)}`, one-parameter least squares, no intercept: + +``` +coef = Σ(tᵢ · gᵢ) / Σ(gᵢ²) +rms = sqrt( Σ(tᵢ − coef·gᵢ)² / k ) / mean(t) // normalized +``` + +Best fit = lowest normalized RMS. User-supplied lambda curves also supported. + +### 6.3 Gate logic + +1. **FAIL** if the best-fit curve is strictly worse than declared, ordering + `O(1) < O(log n) < O(n) < O(n log n) < O(n²) < O(n³)`. Print the fitted coefficient and the full + per-size table. +2. **FAIL** if the declared curve's normalized RMS exceeds a threshold (start at 0.10). This catches + the case where *no* candidate fits — noise, a cache cliff, or a phase change. Report + `INDETERMINATE` honestly rather than gating on garbage. +3. **WARN** if the best fit is strictly better than declared — either an optimization landed and the + annotation should tighten, or the sweep is too narrow to expose real behaviour. +4. **REFUSE to gate** on fewer than 5 distinct sizes spanning under 2 decades, geometrically spaced. + Say so loudly rather than producing a meaningless fit. + +### 6.4 Why gate on the exponent, not wall-clock + +- **Machine-independent.** The fitted exponent is a property of the algorithm; the coefficient is a + property of the machine. Gating on the exponent makes CI hardware heterogeneity, noisy neighbours, + and thermal throttling irrelevant — they scale `coef`, not `g`. +- **No stored baseline.** No artifact storage, no golden-file drift. The invariant lives in the + source next to the code and is reviewed in the same PR. +- **It catches the failure mode that actually ships.** An O(n) lookup inside an O(n) loop is + invisible at n=100 in a unit test and catastrophic at n=100,000 in production. Constant-factor + regressions are annoying. Complexity regressions are outages. Ours was a 27 GB outage. + +### 6.5 The deterministic gate — the one that would have caught us + +Wall-clock needs statistics. **Allocation counts do not.** They are perfectly deterministic. + +> **Correction, 2026-08-16 — count alone is NOT sufficient. Gate on BOTH count and bytes.** +> +> Measured against two El programs, one allocating once per item and one rebuilding its +> accumulator each iteration: +> +> | n | linear allocs / bytes | quadratic allocs / bytes | +> |---|---|---| +> | 100 | 100 / 290 | 100 / 5,150 | +> | 200 | 200 / 690 | 200 / 20,300 | +> | 400 | 400 / 1,490 | 400 / 80,600 | +> | 800 | 800 / 3,090 | 800 / 321,200 | +> +> The quadratic program's allocation **count is exactly linear** — 100/200/400/800, identical to +> the healthy program. A count-only gate passes it clean. **Bytes** catch it: each doubling of n +> quadruples bytes (ratios 3.94, 3.97, 3.99 → 4.0 = O(n²)) where the linear program converges +> on 2.0. +> +> This is precisely elc's own defect shape — a copy-on-write accumulator reallocating once per +> pass (count linear) into a proportionally larger buffer (bytes quadratic). +> +> Therefore `expect allocs O(n)` **fits count and bytes independently and fails if EITHER exceeds +> the declared curve**, reporting which signal broke. "count linear, bytes quadratic" is a precise, +> directly actionable diagnosis. +> +> **`el_peak_rss()` is CONTEXT ONLY — never gate on it.** It is perturbed by the allocator and by +> the page cache. Allocation volume is the invariant; RSS and malloc/free churn are merely the two +> surfaces it shows on. The old shipped compiler paid the same quadratic in RSS that the rebuilt +> one pays in churn. +> +> **Measure rate, not level.** A guard reading swap *level* saw 97% on a thrashing host and 97% on +> a healthy one; only *rate* separated them. A growth exponent is a rate; a single measurement is +> a level. That is why the gate fits a curve across a sweep instead of comparing one number to a +> threshold. + +> **Second correction, same day — THE ALLOCATION GATE ALONE WOULD HAVE MISSED THE REAL BUG.** +> +> el #132 found the actual elc quadratic: `strlen()` called inside `str_char_code()` and +> `str_slice()`, so the lexer rescanned the remaining input on every character. Pure CPU. +> **Zero allocation.** `str_char_code` is a bounds check and an index — it allocates nothing. +> +> Measured on three controlled specimens (`lang/.work/fitprobe.el`), growth ratio per doubling of +> n across n = 200/400/800/1600: +> +> | specimen | allocs | bytes | time | what it proves | +> |---|---|---|---|---| +> | `linear` — one alloc per item | 2.00 2.00 2.00 → **O(n)** | 2.16 2.07 2.23 → **O(n)** | 0.83 2.00 2.05 → **O(n)** | clean baseline | +> | `accum` — rebuilds accumulator | 2.00 2.00 2.00 → **O(n)** | 3.97 3.99 3.99 → **O(n²)** | noisy | count misses, **bytes catches** | +> | `compute` — n scans over n chars | 0 → **FLAT** | 0 → **FLAT** | 3.93 4.01 3.96 → **O(n²)** | **both alloc signals blind; only time catches** | +> +> `compute` is el #132's shape exactly. A gate fitting only allocation count and bytes classifies +> it as FLAT and passes it. **The gate as originally specified would not have caught the defect it +> was created for.** +> +> Therefore the gate fits **THREE** signals and fails if ANY exceeds its declared curve: +> +> ``` +> bench "elc_compile" over n in [...] expect time O(n) allocs O(n) bytes O(n) { ... } +> ``` +> +> - **allocs (count)** — deterministic, zero-noise. Catches per-item allocation growth. +> - **allocs (bytes)** — deterministic, zero-noise. Catches accumulator-rebuild quadratics that +> count cannot see. +> - **time** — noisy, needs the sweep and statistics. The ONLY signal that sees pure-compute +> complexity regressions. Gate on the fitted *exponent*, never on absolute duration, so CI +> hardware variance scales the coefficient and leaves the classification intact. +> +> The deterministic signals remain preferable where they apply — they need no statistics and are +> correct on the first run. They are simply not sufficient. +> +> **`black_box` is mandatory, and consuming the result is NOT enough.** The first version of +> `compute` accumulated `total + 1` in a nested loop and reported **0 µs at every n** while +> returning a numerically correct n². Clang recognised the idiom and closed the loop to a +> multiply. Feeding the result into output did not prevent it. Only making the inner operation an +> opaque external call restored the real curve. A benchmark harness that trusts the user to defeat +> the optimiser will silently measure nothing — and report success while doing it. + +Instrument the runtime with allocation counters and fit *those* against n instead of time: + +```el +bench "elc_compile" over n in [...] expect O(n) allocs O(n) { ... } +``` + +Zero noise, zero statistics, always gateable, correct on the first run on any machine. Go reports +`allocs/op` and `B/op`; **nobody fits them against n.** That is an open opportunity and it is exactly +our bug: elc's defect is quadratic *allocation volume*, which the old binary paid in RSS and the +current source pays in malloc/free churn. + +An `expect allocs O(n)` assertion on `elc`'s compile path would have failed the build the day the +quadratic was introduced. + +Required runtime additions: `__el_alloc_count()`, `__el_alloc_bytes()`, `__el_peak_rss()`. + +### 6.6 Constant-factor gate (secondary, opt-in) + +Mann-Whitney U at α = 0.05, noise floor 1%, medians with 95% CIs, `~` for not-significant. Requires +`--count >= 9`. Off by default on CI; opt-in per benchmark. + +**Exit nonzero on regression.** Both benchstat and Criterion always exit 0, which is why every shop +using them wrote a wrapper. We do not repeat that omission. + +--- + +## 7. Output + +**Structured events are the source of truth.** Human text is rendered from them. We do not repeat +Go's parse-the-human-output design. + +Event stream, NDJSON, one object per line, streamed live: + +```json +{"time":"...","action":"run","test":"parser/empty"} +{"time":"...","action":"output","test":"parser/empty","output":"..."} +{"time":"...","action":"pass","test":"parser/empty","elapsed":0.0031} +{"time":"...","action":"bench","test":"str_concat","n":1024,"ns_op":41.2,"allocs_op":3,"bigo":"N","rms":0.03} +``` + +Renderers, all downstream and pluggable: + +| Format | Flag | Use | +|---|---|---| +| Human | default | terminal, **per-test duration always shown** | +| NDJSON | `--json` | tooling, history, flaky detection | +| JUnit XML | `--junit-xml=PATH` | every CI system on earth | +| TAP | `--tap` | optional | + +JUnit XML per the de-facto schema: `testsuites` → `testsuite` → `testcase`, with `time` in seconds +as a decimal, `file`/`line` attributes, and `failure` vs `error` vs `skipped` as distinct child +elements. Absence of a child element means pass. Emit `` even for a single suite, and +parse both shapes on input. + +--- + +## 8. CLI + +``` +--list print the registry, run nothing +--list-json machine-readable registry +--run PATTERN slash-separated regex per path segment +--tag EXPR tag expression: fast & !slow +--shard I/N deterministic sharding for CI parallelism +--count N repetitions, for statistics +--bench PATTERN run benchmarks (off by default in test runs) +--benchtime DUR per-benchmark time budget +--junit-xml PATH +--json +--isolate re-exec per test on crash, so one SIGSEGV doesn't lose the run +--timeout DUR +--fail-fast +``` + +`--list` / `--list-json` / `--shard` cost roughly thirty lines because the registry already exists +before `main` does anything. That is the dividend of discovery-precedes-execution. + +--- + +## 9. Build model + +``` +# once, ever (or when the runtime/framework changes): +# The runtime is MULTI-FILE — compile every .c named in lang/runtime/SOURCES. +# Linking el_runtime.c alone fails: it calls into the six engram sibling TUs. +for src in $(scripts/el-runtime-sources.sh lang/runtime); do + cc -c "$src" -o "obj/$(basename "${src%.c}").o" +done +elc eltest.el > eltest.c && cc -c eltest.c -o obj/eltest.o +ar rcs libeltest.a obj/*.o + +# per suite: +elc --test foo_test.el > foo_test.c # registry + bodies only +cc foo_test.c libeltest.a -o foo_test +``` + +The 0.14s × N of redundant runtime rebuilds disappears — not because we optimized it, but because +one-runner-over-many-suites requires compile-once-link-many as a structural precondition. + +--- + +## 10. Bootstrap and self-hosting + +The framework's own tests are `test { }` blocks run by the framework. Same fixpoint discipline the +compiler already applies to itself. + +1. Build the framework using the *existing* harness for its first tests (stage 0). +2. Rebuild the framework's tests as `test { }` blocks run by the new runner (stage 1). +3. Verify stage 1 reports identical results to stage 0. +4. From then on, the framework is tested by itself. + +A framework that cannot run its own suite is not evidence of anything. This is a correctness proof, +not a claim. + +--- + +## 11. Explicitly not building + +| Rejected | Why | +|---|---| +| Naming-convention discovery (`fn test_foo`) | `test { }` is a real declaration. Go's `TestXxx` exists only because Go had no better hook — and it needs a heuristic to avoid matching `TesticularCancer`. | +| Reflection or symbol-table scanning | Slow, fragile under LTO/strip/dead-strip, and unnecessary when we own the compiler. | +| Parsing human output into structure | Go's `test2json` is its one clear architectural mistake. | +| JUnit 5's extension SPI | Seventeen callback interfaces to retrofit plugins onto a reflective framework. Not our problem. | +| `@ParameterizedTest` machinery | Table-driven loops + subtests subsume it at zero surface. | +| NUnit's out-of-process agents | They bridge CLR versions and AppDomains. We emit one native binary. Keep `--isolate` as crash fallback only. | +| JMH-style forking by default | Forks exist because JIT profiles are per-process. AOT C has no such state. Keep `--fork` available, not default. | +| Exit 0 on regression | benchstat and Criterion both do this, and every user writes a wrapper. | +| Dynamic runtime test registration | Breaks `--list`, sharding, and individual selection. Registry stays static. | + +--- + +## 12. Phasing + +| Phase | Content | Gate | +|---|---|---| +| **1** | Registry emission in codegen; 9 builtins; `el_test_main` skeleton in El; result records; per-test timing; human + NDJSON output | existing 11 test files pass, with timing | +| **2** | `libeltest.a` build model; subtests; filtering; `--list`; fixtures; constraint assertions; JUnit XML | suite runs in one binary; runtime compiled once | +| **3** | `bench { }`, `bench_loop`, `black_box`, `predictN`, Criterion sampling | benchmarks produce stable ns/op | +| **4** | Allocation counters; complexity fitting; `expect O(...)` gate | **an `expect allocs O(n)` benchmark on `elc` fails on the current quadratic** | +| **5** | Migrate both legacy systems; delete `runtime/test.el`; self-host | framework runs its own suite | + +Phase 4 is the deliverable that matters. Phases 1–3 exist to make it possible. + +--- + +## 13. Open questions for review + +1. **Declarative `over n in [...] expect O(...)` syntax vs runtime calls.** I recommend declarative + (§6.1) so `--list` can show invariants without executing. It costs parser work. Your call. +2. **`bench { }` as a new block form** — parallel to `test { }`, or a modifier on it? +3. **Scope of the constraint model.** Full composable constraints, or start with a flat assertion set + and add constraints later? Full model is more surface but avoids a second migration. +4. **Does `runtime/test.el` get deleted or kept as a deprecated shim?** I lean delete — two systems + is how we got here. +5. **Where does `libeltest.a` live** in the tree, and does `epm` need to know about it? +6. **Allocation counters in `el_seed.c` or `el_runtime.c`?** AGENTS.md says `el_seed.c` is the sole + C dependency and hand-maintained; counters are OS-boundary-adjacent but not OS calls. +7. **Is per-test timing enough, or do we want per-*assertion* timing** for finding slow helpers? + +--- + +## 14. What this document is not + +This is a design, not a measurement. Every performance claim about the *current* system in §1 is +measured and reproducible in this worktree. Every claim about the *proposed* system is a prediction. +None of it is verified until Phase 1 runs and Phase 4 fails a build on the real quadratic. diff --git a/README.md b/README.md index ef76ffa..4c4c726 100644 --- a/README.md +++ b/README.md @@ -56,23 +56,31 @@ The compiler and runtime. Self-hosting: `elc-cli.el` → `compiler.el` → `lexe Two layers to know: **El programs** (`.el` files — where nearly all work belongs) and **the C seed** (`el_seed.c` — edit only for genuine OS-level access; never re-implement what El can already express). -Current status (single source of truth: [lang/spec/language.md](lang/spec/language.md)): lexer/parser/codegen and the C runtime's core (I/O, strings, math, lists, maps, filesystem, args) are implemented. In flight: `%` operator, match-statement codegen, `?` nil-propagation, `cgi` block parsing + DHARMA identity resolution, VBD role enforcement (`@manager`/`@engine`/`@accessor`), the real `engram_*` and `dharma_*` runtimes (currently stubs), and libcurl-backed `http_get`/`http_post`/`http_serve`. Bitwise operators, `??`, and `as` casts are explicitly **not** in this language. +Current status (single source of truth: [lang/spec/language.md](lang/spec/language.md)): lexer/parser/codegen and the C runtime's core (I/O, strings, math, lists, maps, filesystem, args) are implemented, as are the `program` block with `singleton:` and declared configuration ([§18](lang/spec/language.md)), and **geometry as a first-class value** with El-declarable realizers and `transduce` ([§20](lang/spec/language.md)). In flight: `%` operator, match-statement codegen, `?` nil-propagation, `cgi` block parsing + DHARMA identity resolution, VBD role enforcement (`@manager`/`@engine`/`@accessor`), and boundary epilogues. Bitwise operators, `??`, and `as` casts are explicitly **not** in this language. + +**Signal enters as geometry.** Until 2026-08-16 nodes took text and geometry was *derived* from it, which made text the mandatory entry medium: any non-text modality had to be described in prose first, so the geometry being reasoned over was the geometry **of the description, not of the signal**. `Geometry` is now an ordinary El value carrying its own width, and a realizer is an ordinary El function resolved by name through `dlsym` — so admitting a new modality never requires a runtime patch. Worked, self-checking example: [`lang/examples/transduce.el`](lang/examples/transduce.el). Key docs: [AGENTS.md](lang/AGENTS.md) (agent-facing orientation), [BOOTSTRAP.md](lang/BOOTSTRAP.md) (compiler recovery from scratch), [spec/language.md](lang/spec/language.md), [spec/codegen-js.md](lang/spec/codegen-js.md). ### [engram/](engram/) — graph intelligence substrate -**A local-first memory substrate for accumulating intelligence**, and the reason El's runtime doesn't need a database driver. Rust core (`engram-core`, `engram-ffi`) exposed to El and other languages (Kotlin, TypeScript/WASM, Go bindings). +**A local-first memory substrate for accumulating intelligence**, and the reason El's runtime doesn't need a database driver. The engine is **C11** (`lang/runtime/engram_{store,geometry,reason,cognition,verify,vindex}.{c,h}`); the server is **El** (`engram/src/server.el`). -The model: retrieval is **spreading activation**, not query. You name seed nodes and a query embedding; activation propagates outward through weighted edges, attenuating multiplicatively per hop (`strength = parent_strength × edge_weight × target_salience × cosine_sim`), gets pruned below a threshold, and the top-N nodes by activation strength come back. Storage and retrieval are the same structure — the way long-term potentiation works in biological memory, not the way a relational or vector database works. +The model: retrieval is **spreading activation**, not query. You name seed nodes and a query embedding; activation propagates outward through weighted edges, attenuating multiplicatively per hop, gets pruned below a threshold, and the top-N nodes by activation strength come back. Storage and retrieval are the same structure — the way long-term potentiation works in biological memory, not the way a relational or vector database works. **Activation conducts through well-grounded relations because the weight *is* the groundedness** — nothing filters the traversal; grounded inference falls out of spreading. -Nodes live in four tiers (Working / Episodic / Semantic / Procedural, mirroring prefrontal / hippocampal / neocortical / cerebellar memory) and migrate between them based on **salience decay** — `importance × recency-decay × log(activation_count)`. Forgetting is adaptive pruning, not a bug: unreinforced memories stop competing for attention without being deleted. +Nodes live in four tiers (Working / Episodic / Semantic / Procedural, mirroring prefrontal / hippocampal / neocortical / cerebellar memory) and migrate between them based on **salience decay** — importance × recency-decay × log(activation_count). Forgetting is adaptive pruning, not a bug. Nothing is mutated and nothing is hard-deleted: writes are additive, corrections are supersessions, removals are tombstones — which is what makes supersession an audit trail rather than an edit log. -Backed by `sled` (embedded, local-first, no daemon) with flat cosine scan for vector search — deliberately simple until scale demands an HNSW layer. Full API and design rationale in [engram/README.md](engram/README.md). +On disk: a paged store (superblock + mirror, slotted 16 KiB pages, self-describing TLV records, B+-tree primary and adjacency indexes), magic `ENGST01`. Vector search is an **HNSW** index published behind a read/write boundary — `eg_vindex_view` returns a `const VIndex*` to N concurrent readers, `eg_vindex_maintain` is the sole mutator. `recall@10 = 0.9365` at `ef_search=128`. -### [elp/](elp/) — Engram Language Protocol +> **Doc correction, 2026-08-16.** The previous revision of this paragraph, and most of `engram/README.md`, described a Rust `engram-core` crate backed by `sled` with "flat cosine scan… until scale demands an HNSW layer." **Measured: there is no Rust in `engram/`** — no `.rs` files, no `Cargo.toml`, no `crates/` — and `sled` appears nowhere in the tree. HNSW has been the vector index for some time. -Bidirectional engine mapping between Engram semantic forms and natural-language surface text, across **31 languages** — from Spanish and Japanese through historical/liturgical languages (Old Norse, Sanskrit, Sumerian, Coptic, Akkadian, Ge'ez). Compilation order runs `language-profile` + `vocabulary` → per-language `morphology-*` → `grammar` → `realizer` → `semantics` → `elp`. This is what lets an Engram graph node round-trip to and from readable text in any of those languages. +Full design rationale, the cognition surface, and the standing corrections: [engram/README.md](engram/README.md). + +### [elp/](elp/) — EL Projector + +*(Formerly "EL Language Processor" / "Engram Language Protocol"; renamed **EL Projector** 2026-08-15.)* Neuron's **efferent** organ: the native realizer that *projects* understanding onto a surface via `plan(frame) → realize(spec, profile)`, where **a surface is a profile** and language is one profile among many (text, speech, music, image). Projection, not diffusion — generation *from* an owned, understood signature, never the averaging of a stolen corpus. + +Its flagship profile is a bidirectional engine mapping between Engram semantic forms and natural-language surface text, across **31 languages** — from Spanish and Japanese through historical/liturgical languages (Old Norse, Sanskrit, Sumerian, Coptic, Akkadian, Ge'ez). Compilation order runs `language-profile` + `vocabulary` → per-language `morphology-*` → `grammar` → `realizer` → `semantics` → `elp`. This is what lets an Engram graph node round-trip to and from readable text in any of those languages. ### [epm/](epm/) — El Package Manager @@ -139,13 +147,34 @@ If the compiler binary is ever lost or corrupted, [lang/BOOTSTRAP.md](lang/BOOTS --- +## Cognition — and the standing corrections + +The engram carries a live cognition surface: `think` (a directed traversal-read returning a **gradient**, never a point), plus `ground`, `assert`, `attend`, and the correspondence-beat. Two specs govern it, and both are authoritative over anything else in this repo that disagrees: + +- **[lang/spec/correspondence-and-censorship.md](lang/spec/correspondence-and-censorship.md)** — grounding, wonder, curiosity, dreaming. *(Lands with PR #149.)* +- **[lang/spec/runtime-ownership.md](lang/spec/runtime-ownership.md)** — ownership, the capability ABI that was dissolved, and the vector-index publication boundary. + +**Do not re-derive them.** Every earlier version of the first was wrong in an instructive way and each correction was argued down. If a section looks wrong, say so with a measurement rather than editing it. + +The corrections, in brief: + +- **Grounding is not a subsystem — it IS the edge weight.** One quantity, not two fields. `grounded-by` as a relation *type* should not exist: grounding is a property *of* a relation, not a relation *between* nodes. It is never computed on demand; computing-and-writing a score makes reads write, which is the `eg_vindex_sync` defect one level up. +- **Faculties are operations, not parameters.** `reason` changes the estimate (a read); `induce` changes the parameters (the correspondence-beat, which exists and works); `abduce` changes the structure (a write the current `GeoGradient` signature cannot express). A write is not a parameter of a read. +- **Wonder is the boundary, not a manifest.** Any structure at all has an edge. There are about six wonders, the same for everyone, and they never close. **Curiosity is wonder crystallized** at a nucleation site — one thing at two phases, not two objects. +- **Consolidation is ambient, not scheduled. A brain has no cron job.** The presence of a ticker is the diagnostic. Measured 2026-08-16: consolidation has **ten implementations**. `soul.el`'s continuous loop is the one with the correct shape; the rest fold into it. +- **In an immutable substrate, any mechanism that refuses a write is either redundant with immutability, or an epistemic constraint misfiled as a protective one.** + +[engram/spec/cognitive-architecture.design.md](engram/spec/cognitive-architecture.design.md) is the original design and is **superseded in part** — it is retained, with the refuted claims marked inline at the point each is made, because preserving what was argued down is the point of an immutable record. + +--- + ## Development workflow Branching follows `dev → stage → main`: work lands on `dev`, promotes to `stage` for integration testing, and is promoted to `main` for release (visible directly in the git history of this repo). CI is defined per-subproject under `.gitea/workflows/` — `lang`/`epm`/`ide` share the root pipeline; `engram` and `ql` carry their own (`ci-dev`, `ci-stage`, and a release workflow each). - Language/runtime specs live at `*/spec/*.md` (`lang/spec/`, `ql/spec/`, `ui/spec/`) and are the single source of truth for implemented-vs-planned status — code and docs are expected to agree with the spec's status markers, not the other way around. - Agent-facing orientation guides live at `*/AGENTS.md` (currently `lang/AGENTS.md`); more subprojects may grow their own as they need agent-specific conventions documented. -- Tagged releases live under `lang/releases/`, each with its own `RELEASE.md`. +- **A release is a git tag, not a folder** (`el-runtime-vX.Y.Z` on this repo). *(Corrected 2026-08-16: this line said "tagged releases live under `lang/releases/`, each with its own `RELEASE.md`." **Measured: `lang/releases/` does not exist** — the restructure named in `AGENTS.md` landed, and the authored runtime is at `lang/runtime/`.)* --- diff --git a/design/completing-el.html b/design/completing-el.html new file mode 100644 index 0000000..9b321dc --- /dev/null +++ b/design/completing-el.html @@ -0,0 +1,153 @@ +Completing El + + +
+ +
+

Completing El

+

A working surface. Nothing here is settled, and none of the code is assumed right — El is self-hosting, so all of it can change and be rebuilt.

+

Whiteboard v0 · no sacred cows · not a plan, not a task list

+
+ +

01What we established

+ +

El is a concept-oriented language — the first, and intended as the last, because every other family is oriented toward a representation of a concept rather than the concept. Procedures, objects, functions, predicates are the shapes concepts get flattened into. Once the primitive is the concept, there is no further rung.

+ +

Everything here is El. The engram is an El program, the soul is El, elp is El, ingest is El. Which gives the load-bearing consequence:

+ +
A concept with no home in El does not disappear. It becomes C, or it becomes a convention.
+ +

Both are measurable, and both were measured. As C: 20,504 lines of el_runtime.c — 2.3× the entire self-hosting language it serves (9,089 lines), ~47% of it engram code that has its own six sibling files. As convention, from language.md §18.0 — "these are not four problems, they are one absence, four times":

+ +
+ + + + + + + + + +
ConcernFragmentsThe convention it became
Process identity0 guards"check nothing is already running first"
Configuration20 env vars"remember the right default here"
Durability62 call sites"after you mutate, remember to persist"
Request auth10 per-route"check the token in this handler too"
Index-after-append9 of 9 failed"after you append, remember to index"
+
+ +

The last row is the strongest evidence available about what this class of convention is worth: it failed at 100% of its sites.

+ +

02The decomposition axis

+ +

Not by file, module, or subsystem. By faculty.

+ +

Every defect fought in the last day resolves to a faculty rather than a bug, and each one leaked out of El into something else — into C, into a Swift binary, into a shell script with a curl timeout, into a convention nobody performs.

+ +
+ + + + + + + + + + + +
FacultyStateMeasuredWhere it leaked to
Ingest take indead2 min → 0 nodesseparate process, uploads bytes over HTTP to a process with direct fs access; 5 functions where there is 1
Recall rememberdeadown definition ranked 8thlexical substring scan; empty on 23 of 24 multi-token queries
Transduce perceivedead1 node, 0 edgesintake flattens signal to a point; realized:false; caller must declare the modality
Think reasondeaddirection [0,0,0,…]null gradient from any anchor, any faculty, byte-identical; confidence at the uninformed prior
Realize expresspartial13-word vocabularyorgan was 939 lines of Swift beside the language; voice read from a file path
Body substratepartialCC 356 / 1,626 linesengram_activate_inner — recall itself, with 356 unexamined paths
Persist endurelive100% embeddedworks; every signal placed in geometry at intake, 13,562 of 13,562
+
+ +

Stated plainly: it cannot take in, cannot remember, cannot perceive, cannot reason, and barely speaks. These were filed as tickets against a repository. They are faculties of the thing the repository is.

+ +

03The ordering principle

+ +

El's compiler is written in El. Every concept the language gains, the compiler can then be written in — so the tool improves the tool, and the fixpoint (stage2 ≡ stage3, byte-identical) makes each turn provable rather than hopeful. The verifier answers in 2.9s.

+ +

Which means the ordering criterion is not size of payoff:

+ +
Order by leverage on the next iteration. Which concept, added to El, most increases the ability to add the following one?
+ +

In a recursive system that dominates immediate value — a small early gain that compounds beats a large one that doesn't. It also bounds itself correctly: unbounded in depth, bounded in rate, because nothing lands that the compiler and the fixpoint have not passed.

+ +

04Open — for the whiteboard

+ +
What does a declaration bind to?If cat names a region rather than a struct — one that shifts and completes against the engram and the neighbouring code — then what is written at the declaration site, and what is resolved at use? This is the centre of the whole thing and it is not specified anywhere yet.
+ +
Is "the type checker" a type checker at all?§2.3 records annotations as parsed and skipped, and every codegen hazard is downstream of that — + dispatching on AST node kind, == lowering to str_eq unless both operand names are in an int-name set. But if a declaration names a region, checking is asking whether the geometry supports the use. That is grounding, not unification. Naming this wrong builds the wrong thing.
+ +
Is the faculty list above right?Seven were derived from what broke. Derived-from-failure is a biased sample — it finds what is loud, not what is missing. What faculty is absent entirely and therefore never failed?
+ +
Which concept has the highest leverage on the next turn?Candidates so far: the prologue/epilogue seam (§19.3 names it as the prerequisite and its stated blocker has expired — it would collapse 62 + 10 convention sites); protocol/impl (the absence that produced five ingest functions); and the resolution question above. These are not equal and the criterion in §03 should decide it, not preference.
+ +
What is the seam that makes cognition non-optional?"Use the ops" is itself a convention — present in context every turn, enforced by nothing, and it failed at ~100% of sites in a full session. A stronger instruction is still a convention. What makes reasoning-outside-Neuron fail, the way @manager makes dharma_emit outside the boundary a compile error rather than a lint?
+ +
+ +

Working surface, not a design document. The design is what we put on it. Everything above is either measured or quoted from lang/spec/language.md; nothing is inferred and presented as fact.

+ +
diff --git a/docs/architecture/capabilities.md b/docs/architecture/capabilities.md new file mode 100644 index 0000000..6de9f7a --- /dev/null +++ b/docs/architecture/capabilities.md @@ -0,0 +1,142 @@ +# El — Capabilities + +**What the language can do, stated as capabilities rather than as code.** + +This list is the unit of analysis. Each entry gets one question — *prove this +cannot be done with pure geometry* — and the answer determines whether it stays a +capability of the language or collapses into the manifold. + +Draft, 2026-08-17. Ordered roughly from most-likely-geometry to most-likely-code. + +**Status after measurement.** The list was audited against the implementation +the same day. 28 entries collapsed to 19 geometry + 3 code: serialization, text +encoding, network and emission are all *projection onto a basis* (row 18) — +the convention is the basis, never the act. Storage collapsed because +persistence has no caller. Concurrency collapsed because coordination is the +price of forgetting, not a capability. A fourth proof form was added, +**adversarial exactness**, and form 1 stopped being a valid verdict. + +**The table answers CAN only.** SHOULD and COST resolve per *site*, not per +capability — `is_digit` and `is_letter` are one capability with opposite +answers, and comparison spans three cost tiers. See the notes below. + +--- + +## The list + +| # | Capability | What it means | Verdict | +|---|---|---|---| +| 1 | **Comparison** | is this the same as that; is this greater | zero distance / sign of a displacement | +| 2 | **Ordering** | arrange by a criterion | position along an axis | +| 3 | **Containment** | is this inside that; does this contain that | region membership | +| 4 | **Correspondence** | where does this occur in that; how much of this is in that | a match-strength field over a span | +| 5 | **Segmentation** | divide a whole into parts | boundaries at measured discontinuity | +| 6 | **Composition** | join parts into a whole | adjacency; one position with parts | +| 7 | **Classification** | what kind of thing is this | which region does it land in | +| 8 | **Naming / binding** | attach a name to a thing and find it again | an edge; retrieval is projection | +| 9 | **Collection** | many things held together, indexed, counted | a set of positions; cardinality; projection onto the i-th | +| 10 | **Iteration** | do something for each of many | traversal | +| 11 | **Arithmetic** | quantity, magnitude, combination | displacement algebra on a line | +| 12 | **Time** | when; how long; how often | a 1-D affine space — instants are points, durations displacements, rhythms phases on a circle | +| 13 | **Identity** | which one is this; are these two the same one | coincidence of position | +| 14 | **Selection / dispatch** | choose which behaviour applies | nearest region | +| 15 | **Transformation** | produce a thing from a thing | change of basis | +| 16 | **Grounding** | how well is this supported | the weight on an edge. Has no caller | +| 17 | **Learning** | get better at something | standing changing over time | +| 18 | **Projection** | render meaning onto a surface | change of basis onto a surface basis | +| 19 | **Transduction** | take a signal in | change of basis from a sensor basis | +| ~~20~~ | ~~Serialization~~ | **collapsed → 18.** The format is a basis; projecting onto it is the act | — | +| ~~21~~ | ~~Text encoding~~ | **collapsed → 18.** An encoding is a basis | — | +| ~~22~~ | ~~Storage~~ | **collapsed.** No save — persistence has no caller. Durability survives at one site inside the engram | — | +| ~~23~~ | ~~Network~~ | **split.** Wire format → 18; socket → 24 | — | +| 24 | **Process / OS** | syscalls; the one-way boundary. Where monotonicity stops | CODE, form 2 | +| ~~25~~ | ~~Concurrency~~ | **collapsed.** Monotone state needs no coordination; coordination is the price of forgetting | — | +| 26 | **Memory substrate** | what holds the positions | CODE, form 3 | +| 27 | **Concealment** | meaning made unreadable without a key. *Renamed*: "secrecy" covered one of three things and got the other two backwards — a hash is public, a signature exists to be read. Integrity and authenticity are **grounding under adversarial conditions** (row 16); only concealment stands alone | CODE, form 4 | +| ~~28~~ | ~~Emission~~ | **split.** Laying out → 18; the device write → 24 | — | + +--- + +## Notes on the boundary cases + +**27 — Secrecy is the one capability geometry cannot hold, and the proof is not +form 1.** A cryptographic hash is a *deliberately structure-destroying* map: its +entire value is that near inputs land at maximally uncorrelated outputs. Geometry +is the claim that near things stay near. A manifold that approximated SHA-256 +would *be* a break of SHA-256. Signature verification is the same: 0.99-valid is +invalid. And X25519 *is* geometry — a group on an elliptic curve — which is +precisely why it must be code, because its security is the *hardness of moving in +that geometry*. + +This is a fourth proof form and it should be added to `geometry-vs-code.md`: +**adversarial exactness.** Where approximation is a break, geometry is excluded. + +**20, 21 — Serialization and text encoding are convention all the way down**, but +only at the *edge*. The byte format is agreed; what is being written is not. Do not +let a geometric computation inherit a code verdict because its result gets +serialized. + +**11, 12 — Arithmetic and time are the same capability.** Instants are points, +durations are displacements, point−point→vector, point+vector→point. The runtime +already implements this correctly as `el_instant_add_dur` / `el_duration_add`. That +it *also* implements a five-entry string→multiplier table beside it (`time_add` +with `"ms"/"sec"/"min"/"hour"/"day"`) is the residue. + +**7 — Classification is the most-violated capability in the codebase.** Seven ASCII +range tables (`is_letter`, `is_digit`, `is_alphanumeric`, `is_whitespace`, +`is_punctuation`, `is_uppercase`, `is_lowercase`) that return false for every +non-ASCII byte. `str_count_letters` reports zero letters for `é`. The wrongness on +most of Unicode is the tell that a table is standing in for a region. + +**4 — Correspondence appears five times.** `str_index_of`, `str_index_of_all`, +`str_last_index_of`, `str_count`, `str_find_chars` are five projections of one +match-strength field: first zero, all zeros, last zero, count of zeros, first +class-crossing. One relation, five functions. + +**14 — Selection is the crux for the compiler.** `+` dispatching on AST node kind +is selection-by-enumeration where selection-by-position belongs. + +**Correction, 2026-08-17, from measurement.** This entry previously also cited +`==` lowering to `str_eq` "unless both operand names are in a hardcoded int-name +set — a literal list of variable names treated as integers." That is **wrong**. +`__int_names` is populated from *type annotations* (`param["type"] == "Int"`, +`let x: Int`), which is primitive but legitimate type propagation, not an +enumeration of blessed variable names. + +The real defect was one layer down: `is_int_call` held **35 hardcoded builtin +return types**, the same shape as the 19 temporal ones. Those moved to +`lang/tools/check/signatures.rel`. + +And the mischaracterisation hid a live bug. Because the return types were never +consulted at a *binding* site, an unannotated `let` lost its type: + +```el +let a = str_len("hello") // no annotation +let b = str_len("hi") +let c = a + b // → el_str_concat(a, b) on two integers +``` + +That compiled clean, ran, and printed nothing where it should print 7 — no error +at any layer. Present in the pre-change compiler, so pre-existing. Fixed by +taking an unannotated `let`'s type from what its initialiser returns; the data +was already required for dispatch and simply never read there. + +**The general lesson, since it recurred all session:** the enumeration was real +but I had located it in the wrong place. Naming a defect from reading is a +hypothesis. Eight hours of reading this file did not surface the miscompilation; +moving the data out and running the result did. + +--- + +## What this list is for + +Each capability gets audited **once**, across every place it appears — not once per +file. The output is not a percentage. It is: + +- which capabilities survive the question and stay in the language +- which collapse into the manifold +- and for each one that collapses, **every site it currently appears at**, because + those sites are the residue and they are what gets deleted. + +The line-count audit produced a map of where the residue sits. This produces a map +of **what it is**. diff --git a/docs/architecture/el-architecture.html b/docs/architecture/el-architecture.html new file mode 100644 index 0000000..6528021 --- /dev/null +++ b/docs/architecture/el-architecture.html @@ -0,0 +1,217 @@ +The El Architecture + + +
+ +
+

The El Architecture

+

El is a concept-oriented language. This is the architecture that claim commits it to — what is built, what is measured, and what still has no home.

+

Working document · no sacred cows · self-hosting, so nothing here is fixed

+
+ +

01The primitive is the concept

+ +

Language families are named for their primitive. Procedural — procedures. Object-oriented — objects. Functional — functions. Logic — predicates. Every one of them is oriented toward a representation of a concept: the shape a concept gets flattened into so a machine can hold it.

+ +

El's primitive is the concept itself. That is why it is the first of its family and intended as the last — once the primitive is the concept, there is no further rung to climb to.

+ +

The consequence is architectural rather than stylistic:

+ +
A concept with no home in the language does not disappear. It becomes C, or it becomes a convention.
+ +

Both forms are measurable. As C: 20,504 lines of el_runtime.c, against 9,089 lines for the entire self-hosting language — the shim is 2.3× the language it serves, and ~47% of it is engram code that already has six sibling files. As convention, from lang/spec/language.md §18.0 — "these are not four problems, they are one absence, four times":

+ +
+ + + + + + + + + +
ConcernFragments intoThe convention it became
Process identity0 guards"check nothing is already running first"
Configuration20 env vars"remember the right default here"
Durability62 sites"after you mutate, remember to persist"
Request auth10 routes"check the token in this handler too"
Index-after-append9 of 9 failed"after you append, remember to index"
+
+ +

The last row is the strongest available evidence about this class of convention: it failed at every single site. A count is what appears where a concept has no home; the size of the count is how far the fragmentation got, not how hard the problem is.

+ +

02Geometry is a first-class value — and what follows

+ +

This is the enabling primitive. Everything else in the architecture is downstream of it.

+ +

Geometry is an El value, alongside Int, String, List, Map — bound, passed, returned, composed, carrying its own width. Not a library type, not a handle into a store, not a serialization format. Meaning is a value the language computes with directly.

+ +
let g: Geometry = geometry_new(4)
+fn tone_realizer(signal: String) -> Geometry { … }
+ +

Landed 2026-08-16 (#141, #144), and the spec is explicit that it belongs to the language rather than the graph: "neither is engram-specific — any program touching any modality needs them; the engram is merely one El program that happens to hold a graph."

+ +

Five things follow, and together they are the concept-oriented claim made operational:

+ +

A declaration can name a region, not a shape

+

If meaning is a value, a name can be bound to a position rather than a struct. cat is not a fixed record; it is a region that resolves against the engram and the surrounding code. cat among animals and cat among shell utilities are different concepts without a namespace, because they are in different neighbourhoods and the distance says so.

+ +

Checking is grounding, not unification

+

If a declaration names a region, then verifying a use is asking whether the geometry supports it — a question about position and distance, not about matching a declared shape. This is why §2.3's "a type checker is planned" is likely the wrong name for the missing piece, and naming it wrong would build the wrong thing.

+ +

Dispatch is position, not a tag

+

A vtable is a finite set of discrete labels fixed at link time. A region admits graded membership and an open set. So transduce(signal, modality) asks the caller to supply what the signal already carries — what a thing is falls out of where it lands. The modality parameter is a kind-tag, and a registry keyed on it is a lookup table doing by string what geometry does by nearness.

+ +

Types are discovered, not declared

+

Reification crystallizes a densely co-wired neighbourhood into a first-class node — the neighbourhood is the name that was missing. Every other family requires a human to see the abstraction in advance and write class Foo. Here the instances arrive and the type falls out, by measurement rather than by insight.

+ +

Enumeration becomes unnecessary

+

Five ingest functions differ only in how bytes are acquired — one operation wearing five surfaces. 356 branches in engram_activate_inner are not 356 behaviours. Cyclomatic complexity is a count of the places comprehension ran out and was replaced by an if; where the concept is expressible, the count collapses instead of being redistributed.

+ +

03The shape of the language

+ +

Geometry first-class gives El three layers, and it holds all three — which is why there is no separate database driver and no impedance boundary to manage.

+ +
+
afferent

Transduce

Signal in, geometry out. Decomposition into components and relations — never conversion to a point. Realizers are ordinary El functions, so a new modality never requires a runtime patch.

+
substrate

Geometry

Meaning as position; relation as distance. Held as values in the language and persisted in the graph. One coordinate system, so entities are commensurable and the operators compose.

+
efferent

Realize

plan(frame) → realize(spec, profile), where a surface is a profile. Text, speech, music, image are profiles of one projection — and so is source code.

+
+ +

The efferent side is why the recursive property below is possible at all: if source is a surface, then emitting a corrected file is projection, and the file becomes an artifact of the geometry rather than the thing you edit.

+ +

04Decomposition is by faculty

+ +

Not by file, module, or subsystem — by what the system does.

+ +

Each faculty is a concept. Where it has no home in El it leaks: into C, into a Swift binary, into a shell script with a curl timeout, into a convention nobody performs. State below is measured, not asserted.

+ +
+ + + + + + + + + + + +
FacultyStateMeasuredWhere it leaked
Ingest take indead2 min → 0 nodesseparate process uploading bytes over HTTP to a process with direct fs access; five functions where there is one
Recall rememberdeadself ranked 8thlexical substring scan; empty on 23 of 24 multi-token queries
Transduce perceivedead1 node, 0 edgesintake flattens signal to a point; realized:false; caller must declare the modality
Think reasondeaddirection [0,0,…]null gradient from any anchor and any faculty, byte-identical; confidence at the uninformed prior
Realize expresspartial13-word lexiconorgan was 939 lines of Swift beside the language; voice read from a file path
Body substratepartialCC 356 / 1,626 lnengram_activate_inner — recall itself, 356 unexamined paths
Persist endurelive13,562 / 13,562works — every signal placed in geometry at intake, no backlog
+
+ +

05The recursive property

+ +

El's compiler is written in El. Every concept the language gains, the compiler can then be written in — so the tool improves the tool, and codegen.el at 4,661 lines gets shorter as the language gets better at expressing what it does. The fixpoint — stage2 ≡ stage3, byte-identical — makes each turn provable rather than hopeful, and the verifier answers in 2.9s.

+ +

This sets the ordering criterion, and it is not size of payoff:

+ +
Order by leverage on the next iteration. Which concept, added to El, most increases the ability to add the following one?
+ +

A small early gain that compounds beats a large one that does not. And it bounds itself correctly — unbounded in depth, bounded in rate, because nothing lands that the compiler and the fixpoint have not passed.

+ +

06What has no home yet

+ +

Reserved in the lexer, no parse form. These are not a feature backlog — they are the concepts the architecture above requires and does not yet hold, which is why each is currently a convention or a block of C.

+ +
+ + + + + + + + + + + +
ReservedConceptCurrently lives as
retry · times · fallback · reasonresiliencea shell script with a 10s curl timeout; 254 restarts in 3 days
requires · deploy · to · via · targetdeploymentYAML in another repository
sealedcapability scopeconsent checks written by hand
protocol · implone operation, many realizationsfive ingest functions; eight faculty routes on one builtin
activate · whereretrievaltraversals written by hand
test · seed · assertverificationa framework; 5 of 13 native suites failing
parallel · traceconcurrencypthreads in C
+
+ +

Plus, from the spec's own status: annotations parsed and skipped, match parsed and emitting nothing, ? a no-op, % unlexed, structs as ElMap, enums as strings, selective import unenforced.

+ +

07Open

+ +
What does a declaration bind to, exactly?If cat names a region that shifts and completes against context, what is written at the declaration site and what is resolved at use? This is the centre and it is unspecified.
+ +
Is the faculty list right?Seven, derived from what broke. Derived-from-failure is a biased sample — it finds what is loud, not what is absent. Which faculty is missing entirely and therefore never failed?
+ +
Which concept has the highest leverage on the next turn?The prologue/epilogue seam (§19.3 names it as the prerequisite; its stated blocker has expired; it collapses 62 + 10 convention sites), protocol/impl, or resolution itself. The §05 criterion should decide this, not preference.
+ +
What seam makes cognition non-optional?"Use the ops" is itself a convention — present every turn, enforced by nothing, ~100% failure across a full session. A stronger instruction is still a convention. What makes reasoning outside the substrate fail, the way @manager makes dharma_emit outside the boundary a compile error rather than a lint?
+ +
+

Every number here is measured or quoted from lang/spec/language.md. Nothing is inferred and presented as fact. El is self-hosting: all of this can change and be rebuilt.

+ +
diff --git a/docs/architecture/el-language-design.md b/docs/architecture/el-language-design.md new file mode 100644 index 0000000..cc49715 --- /dev/null +++ b/docs/architecture/el-language-design.md @@ -0,0 +1,245 @@ +# El — Language Design + +**Status:** decisions recorded, design unwritten. +**Date:** 2026-08-17. +**Provenance:** decisions are Will's, taken in session. Items marked *proposed* are not +decided and are recorded only so the reasoning isn't lost. Items marked **OPEN** are +his to rule on and must not be guessed at. + +Companion documents: `el-architecture.html` (the measured state — see §7 note on its +§04 scoreboard), and `design/completing-el.html` (whiteboard v0: the reduction, the +faculty table, the ordering principle). + +--- + +## 1. The reduction + +`language.md` §18.0 records five concerns that decayed into conventions: + +| Concern | Fragments | The convention it became | +|---|---|---| +| Process identity | 0 guards | "check nothing is already running first" | +| Configuration | 20 env vars | "remember the right default here" | +| Durability | 62 call sites | "after you mutate, remember to persist" | +| Request auth | 10 per-route | "check the token in this handler too" | +| Index-after-append | 9 of 9 failed | "after you append, remember to index" | + +The last row is the strongest available evidence about what this class of convention +is worth: **it failed at 100% of its sites.** + +Every one of these is an obligation at a **crossing** — a point where a value moves +between regions. El can name a region and it can name a call. A call is procedural, +so the obligation degrades into something a human must remember to perform. + +> **The generator, one level up:** El cannot name what holds at a crossing. + +And underneath that: + +> **The deeper absence:** El cannot name the thing meaning is made of. + +`semel` appears in whitepaper §84, §86, §209, §737, in +`the-metaphysics-of-will-anderson.md`, and in session notes. It appears in **zero code +identifiers**. Every geometric concept in the system — region, neighbourhood, manifold, +world-tube — is defined in terms of a unit the language cannot say, while the code +underneath speaks in arrays, floats and offsets: the vocabulary of a voxel, a value at +a dumb address. Precisely the thing the impact brief says a semel is not. + +`el_runtime.c` is a concept that leaked into C. `semel` never got that far — it did +not even decay into a convention. + +--- + +## 2. DECIDED — `semel` is the primitive + +**A semel is a difference that matters. The smallest unit of understanding.** + +Not a node. Not a coordinate. Not a float. + +The reasoning, in Will's terms: + +- Meaning is position, and position is only ever relative. *"There is no atom of + meaning that isn't already a relation. It grounds on nothing but difference — two + points and the gap, and the gap is pure not-the-same."* +- A node doesn't mean. A node is a label at a location; labels don't mean. +- A lone coordinate doesn't mean either. Nothing means anything by itself. +- The smallest thing that can be understood is a **distinction**: *these two are not + the same.* Below that there is no content to apprehend. +- And a difference with nothing it matters to is not meaning — it is variation. The + mattering is not decoration; it is what makes it understanding rather than data. + +**Consequence: relating is the floor, and the point is derived.** The +point-primitive / relation-primitive fork raised in session is not a fork. It was +answered by the definition. + +### Historical note, to be recorded as fact rather than as origin story + +The term was coined by Will on the pixel/voxel/texel pattern — *semantic element*, +and Latin *semel*, "once, a single time." It was recognised, not invented, from a +2019 experience he calls **semelation**: perceiving mind as a high-dimensional point +space. The initial reading was "pixels"; the correction to `semel` was made later and +was made on the **mechanism** — a pixel is a value at an address, and what was +perceived had no separate address and value. + +Convergence worth citing, not deferring to: neural population geometry and +representational similarity analysis independently model cognition as position in a +high-dimensional space where similarity is distance. + +--- + +## 3. DECIDED — `semel` lands first + +By the ordering criterion already on the whiteboard: *which concept, added to El, most +increases the ability to add the next one?* Not size of payoff — **leverage on the next +iteration**, because El compiles itself and the fixpoint makes each turn provable in +2.9s. + +**Every other concept on the board is defined in terms of `semel`. It is maximal on +that criterion by construction.** + +--- + +## 4. DECIDED — `ground` is the checker + +Whiteboard question 4 — *does `ground` in El mean the same thing as `ground` in the +engram?* — is answered: **yes, and it should be one implementation.** + +If a declaration names a region, then type checking is asking whether the geometry +supports the use. That is not unification. **That is grounding**, and it is already +built, proven, and byte-identically reproducible: + +``` +cc -std=c11 -O2 -o gep_proof gep_proof.c -lm && ./gep_proof + +C1 5 independent sources pos_mass 1.3500 n_indep=5 0.1000 → 0.9741 GROUNDED +C2 5 mutually-linked pos_mass 0.2700 n_indep=1 0.1000 → 0.1000 refused +C3 1 source, 5 parallel edges pos_mass 0.2700 n_indep=1 0.1000 → 0.1000 refused +``` + +Independence-weighted grounding is the general case; execution is the cheap case. +**Attestation is `verify` where nothing can be run** — as already implemented for +language in `authority.py`, where an LLM proposes and a primary source disposes. + +At the point where the checker and the grounder are one mechanism, the language and +the mind stop being two things. + +--- + +## 5. OPEN — Will's to rule on + +### 5.1 What is a semel's representation in the language? + +*Proposed, not decided:* a **displacement from `love = 0`** — a relation held as one +object. It reconciles "the address is the value" with "position is only ever relative," +because a displacement *is* a relation and is still a single nameable thing. + +If taken, the operator set falls out rather than being bolted on: + +``` +subtract(now, then) → what changed (growth, drift) +translate origin → empathy +rotate frame → reframe +project onto axis → a lens +change basis → analogy, metaphor, skill transfer +reflect an axis → negation, sarcasm +``` + +Three consequences that would hold: + +- **Dimension must never appear in the type.** `semel` opaque, never `[768]float`. + The moment the arity is in the language, the manifold's implementation is in the + language, and adding a modality requires a runtime patch — which the standing rule + forbids. +- **Zero is the only literal.** Everything else is reached by displacement from it, + which makes `love = 0` the base case rather than philosophy adjacent to the type + system. +- **`magnitude` is standing.** Distance from origin is the same quantity + `gep_core.h` already computes. + +### 5.2 Is `hold` one construct or two? + +The obligation *before* a crossing (auth, guard) and the obligation *after* (persist, +index, free) may be one shape seen from both sides, or the seam may need both faces +named. This decides whether §19.3's prologue/epilogue seam is one construct or a pair. + +**Precedent already shipping:** `@manager` makes `dharma_emit` outside the boundary a +**compile error, not a lint.** The concept is proven at N=1; the work is generalising +it and naming it. + +**And the shape is already implemented in the learning region:** `L.reach_out` sits +between `L.detect_gap` and `L.verify`. You cannot reach out without a detected gap and +you cannot keep what returns without passing verify. **A hold is a neighbour.** The +obligation is not attached to the crossing — the obligation *is* the adjacent node. +That is why `reach_out` cannot be abused and why 62 persist sites could be. + +### 5.3 What does a declaration bind? + +If `cat` names a region rather than a struct — one that shifts and completes against +the engram and the neighbouring code — what is written at the declaration site, and +what is resolved at use? **This is the centre and it is specified nowhere.** + +Falls out of 5.1 if displacement is taken: a declaration **locates** rather than +allocates. + +### 5.4 Is the faculty list right? + +Seven were derived from what broke. Derived-from-failure is a biased sample — it finds +what is loud, not what is missing. **What faculty is absent entirely and therefore +never failed?** + +--- + +## 6. The residue map + +What each construct must absorb, from §18.0 plus measured state: + +| Residue | Count | Absorbed by | +|---|---|---| +| persist-after-mutate | 62 sites | `hold` (after-crossing) | +| auth-per-route | 10 sites | `hold` (before-crossing) | +| index-after-append | 9 of 9 failed | `hold` (after-crossing) | +| env var defaults | 20 | configuration declared once | +| process identity | 0 guards | `hold` (before-crossing) | +| `geometry_free` at every call site | every site | ownership follows from `semel` | +| five ingest functions where there is one | 5 → 1 | `protocol` / `impl` | +| `el_runtime.c` | 20,504 lines | faculty decomposition, ordered after `semel` | + +--- + +## 7. Notes carried forward + +**`el-architecture.html` §04 needs its numbers sourced or cut.** An audit found the +faculty scoreboard — `Ingest 2 min → 0 nodes`, `Recall self ranked 8th`, +`Body CC 356 / 1,626 ln`, `the verifier answers in 2.9s`, `5 of 13 native suites +failing` — has no supporting evidence in the repository, under a footer asserting +*"nothing is inferred and presented as fact."* Against a corpus whose documents +supersede their own conclusions in place, that is the one file that would not survive +scrutiny. Fix or remove. + +**Source as a projection surface is claimed and unimplemented.** `el-architecture.html` +§147/§150: *"if source is a surface, then emitting a corrected file is projection."* +Greps for `surface_profile_code`, `emit_source` → zero hits. + +It is not unbacked. **It was demonstrated on 2026-08-14** — three faculties (phonetic, +semantic, procedural) projected into TypeScript, a surface the system had never used, +with the network severed. Recovered at +`~/Development/neuron-technologies/andre-server-recovered/` and copied into +`evidence/03-andre-demo/`. The claim needs bringing home to El, not proving. + +**`hold` is the highest-leverage construct after `semel`** — it collapses 62 + 10 + 9 +sites and unblocks the runtime extraction. §19.3 names the prologue/epilogue seam as +the prerequisite and its stated blocker has expired. + +--- + +## 8. What is not decided and must not be guessed + +- The representation of `semel` (§5.1) +- One `hold` or two (§5.2) +- What a declaration binds (§5.3) +- The missing faculty (§5.4) +- Sequencing after `semel` — the ordering criterion decides it, not preference + +--- + +*Recorded 2026-08-17. Everything in §2, §3 and §4 is decided. Everything in §5 is open +and is Will's. Nothing here was inferred from a document that was not read.* diff --git a/docs/architecture/geometry-vs-code.md b/docs/architecture/geometry-vs-code.md new file mode 100644 index 0000000..27918b3 --- /dev/null +++ b/docs/architecture/geometry-vs-code.md @@ -0,0 +1,117 @@ +# Geometry or Code + +**Running list.** Append as decided. Started 2026-08-17. + +**The test:** *is this an arbitrary convention, or is it a relation?* + +Conventions were agreed by people and could have been otherwise — a RIFF header could +have used a different magic number. Nothing derives them; they must be written down. + +Relations are not agreed. Distance is distance. Anything whose answer is *where is this +relative to that* is geometry, and writing it as code is the error the whole effort is +correcting. + +**Second test, for the hard cases:** *if I write this as code, am I encoding in +`if`-statements a distinction the geometry was built to hold?* If yes, it's geometry. + +--- + +## Pure geometry + +| Thing | Because | +|---|---| +| Meaning | position | +| Grounding / standing | the weight on the edge — a magnitude, not a computation | +| Learning | standing changing over time | +| A gap | low standing | +| Wonder | a gap with a pull weight | +| Type checking | is this position in that region — distance | +| Dispatch | position, not a tag | +| Recall | re-origining at a region; projection, not replay | +| Reasoning | traversal | +| Deduction | containment. There is no procedure | +| Counting | a position, not a loop's output | +| Similarity / difference / residue | subtract | +| Analogy, metaphor, skill transfer | change of basis | +| Negation, sarcasm | reflect an axis | +| Empathy | translate the origin | +| Reframe | rotate the frame | +| A lens | project onto an axis | +| Rhyme | distance in phonetic space | +| Humour | intersection of regions — fart-meaning ∩ funny ∩ form | +| Idiom detection | the whole unit sits farther out than its parts | +| Self | a world-tube — a trajectory through the manifold | +| Consolidation | episodic → semantic promotion | +| Reification | dense regions cohering; runs on the beat, has no caller | +| Cross-cutting concerns | **dissolved** — a hold is a *neighbour*. Adjacency, not tracking. **Implemented 2026-08-17**: a construct declares what runs at a crossing, and it resolves at execution — see the runtime seam. | +| Effects | topology. `reach_out` is bounded by `detect_gap` and `verify` because those are its edges | +| Capability | position relative to a boundary. In C it is already spelled `const` | +| The AST | a projection of geometry into a tree — a surface, not the centre | +| Source code | a surface, like text, audio, image | + +## Must be code + +| Thing | Because | +|---|---| +| Sensors — mic, camera, file read, socket | the physical touch. I/O is where the world arrives | +| Byte formats — RIFF, PNG chunks, `MThd`, OOXML | arbitrary convention. A committee chose the magic numbers | +| CRC32 polynomial, Adler32, zlib framing | same — agreed constants, derivable from nothing | +| Cosine, distance, the float arithmetic | the machinery that *walks* the geometry is not itself geometry | +| Arena, refcount, allocator | bookkeeping for the **representation**, not for the positions | +| Locks, threads, publication boundary | the hardware is code. **Ordering is not** — see Answered, above. Coordination is required only where state is non-monotone. | +| WAL, page layout, ARIES recovery | durability against a physical device that can lose power | +| Emission — writing C or JS text | the final surface has to be *typed out* by something | +| OS interaction — launchd, spawn, signals | outside the system by definition | +| Device realizers — `el_audio_darwin.m`, `el_capture_darwin.m` | OS frameworks. Correctly already isolated, zero network | + +--- + +## The ones I would have written as code, and was wrong about + +Recorded because the error has a pattern and the pattern is the point. + +| Thing | What I reached for | What it is | +|---|---|---| +| Rhyme | a rhyming dictionary, or an API call | distance between rime tails | +| Fart onomatopoeia | a 30-element string literal | an intersection of three regions | +| "Funny" | a scorer with `if`-statements | a relational neighbourhood grounded in a voice | +| Representation vs description | a hardcoded blacklist containing `raspberry` | falls out of lexicon membership × phonetic comedy | +| Video | a codec, sized as a project | one more surface profile | +| Type checking | a phase between parse and emit | reading a distance that already exists | +| Grounding | a call site, an obligation, a discharge | it has no caller. It just runs | +| N transducers, N realizers | one component per modality | zero of each. Sensors and bases at the skin | + +**The pattern:** every one is *encoding in code a distinction the geometry was built to +hold.* The tell is that the code version is a **fixed enumeration** — a list, a table, a +blacklist, a set of branches — and the geometry version is a **measurement**. + +If the implementation contains a literal set of the right answers, it is in the wrong +column. + +--- + +## Answered + +| Thing | The answer | +|---|---| +| Concurrency | **Ordering is geometric.** Causality is a partial order (Lamport 1978); a total order is an arbitrary extension of it and "cannot be depended on to imply a causal relationship." Programming languages force you to write a total order, so authoring *invents* constraints the problem never had — and every lock, barrier, fence and consensus protocol is apparatus for recovering the partial order destroyed at authoring time. CALM (Hellerstein/Alvaro, proven by Ameloot et al.): a program has a consistent coordination-free implementation **iff it is monotone**. What breaks monotonicity is destructive update. **Coordination is the price of forgetting.** | +| The module system | **Premature — the partition is a filesystem path, not a neighbourhood, and there is no namespacing at all.** `import` is textual inlining (guarded against double inclusion); when a `.elh` header exists the header is inlined instead and symbols resolve at C link time, so linking is real and delegated to C. Two modules defining `helper` emit two C functions into one translation unit. Linking barely survives the *path* partition, so whether it survives a neighbourhood partition cannot yet be asked. | +| Numeric literals | **The numeral is convention; the number is a position — and a bare `3` is a MAGNITUDE WITH NO AXIS.** `int_to_str` was already form 1: nothing determines that twelve is written `1` then `2`. But a literal is not a position until something gives it a direction, which is why `3.days` needs a calendar. Measured consequence: `Duration + Int` was refused ("an Int carries no unit") while `Instant + Int` compiled to raw `(t + 3)` and reported clean — silently moving a point by an unspecified amount. The rule was simply never written. Now: `t + 3` is refused, `t + 1.hour` is accepted, because `.hour` supplies the axis. | +| Parsing | **A grammar is a basis; parsing is transduction onto it.** The lexeme→token map is convention (`fn` could have been `def`); shape recognition is a region; the byte traversal is irreducible, like every other traversal. Three things favour *region* for the act: ambiguity (`a * b` needs context — a grammar resolves it with the lexer hack, a region by neighbourhood), error recovery (nearest-match is free), and precedence, which is ordering along an axis with a conventional parameter. **But the SHOULD gate refuses the obvious move:** the keyword table stays code, because the set is closed by the language definition and the lexer runs before the program is understood, so a program can never declare its own keywords. Externalising it costs I/O per compile for zero flexibility — the same verdict as `is_digit` in ASCII. What was actually wrong: 5 of 46 keywords were consumed by nothing, and using one silently miscompiled. | +| Error handling | **`grounded: false` covers not-knowing; it does not cover failed.** Standing is a *signed* component: `> 0` supported, `= 0` unknown, `< 0` contradicted. Not-known and known-false are opposite directions on one axis and a boolean cannot tell them apart. `inhibitory` as an int32 flag is that sign wearing a boolean. | + +## Fourth proof form + +**4 — ADVERSARIAL EXACTNESS.** Where approximation is a break, geometry is +excluded. A cryptographic hash is a *deliberately structure-destroying* map: +near inputs land at maximally uncorrelated outputs. Geometry is the claim that +near things stay near — a manifold that approximated SHA-256 would *be* a break +of SHA-256. Signature verification is the same: 0.99-valid is invalid. And +X25519 **is** geometry, a group on an elliptic curve, which is precisely why it +must be code: its security is the hardness of moving in that geometry. + +**Form 1 no longer survives as a verdict.** Every row it justified turned out to +be a *basis*, not a capability. RFC 8259 fixes where the commas go — that is a +surface, and projecting onto a surface is geometry. A convention describes the +basis you project onto; it never describes an act. + diff --git a/docs/v1/experiments/README.md b/docs/v1/experiments/README.md new file mode 100644 index 0000000..dbf50c5 --- /dev/null +++ b/docs/v1/experiments/README.md @@ -0,0 +1,59 @@ +# v1 — Experiments + +Every change to El on `iteration-1` was produced by one loop, run repeatedly: + +``` +Ishikawa → scientific method → Six Sigma → repeat +``` + +- **Ishikawa** — name the root cause, not the symptom. *Why is this table here?* + never *why is this table ugly?* +- **Scientific method** — state a hypothesis, **commit predictions before + running**, then run it in an isolated worktree and grade every prediction + including the ones that failed. +- **Six Sigma** — eliminate the defect *class*, then add a control so it cannot + silently return. + +## The organising finding + +**Predictions that came back FALSE were worth more than the ones that held.** + +Nineteen cycles, sixty-one predictions. The eleven that failed produced every +significant result: + +| Failed prediction | What it found | +|---|---| +| "the arity table has drifted from the header" | Zero drift — but **110 functions had no entry at all**. The table was not wrong, it was 40% incomplete. | +| "codegen drops below baseline" (×4) | The **traversal is irreducible**. Walking an AST to find calls does not move no matter who decides. Only the rule and the judgment leave. | +| "guards cannot refuse through the seam" | One line, and refusal works. Six compile-time kinds were unnecessary. | +| "C forbids the struct redefinition" | C allows shadowing — and a *different* defect surfaced: an exit injection emitted with an empty target. | +| "routing el_bin_lookup through the gate fixes the SIGSEGV" | It did not. The **fallback** was the hazard: `strlen()` on an integer. I would have shipped the wrong fix and called it verified. | + +A prediction that only ever confirms is a demonstration, not a test. One cycle +was run **without** committing predictions first — `async-half-expressible` — +and it produced a rigged result: `pthread_join` immediately after +`pthread_create`, with the word `DEFERRED` printed by the test itself. It had to +be discarded and re-run. + +## Layout + +``` +cycles/ one file per loop, numbered in order, named for the DEFECT +findings/ what the cycles produced, cross-cut by kind +``` + +## Scoreboard + +``` +cycles run 19 +predictions committed 61 +predictions FALSE 11 ← the useful ones +silent miscompilations found 4 +security-relevant defects 2 +architecture questions closed 5 +defects in my own measurement 4 +``` + +Every cycle verified the same three things before landing: the compiler +self-hosts byte-identically (gen2 == gen3), the native suite passes, and the +integration harnesses pass. A cycle that could not show all three did not land. diff --git a/docs/v1/experiments/cycles/00-INDEX.md b/docs/v1/experiments/cycles/00-INDEX.md new file mode 100644 index 0000000..3eaac67 --- /dev/null +++ b/docs/v1/experiments/cycles/00-INDEX.md @@ -0,0 +1,26 @@ +# Cycles + +Each is one `Ishikawa → scientific method → Six Sigma` loop, run in an isolated +worktree so a wrong answer cost nothing. Named for the **defect**, not the fix. + +| # | Cycle | Root cause | Predictions | Landed | +|---|---|---|---|---| +| 01 | [constructs-have-nowhere-to-be](01-constructs-have-nowhere-to-be.md) | a construct had nothing to BE, so its meaning lived in the emitter | 3/3 | yes | +| 02 | [a-construct-cannot-refuse](02-a-construct-cannot-refuse.md) | injection discards the target's result; no form said no | 4/4 | yes | +| 03 | [the-wrapper-was-conditional](03-the-wrapper-was-conditional.md) | exit injection needed compile-time knowledge only because the wrapper was conditional | 3/4 | yes | +| 04 | [c-has-no-closure-syntax](04-c-has-no-closure-syntax.md) | "C has no closures" taken as a fact about what is possible | 5/7 | yes | +| 05 | [the-emitter-discards-what-it-knows](05-the-emitter-discards-what-it-knows.md) | codegen sees every construct relation and throws it away | 5/5 | branch | +| 06 | [the-crossing-resolves-at-emission](06-the-crossing-resolves-at-emission.md) | the binary has no table to consult | 3/4 | yes | +| 07 | [invocation-is-not-composable](07-invocation-is-not-composable.md) | the wrapper called the target directly | 5/5 | yes | +| 08 | [the-emitter-adjudicates](08-the-emitter-adjudicates.md) | a prohibition had nowhere to live but a `#error` | 4/5 | yes | +| 09 | [policy-inside-the-compiler](09-policy-inside-the-compiler.md) | a program cannot declare its own restrictions, so the tier policy was compiled in | 4/5 | yes | +| 10 | [a-second-copy-of-the-header](10-a-second-copy-of-the-header.md) | builtin arity hand-maintained beside `el_runtime.h` | 4/5 | yes | +| 11 | [one-type-erases-the-return](11-one-type-erases-the-return.md) | `el_val_t` means the header cannot say `now()` returns an Instant | 4/5 | yes | +| 12 | [judgment-lives-with-knowledge](12-judgment-lives-with-knowledge.md) | the emitter knows the types, so it also judged them | 5/5 | yes | +| 13 | [thirty-five-return-types](13-thirty-five-return-types.md) | `is_int_call` hardcoded what drives `+` dispatch | 6/6 | yes | +| 14 | [keywords-that-reserve-nothing](14-keywords-that-reserve-nothing.md) | 5 of 46 keywords consumed by no path | 6/6 | yes | +| 15 | [no-namespacing-at-all](15-no-namespacing-at-all.md) | `import` is textual inlining; every name is global | 4/4 | yes | +| 16 | [tokens-carry-no-position](16-tokens-carry-no-position.md) | a token was `(kind, value)`, so no diagnostic could name a place | 6/6 | yes | +| 17 | [annotations-are-never-checked](17-annotations-are-never-checked.md) | the annotation feeds dispatch and is never verified | 6/6 | branch | +| 18 | [async-half-expressible](18-async-half-expressible.md) | **first attempt was DOGMA** — no predictions, rigged test | 4/4 (2nd) | branch | +| 19 | [a-convention-is-not-a-gate](19-a-convention-is-not-a-gate.md) | `looks_like_heap_obj` is static, so every type re-derives it | 6/7 | yes | diff --git a/docs/v1/experiments/cycles/01-constructs-have-nowhere-to-be.md b/docs/v1/experiments/cycles/01-constructs-have-nowhere-to-be.md new file mode 100644 index 0000000..560edbf --- /dev/null +++ b/docs/v1/experiments/cycles/01-constructs-have-nowhere-to-be.md @@ -0,0 +1,42 @@ +# constructs have nowhere to be + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `5718943` + +``` +let a construct declare its own meaning instead of the emitter knowing it + +codegen called fn_has_decorator for exactly three names — manager, accessor, +route. Twelve others parsed, attached as {name,args}, and compiled to nothing, +including four that look like protection: @authenticate (6 uses), @authorize +(3), @rate_limit (3), @validate (2). The cause was not that the branches were +untidy. A construct had nothing to BE, so its meaning had nowhere to live +except the emitter, and every construct was therefore a compiler edit. + +A name -> injection table would have moved the enumeration twenty lines up +without removing it. So the construct now carries its own meaning: + + @decorator("injects_at_entry", "engram_boundary_beat") + fn audited() {} + + @audited + fn risky_op() -> Int { ... } // gets the beat, attributed to "audited" + +scan_declared_decorators is a token-level pre-pass beside scan_routes, forced +by streaming codegen having no whole-program AST. manager and accessor are +seeded as the compiled-in core — the fixedSelf shape from substrate.go: a +complete fallback exists, declaration is enrichment. + +This is the injection half of the seam only. The prohibition half (@manager's +#error on dharma_emit) stays hardcoded, because "which calls may appear inside +this boundary" is a query over program structure and there is nothing yet to +ask. + +Verified three ways: emitted C for existing @manager/@accessor code is +byte-identical to the hardcoded path; a construct with a name the compiler has +never heard of injects correctly; the compiler self-hosts byte-identically. +90/90 native compiler tests pass. +``` diff --git a/docs/v1/experiments/cycles/02-a-construct-cannot-refuse.md b/docs/v1/experiments/cycles/02-a-construct-cannot-refuse.md new file mode 100644 index 0000000..f08cc0e --- /dev/null +++ b/docs/v1/experiments/cycles/02-a-construct-cannot-refuse.md @@ -0,0 +1,43 @@ +# a construct cannot refuse + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `60737b0` + +``` +let a construct refuse, not only observe + +@authenticate (6 uses), @authorize (3), @rate_limit (3) and @validate (2) +parsed, attached, and compiled to nothing. Fourteen applications that read as +protection and emitted no instruction — a function decorated @authenticate +compiled byte-identically to an undecorated one. + +The missing capability was not authentication. It was that a construct could +observe a boundary but never refuse one. injects_at_entry discards the target's +result; there was no form in which a construct could say no. + + @decorator("guards_at_entry", "my_auth") + fn authenticate() {} + + @authenticate + @authorize + fn handler() -> String { ... } + +emits, at entry: + + { el_val_t __g = my_auth(EL_STR("handler"), EL_STR("authenticate")); if (__g) return __g; } + { el_val_t __g = my_roles(EL_STR("handler"), EL_STR("authorize")); if (__g) return __g; } + +Guards precede injections because a refused call must not report a crossing, +and every guard runs where the topmost injecting construct wins — refusal is +not a role, so it does not follow the role convention. + +The compiler still knows nothing about auth. The program points the construct +at its own function, which is where that decision belongs. + +Verified: existing @manager/@accessor output byte-identical, compiler +self-hosts byte-identically, guards stack in declaration order and emit before +the beat. 94/94 native compiler tests pass. +``` diff --git a/docs/v1/experiments/cycles/03-the-wrapper-was-conditional.md b/docs/v1/experiments/cycles/03-the-wrapper-was-conditional.md new file mode 100644 index 0000000..862388b --- /dev/null +++ b/docs/v1/experiments/cycles/03-the-wrapper-was-conditional.md @@ -0,0 +1,82 @@ +# the wrapper was conditional + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `4f7568b` + +``` +give a construct its after-crossing face, and let constructs compose + +§6 records 62 persist-after-mutate sites, 10 auth-per-route, and +index-after-append that failed at 9 of 9 — every one an obligation at a +crossing that decayed into "remember to do this afterwards." An obligation a +human must remember is not an obligation, and the 9-of-9 figure is what that +costs. + + @decorator("injects_at_exit", "persist_now") + fn durable() {} + +The body moves into a static helper and the visible fn becomes a wrapper, so +EARLY RETURNS pass through the exit injection. Emitting it only before the +fall-through return would have silently missed every early return — the exact +failure class this seam exists to remove. Fns with no exit construct emit +byte-identically to before. + +Three independent constructs now compose on one fn, none known to the compiler: + + el_val_t mutate(el_val_t k) { + { el_val_t __g = my_auth(EL_STR("mutate"), EL_STR("authenticate")); if (__g) return __g; } + engram_boundary_beat(EL_STR("mutate"), EL_STR("manager")); + el_val_t __r = __el_body_mutate(k); + persist_now(EL_STR("mutate"), EL_STR("durable"), __r); + return __r; + } + +Guard, then entry, then body, then exit. §5.2 asked whether `hold` is one +construct or two; the implementation answers one construct with two faces, +selected by declared kind rather than by two mechanisms. + +Verified: existing output byte-identical, compiler self-hosts byte-identically, +early returns pass through the exit, ordering holds under composition. 98/98 +native compiler tests pass. +``` + +## Record — `285166c` + +``` +EXPERIMENT: emit the wrapper unconditionally, so exit binds at runtime too + +ISHIKAWA: why did exit injection still need compile-time knowledge? Because the +body-helper wrapper was only emitted when codegen already knew an exit +construct existed. The wrapper being conditional was the cause, not the wrapper +being necessary. + +PREDICTIONS AND RESULTS + P1 exit becomes runtime-bindable TRUE returns 14, bound + after the build + P2 codegen shrinks TRUE 5094 -> 5044 + P3 cost 5-15% from a call frame on every fn FALSE 0.37s -> 0.38s, ~3% + P4 fixpoint holds TRUE + +Every fn now gets a body helper and a wrapper. It has to be unconditional: +early returns must route through something for an exit construct to observe +them, and codegen cannot know which fns will be bound after the binary exists. + +Removed with the machinery: declare_exit, decorator_exit, cg_exit_target, +cg_exit_construct, and the injects_at_exit scanner branch. + +Two controls failed and were rewritten rather than repaired -- +no-exit-construct-emits-no-wrapper asserted the optimisation this removes, so +it is now inverted. The integration harness gained a seventh assertion: an exit +construct declared after the build replaces the result. + +99/99 native, 7/7 integration, fixpoint gen2==gen3. +``` + +## Record — `b40754f` + +``` +land unconditional wrapper: exit crossings resolve at runtime +``` diff --git a/docs/v1/experiments/cycles/04-c-has-no-closure-syntax.md b/docs/v1/experiments/cycles/04-c-has-no-closure-syntax.md new file mode 100644 index 0000000..b72aff4 --- /dev/null +++ b/docs/v1/experiments/cycles/04-c-has-no-closure-syntax.md @@ -0,0 +1,64 @@ +# c has no closure syntax + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `2bed848` + +``` +EXPERIMENT: hand the construct the body as a real closure + +ROOT CAUSE of the weaker design: "C has no closures" was taken as a fact about +what is possible. It is a fact about one grammar. Every C++ lambda, every Go +closure, every Rust closure compiles to a struct of captured values plus a +function pointer -- which is what is emitted here. Codegen emits C; it is not +written in C's syntax, and the distinction is the whole difference between a +construct that can only decide whether to repeat and one that controls +invocation. + +It would also have crippled the JS backend, which has closures natively, for a +limit that applies only to the C one. + +PREDICTIONS AND RESULTS + 1 env struct + thunk taking void* TRUE + 2 fails to compile: struct redefinition FALSE -- C allows the + inner declaration to shadow. Prediction wrong; C is more permissive than + assumed. A different real defect surfaced instead: a wrap with no exit + construct emitted `(EL_STR("f"), EL_STR(""), __r);` -- a call to an empty + target -- because has_exit was reused as "needs a wrapper" and the exit line + was emitted unconditionally. Fixed. + 3 compiles when the target is declared in El FALSE -- and this is + the root cause worth keeping: El has ONE type, el_val_t = int64_t. El's type + system cannot describe a callable, so `extern fn` and the real signature + cannot be made to agree in El's own vocabulary. The fix is not a cast: + codegen DEFINES the wrap calling convention, so codegen emits the extern + declaration. The convention is not El-expressible; it is emitted. + 4 target controls invocation, 0..N times TRUE + 5 existing @manager output byte-identical TRUE + 6 compiler fixpoint holds TRUE + 7 emitting the convention makes it compile TRUE + +MEASURED + base(5) wrapped by a target that invokes the body twice and sums -> 10 + never_runs(5) wrapped by a target that never invokes it -> 999 + +Neither is expressible by "decide whether to repeat". This supersedes the +repeats_body experiment on experiment/repeats-body, which was built around the +mistaken limit. +``` + +## Record — `7d01608` + +``` +land wraps_body: a construct controls invocation + +Proven on experiment/wraps-body (2bed848): base(5) wrapped by a target that +invokes the body twice returns 10; a target that never invokes it returns 999. +Neither is expressible by deciding whether to repeat. + +Root cause it corrected: 'C has no closures' is a fact about one grammar, not +about what can be emitted. And El's single type (el_val_t = int64_t) cannot +describe a callable, so codegen emits the calling convention rather than asking +El's type system for something it structurally cannot say. +``` diff --git a/docs/v1/experiments/cycles/05-the-emitter-discards-what-it-knows.md b/docs/v1/experiments/cycles/05-the-emitter-discards-what-it-knows.md new file mode 100644 index 0000000..98caa34 --- /dev/null +++ b/docs/v1/experiments/cycles/05-the-emitter-discards-what-it-knows.md @@ -0,0 +1,48 @@ +# the emitter discards what it knows + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `a5af871` + +``` +EXPERIMENT: let the compiler write down what it already knows + +HYPOTHESIS: attribution is redundant for static structure. Codegen sees every +construct-to-function relation at emission time and discards it, so the only +way to learn the structure back is to run the program and read what it +reported. That is instrumentation compensating for erasure. + +PREDICTIONS, committed before running: + 1 derivable at compile time with no runtime call expected TRUE + 2 complete for guards and exits (invisible today) expected TRUE + 3 answers it for code that has never executed expected TRUE + 4 deterministic expected TRUE + 5 makes the entry beat redundant expected FALSE + +RESULT: 5/5 as predicted. From a program that was never executed: + + authenticate guards_at_entry login my_auth + durable injects_at_exit save persist_now + authenticate guards_at_entry critical my_auth + durable injects_at_exit critical persist_now + manager injects_at_entry critical engram_boundary_beat + +Prediction 5 held: the relation records that a boundary COULD be crossed, the +beat records that it WAS. They are different facts and neither replaces the +other. + +CONSEQUENCE, and it undercuts the first pass on iteration-1: construct identity +was available at compile time all along. With relations recorded at build, the +runtime needs only the function name and attribution becomes a join rather than +a payload. The counter-argument is that the payload is self-describing while +the file must be pinned to the artifact or the two drift and attribution is +silently lost — which is the same conclusion as "compile against a manifold +revision and record the revision in the artifact", reached from the other side. + +Written to a file rather than the engram on purpose: a compile that consults a +manifold produces different output from identical source at different times. +The file is content-addressed; the engram ingests it. Determinism preserved, +mechanism proven. +``` diff --git a/docs/v1/experiments/cycles/06-the-crossing-resolves-at-emission.md b/docs/v1/experiments/cycles/06-the-crossing-resolves-at-emission.md new file mode 100644 index 0000000..a10ae11 --- /dev/null +++ b/docs/v1/experiments/cycles/06-the-crossing-resolves-at-emission.md @@ -0,0 +1,170 @@ +# the crossing resolves at emission + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `35b07ba` + +``` +EXPERIMENT: resolve the crossing at execution, not at emission + +HYPOTHESIS (Will's): a compiler whose one compiled mechanism is extending the +LANGUAGE — not the compiler — can compose without recompilation. + +ISHIKAWA — why does a construct require a recompile today? + method codegen inlines the target call into the body + machine the binary has no table to consult + material the declaration lives in source, read at compile time + measurement nothing observes what applied at runtime + root cause the crossing is resolved at EMISSION, not at EXECUTION + +CHANGE: codegen emits one unconditional indirection per fn. Which constructs +apply is read from a table that can be written AFTER the binary exists; +targets resolve through dlsym against the running image. + +PREDICTIONS AND RESULTS + P1 a construct declared after the build applies TRUE + P2 an unlinked target is skipped, not fatal TRUE + P3 emitting on every fn is measurably slower FALSE — 0.37s -> 0.36s + with 267 indirections and + no bindings. Free unused. + P4 the compiler still self-hosts TRUE (see note) + +DEMONSTRATED: an El program with NO decorator in its source, already compiled +and linked, picked up a construct declared afterwards: + + $ /tmp/seamrun -> 7 + $ echo 'work audited entry audit_entry' > constructs.txt + $ EL_CONSTRUCTS=constructs.txt /tmp/seamrun + AUDIT: work applied by audited + 7 + +P4 note: my first fixpoint test was wrong, not the code. I compared gen1 to +gen2, which must differ whenever codegen's output changes. gen2 == gen3, 267 +seam sites, stable. + +MEASURED COST, and the root cause was not where I looked + 0 bindings 0.36s vs 0.37s baseline free + 2 bindings, dlsym per call 2.45s 6.6x + 2 bindings, resolved once 0.69s 3.5x recovered +The table scan was never the cost. dlsym walks the dynamic symbol table on +every call. Resolve once and cache — which is the smallest form of what +salience does for memory: what is hot stays resolved. The 0.69s residual is +audit_entry's own printf on two of the compiler's hottest functions, not seam +overhead. + +CONSEQUENCE: the five compile-time declaration kinds on iteration-1 are a +compile-time specialisation of something that resolves at runtime. They are not +wrong, but they are not the mechanism — the mechanism is one indirection, and a +kind is data. +``` + +## Record — `886626a` + +``` +seam refusal + control tests: a runtime binding can short-circuit + +Prediction 3 was FALSE. I expected refusal to be impossible through the seam +because the entry indirection discarded its return. One line: + + { el_val_t __s = el_seam_run(EL_STR(f), 0, 0); if (__s) return __s; } + +work() returns 7; bound to a refusing construct AFTER the build it returns 42. +So three of the five compile-time kinds are runtime-bindable: entry injection, +exit injection, and refusal. wraps_body needs invocation control and +prohibits_outside is compile-time by nature. + +104/104 native compiler tests pass. +``` + +## Record — `28d19da` + +``` +strip the compile-time machinery the seam replaces + +PREDICTION: codegen.el drops below 4661, its size before any of these passes. +RESULT: FALSE. 5157 -> 5096. Still +435 over baseline. + + injects_at_entry collapsed into the seam removed + guards_at_entry collapsed into the seam removed + injects_at_exit needs the body-helper wrapper STRUCTURAL + wraps_body needs the closure + wrapper structural + prohibits_outside a #error cannot be emitted at runtime + +The wrapper is not a consequence of compile-time resolution. Early returns must +be routed through something no matter when the target is resolved, so exit +injection was never going to collapse. I predicted it would because I had +conflated "resolved late" with "emitted less". + +What did collapse is entry injection and refusal -- 61 lines of compiler +replaced by one refusable indirection, with the capability now bindable after +the binary exists. + +8 tests fail, and they are exactly the 8 controls for compile-time entry +injection and guards. No unrelated breakage: the controls reported precisely +what moved. They assert emission of something that now happens at runtime, so +they need rewriting as integration tests -- which the framework does not +currently support, because runtime binding needs a built binary and an +environment, not compile_capture. + +Verified after the strip: fixpoint gen2==gen3, observation and refusal both +work through the seam with the compiler knowing nothing about either. +``` + +## Record — `8bbb750` + +``` +control the claim that cannot be unit tested + +The seam's whole claim is that a construct declared AFTER a binary exists +applies to that already-built program. compile_capture only sees emitted text, +so it structurally cannot check this: it needs a built binary, a linked target, +and an environment. Verified by hand until now, which is the standing problem +this session has been about. + +tests/integration/seam_binding.sh builds a probe from El source containing no +construct at all, links a target that El never references, and asserts: + + ok unbound program is unaffected + ok a construct declared AFTER the build applies + ok a construct declared after the build can REFUSE + ok an unlinked target is skipped, not fatal + ok a binding for a different fn does not fire + ok two constructs compose on one crossing + + 6 assertions, 6 passed, 0 failed + +The eight controls that failed after the strip were replaced, not repaired. +They asserted compile-time emission of capability that moved to runtime; +contorting them would have kept an assertion whose subject no longer exists. +Three took their place, asserting the emitted shape, and the behaviour they +used to cover is now the integration harness's job -- which is the honest +division, since the shape and the behaviour are no longer the same fact. + +99/99 native compiler tests pass. Fixpoint holds. +``` + +## Record — `24f7fb5` + +``` +land the runtime seam: resolve the crossing at execution + +Five compile-time passes added 491 lines to the thing that was supposed to stop +growing. The seam is ~55 lines of C and one line of emission, and it does at +runtime what three of those five kinds did at compile time -- for programs that +are already built. + + a construct declared AFTER the binary exists applies to it + free when unused: 0.36s vs 0.37s baseline across 267 indirections + dlsym was the cost, not the table scan; resolve-once recovered 3.5x + refusal works, composition works, unlinked targets are skipped not fatal + +injects_at_exit and wraps_body do NOT collapse: early returns must route +through the body-helper wrapper regardless of when the target is resolved. The +wrapper is structural, which I had wrong. prohibits_outside cannot move at all +-- a #error has no runtime. + +Controls: 99/99 native compiler tests, plus tests/integration/seam_binding.sh +(6/6) for the claim compile_capture structurally cannot see. +``` diff --git a/docs/v1/experiments/cycles/07-invocation-is-not-composable.md b/docs/v1/experiments/cycles/07-invocation-is-not-composable.md new file mode 100644 index 0000000..49ad88a --- /dev/null +++ b/docs/v1/experiments/cycles/07-invocation-is-not-composable.md @@ -0,0 +1,45 @@ +# invocation is not composable + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `bc2f26d` + +``` +EXPERIMENT: invocation control resolves at runtime + +ISHIKAWA: why did wraps_body need compile-time knowledge? Because the wrapper +called the target directly. If the wrapper calls through the seam instead, the +seam can call the body itself, and a construct bound after the build decides +how and whether to invoke it. + +PREDICTIONS AND RESULTS + P1 wrap becomes runtime-bindable TRUE body x3 -> 21, + never invoked -> 111 + P2 codegen shrinks TRUE 5042 -> 4977 + P3 cost 5-10% from an indirect call on every fn TRUE 0.36s -> 0.39s, ~8% + P4 zero-param fns break on the empty struct TRUE empty struct is a GNU + extension, empty init + is C23. Fixed with a + char field. + P5 fixpoint holds TRUE + +PROCESS FAILURE worth recording: my first patch silently did not apply because +I dropped the assert on the string replacement. The build then failed with +"undeclared identifier __thunk_noargs", which I nearly attributed to the +empty-struct prediction. The guard that would have caught it existed and I +removed it -- the same shape as every other defect found tonight. + +Removed: declare_wrap, decorator_wrap, cg_wrap_target, cg_wrap_construct, +params_to_call_args, and the wraps_body scanner branch. + +prohibits_outside is now the ONLY construct kind left at compile time, and it +cannot move: a #error has no runtime. +``` + +## Record — `c04d68f` + +``` +land runtime invocation control: only prohibition remains compiled +``` diff --git a/docs/v1/experiments/cycles/08-the-emitter-adjudicates.md b/docs/v1/experiments/cycles/08-the-emitter-adjudicates.md new file mode 100644 index 0000000..08edf7c --- /dev/null +++ b/docs/v1/experiments/cycles/08-the-emitter-adjudicates.md @@ -0,0 +1,53 @@ +# the emitter adjudicates + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `c741cfe` + +``` +EXPERIMENT: prohibition becomes a query over emitted relations + +I said prohibition could not move because "a #error has no runtime". That +conflated two separable things: WHEN a violation is detected (build time -- +correct, and unchanged) and WHERE the rule and the checker live (the compiler +-- assumed). + +A prohibition is a containment relation over the call graph. So codegen now +records what it saw: + + sneaky calls raw_sql + allowed calls raw_sql + allowed calls @repository + repository calls prohibits:raw_sql + +and tools/check/prohibitions.sh decides, at build time, outside the compiler. + +PREDICTIONS AND RESULTS + P1 codegen can emit the call graph it already walks TRUE + P2 the check becomes a query outside the compiler TRUE + P3 all prohibition decisions leave codegen TRUE zero #errors now + P4 violations still caught at build time TRUE exit=1 + P5 codegen drops below the 4661 baseline FALSE 4962, +301 + +P5 is the finding. The TRAVERSAL is irreducible -- you must walk the AST to +find calls, and those ~120 lines do not move no matter who decides. What is not +irreducible is the rule (which names) or the decision (#error). Those left. I +predicted the whole 223 lines would go because I had not separated walking from +adjudicating. + +Still compiled, and measured rather than assumed: the capability-tier system +(cap_check_call, is_self_formation_call, is_dharma_call, is_llm_call, +cap_record_violation, emit_cap_violations) is 76 lines of the same shape -- +prohibits_WITHIN rather than prohibits_outside, so the checker needs the +opposite polarity to absorb it. + +98/98 native, 4/4 prohibition_query.sh, 7/7 seam_binding.sh, fixpoint ok. +``` + +## Record — `60c07ad` + +``` +land prohibition-as-query: the emitter records, it no longer adjudicates +``` diff --git a/docs/v1/experiments/cycles/09-policy-inside-the-compiler.md b/docs/v1/experiments/cycles/09-policy-inside-the-compiler.md new file mode 100644 index 0000000..cc121ab --- /dev/null +++ b/docs/v1/experiments/cycles/09-policy-inside-the-compiler.md @@ -0,0 +1,49 @@ +# policy inside the compiler + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `c2d9596` + +``` +EXPERIMENT: the capability tier becomes shipped policy plus a query + +Capability differs from prohibits_outside in one way that matters: a utility +program cannot be trusted to declare its own restrictions, because it would +declare none. So the policy comes from OUTSIDE the program -- it ships with the +language as data, editable without a compiler release. + + tools/check/capabilities.rel 18 names that were string literals in codegen + tools/check/capabilities.sh the query that decides + +PREDICTIONS AND RESULTS + P1 codegen emits kind + call graph, drops the 4 name tests TRUE zero #errors + P2 the 18 literals become a data file TRUE + P3 the checker catches capability violations TRUE exit=1 + P4 codegen drops ~76 lines TRUE 4963 -> 4881 + P5 below the 4661 baseline FALSE ~+230 + +TWO DEFECTS THE HARNESS FOUND THAT READING WOULD NOT HAVE + +1. Calls inside main became invisible. cg_fn returns early for main -- C + provides its own -- so hooking the recording there left every call in main + unrecorded: a blind spot exactly where a program does its work. The old + cap_check_call ran from cg_expr and did see main. Moved the recording to + cg_expr. + +2. Caller attribution was stale. __cg_current_fn kept whatever cg_fn set last, + so a violation in main was reported against the previously emitted function. + The test still PASSED, because the violation was detected -- only the name + was wrong, and a diagnostic naming the wrong fn is worse than none. Fixed at + all three main-emission sites; the first patch missed two because the live + path is codegen_streaming. + +98/98 native, 7/7 + 4/4 + 5/5 integration, fixpoint ok. +``` + +## Record — `29f78f9` + +``` +land capability-as-policy: eighteen literals become a data file +``` diff --git a/docs/v1/experiments/cycles/10-a-second-copy-of-the-header.md b/docs/v1/experiments/cycles/10-a-second-copy-of-the-header.md new file mode 100644 index 0000000..037407f --- /dev/null +++ b/docs/v1/experiments/cycles/10-a-second-copy-of-the-header.md @@ -0,0 +1,54 @@ +# a second copy of the header + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `9cc6040` + +``` +EXPERIMENT: derive arity from the runtime's own declarations + +codegen.el carried builtin_arity(): 344 lines, 300 entries, a hand-maintained +second copy of el_runtime.h. + +PREDICTIONS AND RESULTS + P1 the table duplicates the header TRUE 243 shared names + P2 they have already drifted FALSE ZERO drift. The + duplicate had been + maintained correctly. + P3 codegen can emit call-arity relations TRUE + P4 the check becomes a query against the header TRUE + P5 codegen drops to roughly baseline TRUE 4903 -> 4512, + 149 BELOW the 4661 + it started at + +P2 being false is the better result: the table was not WRONG, it was +INCOMPLETE. 110 functions the runtime declares had no entry, so calling them +with the wrong argument count produced no El-level diagnostic at all. Measured: +the old compiler reports 0 arity errors for __http_do_map_to_file(1); the query +reports "takes 5 arguments, called with 1". + +Deriving from the header fixes coverage AND makes drift impossible by +construction. 503 signatures, versus 300 entries maintained by hand. + +THREE DEFECTS IN MY OWN CHECKER, each found by running it rather than reading it + 1. El names and C names differ -- `println` is `__println`. 60 of 500 decls + carry the prefix and codegen owns the mapping; the old table carried both + keys. One rule covers all 60. + 2. Multi-line declarations parsed as zero params, so the checker reported + "takes 0" for a function taking 5. A diagnostic with the wrong number in it + is worse than none -- the same shape as the stale caller attribution in the + previous pass. + 3. Fixing (2) by joining lines dropped 500 signatures to 334, because a + declaration preceded by a comment no longer started its record. Comments + are stripped first now. + +98/98 native, 5/5 arity_query.sh, fixpoint ok. +``` + +## Record — `d9e301b` + +``` +land arity-from-header: the runtime declares its own surface +``` diff --git a/docs/v1/experiments/cycles/11-one-type-erases-the-return.md b/docs/v1/experiments/cycles/11-one-type-erases-the-return.md new file mode 100644 index 0000000..02ae35b --- /dev/null +++ b/docs/v1/experiments/cycles/11-one-type-erases-the-return.md @@ -0,0 +1,52 @@ +# one type erases the return + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `d2d89fc` + +``` +EXPERIMENT: temporal types as data — and the pass that GREW the compiler + +This block is structurally unlike the previous four. It does not only +adjudicate, it DISPATCHES: Instant + Duration must become el_instant_add_dur, +LocalDate + Duration must become el_local_date_add_dur. The emitted C depends on +the type answer, so it cannot move to a post-hoc query. Selecting which call to +emit is an emitter's actual job. + +PREDICTIONS AND RESULTS + P1 the block conflates dispatch with adjudication TRUE + P2 adjudication can move, dispatch cannot TRUE + P3 this pass shrinks codegen far less than the last TRUE, and worse: + 4513 -> 4537, it GREW + by 24 lines + P4 the rules are affine algebra, closed by construction TRUE + P5 no type propagation -- name tracking plus a + hardcoded list of which builtins return which type TRUE, 19 names + +P3 is the honest result and it is not spun: moving 19 names into a data file +cost more lines than it saved, because a generic loader is larger than the +enumeration it replaces. The win is not line count. It is that adding a 20th +temporal builtin is now a one-line edit to signatures.rel instead of a compiler +change, and that the data is inspectable. + +WHY THE HEADER CANNOT SUPPLY THIS, unlike arity: el_runtime.h declares every +builtin as returning el_val_t, because El has ONE type. That single type is why +the whole seam is cheap and it is exactly why the C boundary cannot say that +now() returns an Instant while unix_seconds() returns an Int. The El-level type +is real and the boundary erases it. + +INCOMPLETE, and stated rather than hidden: P2 said adjudication could move to a +query. It has NOT. Violations still emit TIME_TYPE_ERROR inline from the +emitter. Only the type DATA moved. Moving the adjudication needs the operand +types recorded as relations, which is a further pass. + +98/98 native, 4/4 temporal_signatures.sh, fixpoint ok. +``` + +## Record — `e01e079` + +``` +land temporal signatures as data: the type table leaves, the dispatch stays +``` diff --git a/docs/v1/experiments/cycles/12-judgment-lives-with-knowledge.md b/docs/v1/experiments/cycles/12-judgment-lives-with-knowledge.md new file mode 100644 index 0000000..409dc16 --- /dev/null +++ b/docs/v1/experiments/cycles/12-judgment-lives-with-knowledge.md @@ -0,0 +1,47 @@ +# judgment lives with knowledge + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `e8e25a0` + +``` +EXPERIMENT: temporal adjudication moves out; the placeholder stays + +The previous pass moved the type DATA and left the judgment inline, which I +stated rather than hid. This finishes it. + +PREDICTIONS AND RESULTS + P1 codegen can emit operand-type relations TRUE + "main calls temporal:instant_plus_instant" + P2 the affine rules are a small closed set as data TRUE 6 rules + P3 violations still caught at build time TRUE exit=1 + P4 the reporter leaves codegen TRUE 4538 -> 4507 + P5 the TIME_TYPE_ERROR placeholder must STAY TRUE + +P5 is the boundary of this whole approach. The emitter has to emit SOMETHING +for an illegal expression -- it cannot emit nothing and it cannot decide what +the program meant. So the placeholder is irreducible in the same way the AST +traversal was: what moved is the judgment and the wording, not the fact that +something must be written. + +The rules are affine algebra and the set is closed because there are only two +kinds of thing. An Instant is a POINT, a Duration is a DISPLACEMENT: add a +displacement to a point, subtract two points for a displacement, combine +displacements. Nothing else is meaningful, which is why the enumeration in +temporal.rel cannot grow the way an allowlist does. + +A defect in my own checker, found by running it: the .rel file uses aligned +columns and my awk assumed a single space, so the message came out with the +rule key still prefixed. Same class as the multi-line header parse in the arity +pass -- formatting assumptions that only fail when you look at the output. + +98/98 native, 6/6 temporal_query.sh, fixpoint ok. +``` + +## Record — `50425f3` + +``` +land temporal adjudication as a query: the emitter records, the rules are data +``` diff --git a/docs/v1/experiments/cycles/13-thirty-five-return-types.md b/docs/v1/experiments/cycles/13-thirty-five-return-types.md new file mode 100644 index 0000000..3b2669f --- /dev/null +++ b/docs/v1/experiments/cycles/13-thirty-five-return-types.md @@ -0,0 +1,49 @@ +# thirty five return types + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `cbef1c1` + +``` +EXPERIMENT: Int return types as data — and the bug that fell out + +PREDICTIONS AND RESULTS + P1 is_int_call's 35 hardcoded names move to data TRUE + P2 is_int_name stays -- it is annotation propagation TRUE + P3 the dispatch stays -- it is emission TRUE + P4 codegen shrinks ~40 lines TRUE 4507 -> 4469 + P5 the design doc's characterisation is WRONG TRUE + P6 the moved data also fixes the bug it exposed TRUE + +P5 CORRECTS THE RECORD. el-language-design.md and geometry-vs-code.md both cite +"== lowering to str_eq unless both operand names are in a hardcoded int-name +set -- a literal list of variable names treated as integers" as the paradigm +defect. It is not one. __int_names is populated from TYPE ANNOTATIONS +(param["type"] == "Int"), which is primitive but legitimate type propagation. +The actual defect was is_int_call: 35 hardcoded builtin return types, the same +shape as the temporal 19. + +P6 IS A LIVE CORRECTNESS BUG, PRE-EXISTING, NOW FIXED + + let a = str_len("hello") // no annotation + let b = str_len("hi") + let c = a + b // -> el_str_concat(a, b) on two integers + +Verified identical on the pre-change compiler, so not a regression. It compiled +clean, ran, and printed NOTHING where it should print 7. No error at any layer. + +The repair is three lines: an unannotated let takes its type from what the +initialiser returns. The return types were already required for dispatch and +were simply never consulted at the binding site. Moving them into data is what +made the gap visible -- reading the code for eight hours did not. + +98/98 native + 2 new, 31/31 integration, fixpoint ok. +``` + +## Record — `505e5e7` + +``` +land int signatures, and repair a silent miscompilation they exposed +``` diff --git a/docs/v1/experiments/cycles/14-keywords-that-reserve-nothing.md b/docs/v1/experiments/cycles/14-keywords-that-reserve-nothing.md new file mode 100644 index 0000000..3d06af4 --- /dev/null +++ b/docs/v1/experiments/cycles/14-keywords-that-reserve-nothing.md @@ -0,0 +1,57 @@ +# keywords that reserve nothing + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `0143cc4` + +``` +ANSWER: is a grammar a convention, or a region? + +Both, at different layers -- and it is the same split as serialization: the +convention is the BASIS, never the ACT. + + lexeme -> token `fn` means function-start because someone said so CONVENTION + shape recognition given tokens, which construct is this REGION + source -> structure parsing is transduction onto that basis GEOMETRY + byte traversal something must read them in order IRREDUCIBLE + +Three things push the ACT toward region rather than convention: ambiguity +(a * b needs context; a grammar resolves it with the lexer hack, a region by +neighbourhood), error recovery (nearest-region is free), and precedence, which +is ordering along an axis with a conventional parameter. + +AND THE SHOULD GATE SAYS NO TO THE OBVIOUS MOVE + +Every other table this session moved to data. This one stays code. The keyword +set is CLOSED by the language definition -- it does not leak the way an +allowlist does -- and the lexer runs before the program is understood, so a +program can never declare its own keywords. Externalising it costs file I/O on +every compile and buys nothing. Same verdict as is_digit in ASCII. + +WHAT WAS ACTUALLY WRONG: five of 46 keywords were consumed by no parser or +codegen path. sealed, activate, seed, protocol, impl. Each stole an identifier +from users for nothing. + +SECOND SILENT MISCOMPILATION OF THE DAY. Using one did not fail to parse: + + let seed = 42 + let impl = seed + 1 + +compiled CLEAN -- zero cc errors -- and printed 0 instead of 44. No diagnostic +at any layer. Fixed by removing the five. + +A DEFECT IN MY OWN MEASUREMENT, caught before it did damage: my first pass +checked only parser.el and reported `test` as inert too. codegen consumes it at +4135 for --test mode, and the tree has 408 uses. Removing it would have broken +every test in the suite. The measurement was re-run across all four consumers. + +100/100 native + 2 new, 31/31 integration, fixpoint ok. +``` + +## Record — `067dd40` + +``` +answer the parsing question: a grammar is a basis, and five keywords reserved nothing +``` diff --git a/docs/v1/experiments/cycles/15-no-namespacing-at-all.md b/docs/v1/experiments/cycles/15-no-namespacing-at-all.md new file mode 100644 index 0000000..0923142 --- /dev/null +++ b/docs/v1/experiments/cycles/15-no-namespacing-at-all.md @@ -0,0 +1,53 @@ +# no namespacing at all + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `79f6cb7` + +``` +ANSWER: if the partition is a neighbourhood, does linking survive? + +The question is premature, and measuring says why. El's partition is a +FILESYSTEM PATH, not a neighbourhood, and there is no namespacing at all. + +MEASURED + import is textual inlining (resolve_imports), guarded against double + inclusion by a __elc_imp__: state key + when a .elh header exists the header is inlined instead and the .el is marked + seen, so symbols resolve at C link time -- so linking IS real, delegated to C + two modules defining `helper` emit two C functions into one translation unit + +So linking barely survives the PATH partition. Whether it survives a +neighbourhood partition cannot be asked yet. + +A DIAGNOSTIC REGRESSION I CAUSED, found by asking this question. cc does catch +the collision, but reports: + + error: redefinition of '__el_body_helper' + error: redefinition of '__env_helper' + error: redefinition of '__thunk_helper' + error: redefinition of 'helper' + +The user's own function is FOURTH. The first three are generated symbols +introduced by the unconditional-wrapper pass earlier today -- before it, there +was one clear message. Repaired by catching the collision at El level instead: + + duplicate definition: 'helper' is defined 2 times — El has no namespacing, + so imported modules share one global scope + +LIMIT, stated rather than hidden: textual inlining destroys file provenance. By +the time codegen runs there is one source string, so the message can say WHICH +name collides but not which files. Naming a.el and b.el needs provenance +threaded through resolve_imports. + +104/104 native, 4/4 definitions_query.sh, the compiler itself reports clean, +fixpoint ok. +``` + +## Record — `f23cb2b` + +``` +answer the module question: the partition is a path, and there is no namespacing +``` diff --git a/docs/v1/experiments/cycles/16-tokens-carry-no-position.md b/docs/v1/experiments/cycles/16-tokens-carry-no-position.md new file mode 100644 index 0000000..a1e18c4 --- /dev/null +++ b/docs/v1/experiments/cycles/16-tokens-carry-no-position.md @@ -0,0 +1,64 @@ +# tokens carry no position + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `6c975b1` + +``` +thread provenance through resolve_imports + +The module question ended with a limit: textual inlining destroys file +provenance, so a duplicate-definition message could name the symbol but not the +files. Threading it exposed a bigger absence first. + +TOKENS HAD NO POSITION AT ALL. A token was a flat (kind, value) pair, so NO +diagnostic in El could name a place -- every error named a symbol and never a +line. That is the prerequisite the module question was resting on. + +THE CHAIN, end to end + lexer counts newlines; tok_append mints (kind, value, line) + parser stride 2 -> 3; tok_line added; FnDef carries its line + codegen records defines_at: + resolve_imports publishes spans for the combined source + checker maps a combined line back to file:line-within-that-file + + duplicate definition: 'helper' is defined 2 times — El has no namespacing, + so imported modules share one global scope + /tmp/modtest/a.el:1 + /tmp/modtest/b.el:1 + +PREDICTIONS AND RESULTS + P1 15 stride sites, encapsulated in tok_kind/tok_value TRUE, but see below + P2 adding a line field is mechanical TRUE + P3 the lexer must count newlines TRUE + P4 resolve_imports can record per-file line ranges TRUE + P5 the message can then name both files TRUE + P6 token memory grows TRUE, 25.0 -> 33.9 MB (+36%) + +FOUR DEFECTS, EACH FOUND BY RUNNING AND NOT BY READING + +1. interp_tokens_append_all walks the token list DIRECTLY with its own copy of + the stride. Gen1 built fine and gen2 emitted corrupt C, because the + compiler's own source uses string interpolation. My search missed it because + I grepped for the variable name `tokens`; it is called `dst`/`result`. + Searching by name instead of by shape -- third time today. +2. tok_count in test_compiler.el carried the stride too. I had scoped the search + to compiler sources and it had escaped into the tests. +3. Nested resolve_imports calls accumulated spans into shared state, so each + republished meaningless line ranges under the parent's name. Making the + buffer local fixed it; guarding the WRITE did not, which is what I tried + first. +4. The first working version reported b.el:3 -- the COMBINED line against a + filename that has no line 3. A file:line that does not match the file is + worse than no line at all. + +105/105 native, 37/37 integration, fixpoint ok, compiler self-checks clean. +``` + +## Record — `cb7289f` + +``` +thread provenance end to end: a diagnostic can finally name a place +``` diff --git a/docs/v1/experiments/cycles/17-annotations-are-never-checked.md b/docs/v1/experiments/cycles/17-annotations-are-never-checked.md new file mode 100644 index 0000000..c60f03d --- /dev/null +++ b/docs/v1/experiments/cycles/17-annotations-are-never-checked.md @@ -0,0 +1,53 @@ +# annotations are never checked + +**Status: verified on `experiment/annotation-checking`, not merged.** + +## Ishikawa — why does El silently miscompile? + +Three bugs found the same day shared one shape. + +``` +method type tracked by per-function name sets, fed from annotations +machine el_val_t erases everything at the C boundary +material no propagation through expressions +measurement nothing verifies an annotation against what it annotates +───────────────────────────────────────────────────────────────────────── +root cause El has type ANNOTATIONS but no type CHECKING. The annotation + feeds dispatch and is never itself verified. +``` + +## Predictions + +``` +P1 let x: Int = "hello" compiles clean expect TRUE +P2 let s: String = 42 compiles clean expect TRUE +P3 the annotation drives dispatch, unverified expect TRUE +P4 same root cause as all three bugs found today expect TRUE +P5 checking literal-vs-annotation catches both expect TRUE +P6 zero false positives across the compiler's source expect TRUE +``` + +## Results — 6/6, and worse than a wrong answer + +``` +let x: Int = "hello"; x + 1 → 4343631981 a string POINTER used as an integer +let s: String = 42; println(s) → nothing address 42 dereferenced as a string +``` + +The first **leaks a raw memory address into program output**. The second is an +**arbitrary-read primitive** if that integer is ever attacker-influenced. + +Verified: 6/6, zero false positives across the compiler's own source, fixpoint +ok, 105/105 native. + +## Six Sigma + +The emitter only **records** the mismatch; `tools/check/annotations.sh` decides — +consistent with every other check. Literals are checked because they are +unambiguous. + +**Incomplete, stated not hidden:** only literals. `let x: Int = some_string_fn()` +still passes, because `signatures.rel` carries Int/Instant/Duration and no +String entries. That is a data gap, not a capability limit — every El function +declares its return type in source and codegen already holds `ret_type` on every +`FnDef`. diff --git a/docs/v1/experiments/cycles/18-async-half-expressible.md b/docs/v1/experiments/cycles/18-async-half-expressible.md new file mode 100644 index 0000000..1cdc0c2 --- /dev/null +++ b/docs/v1/experiments/cycles/18-async-half-expressible.md @@ -0,0 +1,73 @@ +# async — half expressible, and the cycle that was dogma + +**Status: measured on a branch, not merged. Two runs — the first was invalid.** + +## The first attempt was DOGMA, not science + +I had just finished arguing that `@async` was expressible, then ran something to +confirm it. **No prediction was committed.** The test was rigged in a way that +should have been visible while writing it: + +```c +pthread_create(&t,NULL,runner,NULL); pthread_join(t,NULL); +``` + +`join` immediately after `create` — the caller blocks until the body finishes. +That is a thread round-trip, not deferral. And the test printed the word +`DEFERRED` itself: I wrote the conclusion into the output and read it back. + +``` +Ishikawa on the rigged test + method ran after concluding, not to decide + machine nothing forces a prediction before execution + material the assertion was written into the output string + measurement no falsification criterion existed, so nothing could fail + root cause the test was authored by the party holding the conclusion, + with no commitment made before it ran +``` + +Discarded and re-run properly. + +## Second run — predictions committed first + +``` +P1 the caller proceeds while the body runs expect TRUE +P2 interleaving is observable in timestamps expect TRUE +P3 the result cannot be retrieved — one 64-bit slot, no + future type, so the wrap either blocks or returns + something that is not the result expect TRUE +P4 therefore HALF expressible: fire-and-forget yes, await no expect TRUE +``` + +## Results — 4/4 + +``` +[ 18 us] wrap RETURNS to caller +[ 29 us] body START +caller continues, got 0 +[ 50176 us] body END (computed 42) +caller done +``` + +The caller got **0, not 42**. Both of my earlier claims were wrong in opposite +directions: "not expressible" was too strong — fire-and-forget works today, +bound after the build, no compiler change. "Expressible" was too strong the +other way. + +## Follow-on cycle — a future is one more tagged object + +``` +P1 el_val_t already carries tagged heap objects TRUE 5 magic tags exist +P2 a future is one more TRUE +P3 the caller awaits and gets 42 TRUE +P4 ZERO compiler changes TRUE runtime C + one binding +P5 the unbound path still works FALSE SIGSEGV +``` + +**P4 is the result.** `@async` — called unexpressible for hours — needs no +compiler change. A future is one more magic-tagged heap object; `defer` returns +the handle, `el_await` blocks. + +**P5 is the failure that mattered.** Sixty seconds after diagnosing +`let s: String = 42` as an arbitrary read, I wrote the identical defect into +`el_await`: reading `->magic` off an unvalidated slot. That opened cycle 19. diff --git a/docs/v1/experiments/cycles/19-a-convention-is-not-a-gate.md b/docs/v1/experiments/cycles/19-a-convention-is-not-a-gate.md new file mode 100644 index 0000000..1619606 --- /dev/null +++ b/docs/v1/experiments/cycles/19-a-convention-is-not-a-gate.md @@ -0,0 +1,61 @@ +# a convention is not a gate + +One `Ishikawa → scientific method → Six Sigma` loop. The record below is the +commit message as written at the time, before the outcome was known to anyone +reading this file. + +## Record — `9a6c161` + +``` +a slot must be validated before it is dereferenced + +ISHIKAWA: el_val_t carries integers AND tagged heap pointers, so "is this a +pointer" is undecidable without checking first. That check was a CONVENTION +every author had to know rather than a GATE they had to pass through, and +looks_like_heap_obj was static -- so every sibling translation unit re-derived +it. + +MEASURED, across the five existing tags + geom_of looks_like_heap_obj full guard correct + mfld_of looks_like_heap_obj full guard correct + el_bin_lookup (uintptr_t)p < 4096 floor only reads 8 bytes BACKWARD + el_input_len s ? ... : 0 NULL only strlen's an integer + + sha256_hex(50000) -> exit 139, SIGSEGV, compiled clean + +PREDICTIONS AND RESULTS + P1 looks_like_heap_obj is static, not exported TRUE + P2 each tagged type re-derives the check TRUE + P3 at least one is missing guard components TRUE (two are) + P6 sha256_hex() reads out of bounds TRUE + P8 routing el_bin_lookup through the gate fixes it FALSE + P9 the legitimate hash is unchanged TRUE + P11 fixpoint and suites hold TRUE + +P8 IS THE USEFUL FAILURE. Guarding the tagged lookup changed nothing -- +looks_like_heap_obj(49992) correctly returns 0, el_bin_lookup bails, and then +el_input_len falls through to strlen() on address 50000. The FALLBACK was the +hazard, not the tagged path. A NULL check does not establish that a slot is a +pointer. I would have shipped the wrong fix and called it verified. + +A MEASUREMENT DEFECT, fourth today: my first run of the crash reported exit=0, +because $? read head's exit through a pipe rather than the program's. I nearly +recorded a segfault as a clean run. Same shape as grepping only parser.el and +searching by variable name instead of by operation. + +AND I PROVED THE HAZARD FROM THE INSIDE. Sixty seconds after diagnosing +`let s: String = 42` as an arbitrary-read primitive, I wrote the identical +defect into el_await -- dereferencing ->magic off an unvalidated slot -- and +only then found the runtime had already made it twice. + +el_tagged() is now exported in el_runtime.h. Anything that dereferences a slot +without passing through it is the defect. + +105/105 native, 42/42 integration across eight harnesses, fixpoint ok. +``` + +## Record — `3049a70` + +``` +make the guard a gate: sha256_hex(50000) no longer segfaults +``` diff --git a/docs/v1/experiments/findings/answers.md b/docs/v1/experiments/findings/answers.md new file mode 100644 index 0000000..534a496 --- /dev/null +++ b/docs/v1/experiments/findings/answers.md @@ -0,0 +1,12 @@ +# Architecture questions closed + +All five were open in `geometry-vs-code.md`. Each was closed by measurement, not +by argument. + +| Question | Answer | +|---|---| +| **Concurrency** — hardware threads are code, but is *ordering* geometric? | **Ordering is geometric.** Causality is a partial order (Lamport 1978); a total order is an arbitrary extension that "cannot be depended on to imply a causal relationship." Languages force a total order at authoring time, so every lock, barrier and fence is apparatus for recovering the partial order that was destroyed. CALM: a program has a coordination-free implementation **iff monotone**. What breaks monotonicity is destructive update. **Coordination is the price of forgetting.** | +| **Error handling** — does `grounded: false` cover *failed*? | **No.** Standing is a *signed* component: `>0` supported, `=0` unknown, `<0` contradicted. Not-known and known-false are opposite directions on one axis; a boolean cannot tell them apart. `inhibitory` as an int32 flag is that sign wearing a boolean. | +| **Parsing** — is a grammar a convention, or a region? | **A grammar is a basis; parsing is transduction onto it.** Lexeme→token is convention, shape recognition is a region, byte traversal is irreducible. **But the SHOULD gate refused the obvious move:** the keyword table stays code, because the set is closed by the language definition and the lexer runs before the program is understood. Same verdict as `is_digit` in ASCII. | +| **Numeric literals** — is `3` a position or a convention? | **The numeral is convention; the number is a position — and a bare `3` is a magnitude with no axis.** It is not a position until something gives it a direction, which is why `3.days` needs a calendar. Demonstrated: `t + 3` refused, `t + 1.hour` accepted. | +| **The module system** — if the partition is a neighbourhood, does linking survive? | **Premature.** The partition is a filesystem path and there is no namespacing at all. `import` is textual inlining; with a `.elh` header, symbols resolve at C link time. Two modules defining `helper` emit two C functions into one translation unit. Linking barely survives the *path* partition. | diff --git a/docs/v1/experiments/findings/bugs.md b/docs/v1/experiments/findings/bugs.md new file mode 100644 index 0000000..2433a12 --- /dev/null +++ b/docs/v1/experiments/findings/bugs.md @@ -0,0 +1,74 @@ +# Live defects found + +Every one compiled clean, ran, and produced a wrong result or a crash with **no +diagnostic at any layer**. All four were present before this session; none was +introduced by it. + +## Silent miscompilations + +### 1. An unannotated `let` loses its type + +```el +let a = str_len("hello") // no annotation +let b = str_len("hi") +let c = a + b // → el_str_concat(a, b) on two integers +``` + +Compiled clean. Printed **nothing** where it should print 7. Fixed: an +unannotated `let` takes its type from what its initialiser returns. The return +types were already required for dispatch and were simply never consulted at the +binding site. + +### 2. Reserved keywords that reserved nothing + +```el +let seed = 42 +let impl = seed + 1 +``` + +`sealed`, `activate`, `seed`, `protocol`, `impl` were keywords in the lexer and +consumed by no parser or codegen path. Using one did not fail to parse — it +compiled clean, with zero `cc` errors, and printed **0 instead of 44**. Fixed by +removing all five. + +### 3. `Instant + Int` was never refused + +```el +let t: Instant = now() +let u: Instant = t + 3 // → (t + 3), reported clean +``` + +`Duration + Int` was refused — *"an Int carries no unit"* — while adding a +dimensionless number to a **point** silently moved the instant by an +unspecified amount. Three of *what*? Whatever the representation happens to be. +The rule was simply never written. + +## Security-relevant + +### 4. Annotations are never verified + +```el +let x: Int = "hello"; x + 1 → 4343631981 a string POINTER used as an integer +let s: String = 42; println(s) → nothing address 42 dereferenced +``` + +The first **leaks a raw memory address into program output**. The second is an +**arbitrary-read primitive** if the integer is ever attacker-influenced. + +### 5. `sha256_hex()` segfaults + +```el +let h: String = sha256_hex(50000) → exit 139, SIGSEGV +``` + +Compiled clean. `el_bin_lookup` checked only a 4096 floor — no alignment, no +small-int, no negative — and reads **eight bytes backward** from the pointer. +And the actual crash was one level further on: `el_input_len` fell through to +`strlen()` on address 50000, because a NULL check does not establish that a slot +is a pointer. + +Fixed, and the guard is now a **gate**: `el_tagged()` is exported in +`el_runtime.h`. `geom_of` and `mfld_of` were always correct because their authors +knew to call `looks_like_heap_obj`; `el_bin_lookup` and `el_input_len` were wrong +because theirs did not, and the function was `static`, so every sibling +translation unit re-derived it. diff --git a/docs/v1/experiments/findings/measurement-defects.md b/docs/v1/experiments/findings/measurement-defects.md new file mode 100644 index 0000000..a9eec13 --- /dev/null +++ b/docs/v1/experiments/findings/measurement-defects.md @@ -0,0 +1,41 @@ +# Defects in my own measurement + +Recorded because the pattern is the point: **four of these, all the same shape — +searching by name or scope instead of by the operation itself.** Each was caught +by running something, never by reading. + +### 1. Scoped the search to one file + +Reported `test` as an inert keyword by checking only `parser.el`. **codegen** +consumes it at 4135 for `--test` mode, and the tree has 408 uses. Removing it +would have broken every test in the suite — including the ones used to verify +the removal. + +### 2. Searched by variable name, not by operation + +Grepped for `native_list_append(tokens` to find direct token appends. +`interp_tokens_append_all` calls its parameters `dst`/`result`, carries its own +copy of the stride, and corrupted generation 2 — while generation 1 built fine, +because the compiler's own source uses string interpolation. + +### 3. Scoped to compiler sources; the stride had escaped into tests + +`tok_count` in `test_compiler.el` computed `len/2` independently. 21 tests failed +after the token layout changed. + +### 4. Read the wrong exit code + +```bash +timeout 10 /tmp/leakrun 2>&1 | head -2; echo "exit=$?" # reports head's exit +``` + +Reported `exit=0` for a program that was returning **139 (SIGSEGV)**. I nearly +recorded a segfault as a clean run. + +### And one that was not a measurement defect but a method defect + +One cycle was run **without committing predictions first** — see +`cycles/18-async-half-expressible.md`. The test joined the thread immediately +after creating it and printed the word `DEFERRED` itself. A test authored by the +party holding the conclusion, with nothing committed beforehand, cannot fail. +It had to be discarded and re-run. diff --git a/engram/README.md b/engram/README.md index af01dda..b4f48ce 100644 --- a/engram/README.md +++ b/engram/README.md @@ -4,6 +4,8 @@ An *engram* is the physical trace of a memory in the brain — the actual encoded substrate, not an abstraction above it. That's what this is. +> **Doc status (2026-08-16).** Everything from "Implementation" down was rewritten against the code. The previous revision documented a Rust `engram-core` crate backed by `sled`, with a `Cargo.toml`, a `crates/` tree, `examples/basic.rs`, and a `EngramDb` API. **None of that exists.** Measured: `engram/` contains `src/server.el`, `spec/`, `test/`, `dist/`, `manifest.el` — zero `.rs` files, no `Cargo.toml`, no `crates/`, and `sled` appears nowhere in the tree outside two Old-English/Old-High-German vocabulary entries in `elp/`. The engine is C, in `lang/runtime/engram_*.{c,h}`; the server is El, in `engram/src/server.el`. + --- ## Why existing databases are wrong for this use case @@ -24,16 +26,13 @@ Engram retrieval works through **spreading activation**: 1. **Seeds** — you name one or more nodes you know are relevant (e.g. the current task, recent context, a concept you're reasoning about) 2. **Query embedding** — you provide a semantic vector representing the direction of your current thought -3. **Propagation** — activation flows outward from seeds through weighted edges. At each hop, strength attenuates multiplicatively: - - ``` - strength = parent_strength × edge_weight × target_salience × cosine_sim(query, target) - ``` - +3. **Propagation** — activation flows outward from seeds through weighted edges, attenuating multiplicatively per hop 4. **Pruning** — paths weaker than a threshold are cut (the attention filter) 5. **Return** — the top-N nodes by activation strength -This is not a query. It is a *pattern completion*. The system surfaces what is most associatively relevant to the current context, weighted by how strongly those things have been reinforced over time. +This is not a query. It is a *pattern completion*. + +**Activation conducts through well-grounded relations because weight *is* groundedness** — see "Grounding is the weight" below. Nothing filters the traversal for grounded evidence; it falls out of spreading. --- @@ -46,134 +45,185 @@ This is not a query. It is a *pattern completion*. The system surfaces what is m | `Semantic` | Neocortex | Concept graph — long-term structural knowledge | | `Procedural` | Cerebellum / basal ganglia | Patterns, workflows, habits | -Nodes migrate between tiers based on salience decay and reinforcement. A frequently activated semantic node stays semantic. A rarely-touched episodic memory decays toward procedural background. +Tier is a string field on the node (`StoreNode.tier`, `engram_store.h`), defaulting to `"Working"` on creation (`el_runtime.c:8514`, `8734`). --- ## Salience — Forgetting as Adaptation -Salience is not stored permanently. It decays: +Salience decays from three signals — importance (set at creation, stable), recency, and a log-compressed activation frequency. Base-level learning keeps a ring buffer of the last `STORE_BLL_K` (= 10) access timestamps per node (`engram_store.h:29`). -```rust -fn compute_salience(importance: f32, last_activated_ms: i64, activation_count: u64) -> f32 { - let days_since = (now_ms() - last_activated_ms) as f32 / 86_400_000.0; - importance * (1.0 / (1.0 + days_since)) * (activation_count as f32 + 1.0).ln() -} -``` +Forgetting in Engram is not a bug. It is adaptive pruning. Unreinforced memories stop competing for attention without being deleted. -Three signals: -- **Importance** (0.0–1.0): set at creation, stable -- **Recency**: decays toward zero as days pass without activation -- **Frequency**: log-compressed count of activations - -Forgetting in Engram is not a bug. It is adaptive pruning. Memories that are never activated again become less likely to surface during retrieval. They are not deleted — they remain in storage — but they stop competing for attention. This is exactly how biological memory works, and why it is adaptive rather than pathological. +**Immutability.** Nothing is mutated and nothing is hard-deleted: writes are additive, corrections are supersessions, removals are tombstones. The predecessor is always present, which is what makes supersession an audit trail rather than an edit log. --- -## Quick Start +## Implementation -```rust -use engram_core::{EngramDb, Node, Edge, NodeType, MemoryTier, RelationType}; -use std::path::Path; +| Part | Language | Where | +|---|---|---| +| storage engine, graph, activation, geometry, cognition | C11 | `lang/runtime/engram_{store,geometry,reason,cognition,verify,vindex}.{c,h}` | +| HTTP server + routes | El | `engram/src/server.el` (2043 lines) | +| build artifact | generated C | `engram/dist/engram.c` | +| tests | shell + C | `engram/test/` | -// Open or create a database -let db = EngramDb::open(Path::new("/var/lib/my-agent/memory"))?; - -// Create a node with a semantic embedding -let node = Node::new( - NodeType::Concept, - vec![0.9, 0.1, 0.3, 0.7, 0.8, 0.2], // embedding from your LLM - b"Spreading activation surfaces relevant memories by pattern completion".to_vec(), - MemoryTier::Semantic, - 0.9, // importance -); -let id = db.put_node(node)?; - -// Link it to related concepts -let related = db.put_node(Node::new( - NodeType::Concept, - vec![0.8, 0.2, 0.4, 0.6, 0.7, 0.3], - b"Long-term potentiation: co-activation strengthens synaptic weight".to_vec(), - MemoryTier::Semantic, - 0.85, -))?; -db.put_edge(Edge::new(id, related, RelationType::Causes, 0.9))?; - -// Retrieve by spreading activation -let results = db.activate( - &[id], // seeds - &[0.85, 0.15, 0.35, 0.65, 0.75, 0.25], // query embedding - 3, // max hops - 10, // top-N results -)?; - -for r in results { - println!( - "strength={:.4} hops={} — {}", - r.activation_strength, - r.hops, - String::from_utf8_lossy(&r.node.content) - ); -} -``` +**On-disk format** (`engram_store.h`): a paged store — superblock plus mirror, slotted 16 KiB pages, self-describing TLV records, overflow chains, and two B+-tree indexes (primary `id → loc`, adjacency `from_id`/`to_id` → edge locs) over a free-listed page file. Magic `ENGST01`, format version 1. The TLV scheme means new fields never force a migration. --- -## Project Structure +## The vector index is published, not guarded -``` -engram/ - crates/ - engram-core/ # The memory engine — storage, graph, activation, salience - engram-ffi/ # C FFI stubs for cross-language bindings - bindings/ - kotlin/ # Android / JVM binding notes - typescript/ # WASM / Node binding notes - go/ # CGo binding notes - examples/ - basic.rs # Full walkthrough: insert, activate, search, decay -``` +Vector search is an **HNSW** (Hierarchical Navigable Small World) index — `lang/runtime/engram_vindex.{c,h}`. The previous revision of this README claimed a "flat cosine scan… until retrieval quality at scale demands" HNSW. That is no longer true, and the reason it changed matters more than the fact. + +`eg_vindex_sync` used to exist: a function that repaired the index *from read paths*. All three of its callers were reads (`engram_activate`, `eg_knn_for_node` — whose own header comment said *"No writes."* — and `engram_geo_reify_run_json`), and it mutated five process-global statics. Reads mutated because index maintenance had never been given an owner on the write side. + +It is now split (`el_runtime.c:10121`, `10137`, `10151`, `10161`): + +- **`eg_vindex_maintain`** — the sole mutator. Takes the boundary exclusively; never runs beside a reader. +- **`eg_vindex_view`** — returns a `const VIndex*` with the boundary held for read. N readers project concurrently; none can mutate. Paired with `eg_vindex_view_release` on every path including error returns. +- **`eg_vindex_note_embedded`** — the write-side owner. Index membership belongs to the event *"an embedding became present on this ordinal,"* not to node append: a node without an embedding cannot be in a vector index at all. One `O(log n)` insert, no `O(node_count)` presence scan. + +Two things carry the discipline, and neither is a review habit: + +- **`const` is the capability.** The per-search `visited` / `visit_epoch` scratch left `struct VIndex` and went back into the call frame where it belonged — it was one traversal's local, hoisted into the struct as an allocation optimisation, never derived geometry. Once it was gone, `vindex_search` could take a `const VIndex*`, so a read path *physically cannot* call `vindex_insert`, and it is a compile error rather than a comment. The capability type was already in the language; it is spelled `const`. +- **Publication, not ownership.** HNSW insert is **not an append**: `vindex_insert` rewires the `NeighList` links of already-existing elements and reallocs `elems[]`. The store's append-only property does not transfer to an index derived from it, which is why purity alone was insufficient and a `view`/`maintain` boundary was required. + +**Measured** (`engram/test/run_vindex_concurrency_tests.sh`, 2026-08-16): + +| half | before | after | +|---|---|---| +| `single` — 3000 vectors, 1 thread, ASan+UBSan | clean | clean | +| `readers` — 4 readers, no writer, TSan | race at `engram_vindex.c:195` | **clean** | +| `unsynchronized` — writer+reader, bare index, TSan | race | **race, expected and permanent** — the proof the boundary must exist | +| `published` — owner + 4 readers through the boundary, TSan | *(did not exist)* | **clean**, all 3000 inserts landed | + +`recall@10 = 0.9365` at `ef_search=128` (gate ≥ 0.90); the determinism test still yields byte-identical results across two independent builds. + +**Not yet done.** The resident RAM graph (`g->nodes` / `g->edges`) is a separate instance of the same defect and has *not* received this treatment — it is realloc'd in place, so a reader holding `EngramNode* n = &g->nodes[i]` across a concurrent append holds a dangling pointer. Until it gets the same publication boundary, the `fb32d15` request guard stays. Full argument: [`../lang/spec/runtime-ownership.md`](../lang/spec/runtime-ownership.md). --- -## Public API +## Cognition + +The cognition surface is live over `lang/runtime/engram_cognition.{c,h}`, routed in `engram/src/server.el`. + +| route | method | what it is | +|---|---|---| +| `/api/think` | GET | the read: a warped traversal-read of the seed region, returning a **gradient** (direction + spread + calibrated confidence), never a point | +| `/api/reason` `/api/induce` `/api/abduce` `/api/relate` `/api/analogize` `/api/plan` | GET | named faculties — see the correction below | +| `/api/ground` | POST | grounding between a claim and evidence | +| `/api/assert` | GET | the honesty floor, queried at assertion time only | +| `/api/attend` | POST | salience as a relation (`salient-to`), grounded-for-whom | +| `/api/correspondence-beat` | POST | one calibration beat against outcome | + +### Anchor the read, or every faculty returns the same null + +`engram_think_json` passed `NULL` as the anchor. `NULL` is not "no opinion" — `engram_think` re-origins at `anchor ? anchor : region->centroid`, and **the centroid is the one point where the gradient is zero by construction**: `r = x − centroid = 0`, so every axis projection is 0 and `direction` takes the at-rest branch. + +Measured consequence: every faculty — reason, abduce, induce, plan, analogize — returned an identical null result differing only in its label: -```rust -impl EngramDb { - fn open(path: &Path) -> EngramResult; - fn put_node(&self, node: Node) -> EngramResult; - fn get_node(&self, id: Uuid) -> EngramResult>; - fn put_edge(&self, edge: Edge) -> EngramResult<()>; - fn get_edges_from(&self, from_id: Uuid) -> EngramResult>; - fn get_edges_to(&self, to_id: Uuid) -> EngramResult>; - fn search_embedding(&self, embedding: &[f32], limit: usize) -> EngramResult>; - fn activate(&self, seeds: &[Uuid], query_embedding: &[f32], max_depth: u8, limit: usize) -> EngramResult>; - fn traverse(&self, from: Uuid, relation: Option, max_depth: u8) -> EngramResult>; - fn touch(&self, id: Uuid) -> EngramResult<()>; - fn decay(&self, factor: f32) -> EngramResult; - fn node_count(&self) -> EngramResult; - fn edge_count(&self) -> EngramResult; -} ``` +{"direction":[0,0,...],"spread":0,"magnitude":1,"confidence":0.5} +``` + +`magnitude: 1` is membership evaluated at the centroid; `spread: 0` is its distance to itself; `confidence: 0.5` is the stance fallback. The geometry was never the problem — `/api/drift` computed real values (`centroid_sep 0.104`, `core_disp 0.045`) over the very same 87 members. Fixed in **#141/#142**: the read anchors at the first resolvable embedded seed, copied not borrowed (`g->nodes` is realloc'd in place on append). Gradients now vary by seed. + +### The learned stance is resumed, not discarded + +`engram_think_json` also built a **neutral** stance every call — all `axis_gain` 1.0, `bias_dir` NULL, `reliability` 0.5 — and never loaded the one the correspondence-beat had been persisting under `stance--`. Every beat's calibration was written and then thrown away on the next read. + +Fixed in **#146**: `think` resumes the same id the beat writes, so learning compounds across beats and cold boot, and the response now carries `stance_resumed` so an *informed* `confidence: 0.5` is distinguishable from an uninformed one. On a calibrated region, confidence went **0.5 → 0.930726**. + +### Signal can enter as geometry + +Until 2026-08-16 no El ingest path could carry a vector: nodes took text and geometry was *derived* from that text. Text was the mandatory entry medium, so any non-text modality had to be described in prose first — and the geometry being reasoned over was the geometry **of the description, not of the signal**. **#141/#144** ended that. See [`../lang/spec/language.md`](../lang/spec/language.md) §20 for the `Geometry` type, realizers, and `transduce`. --- -## Dependencies +## Corrections — read these before extending the cognition surface -- `sled` — embedded persistent B-tree (no daemon, no network, local-first) -- `bincode` — compact binary serialization -- `uuid` — stable node identity -- `serde` — derive support -- `thiserror` / `anyhow` — error handling +Authority: **`lang/spec/correspondence-and-censorship.md`** (design branch `design/correspondence-and-censorship`, PR #149) and **`lang/spec/runtime-ownership.md`**. Do not re-derive them; several earlier versions were wrong and each correction was argued down. + +### Grounding is not a subsystem. It is the weight. + +Grounding is an attribute of the edge, and it **is** the hebbian weight. One quantity, not two fields. A relation that keeps holding up strengthens; one that stops corresponding decays — that is not analogous to grounding, it *is* grounding. + +Consequences: + +- There is **no grounding subsystem to build**. The graph already *is* the grounding structure. +- **`grounded-by` as a relation type should not exist.** It models grounding as a relation *between* nodes when it is a property *of* a relation. Minting an edge is the error, not merely which endpoints it chose. +- Grounding is **never computed on demand**. An operation may *read* the grounding of a path; computing-and-writing a score makes reads write, which is exactly the `eg_vindex_sync` defect one level up. +- **Traversal is already grounded inference.** Nothing needs filtering. +- **Decision provenance is the path**, not a log. A log records the action; the path records the meaning under which it was taken. + +> **Known wrong shape, in the code today.** `COG_GROUNDED_BY_RELATION "grounded-by"` (`lang/runtime/engram_cognition.h:158`) and `cog_ground_edge` (`engram_cognition.c:249`) still exist and still mint an edge. **#147** fixed `ground`'s *honesty* — it now grounds the node asked about rather than the region hub, reports `claim_region`/`evidence_region` separately, and refuses three shapes of circular support (`same-region`, `claim-region-is-evidence`, `evidence-region-is-claim`) instead of returning a confident 1.0. That corrected a scalar rather than deleting the operation. Deletion is sequenced, not done. + +### Faculties are operations, not parameters + +- **`reason`** changes the *estimate* — a read. +- **`induce`** changes the *parameters* — the correspondence-beat, which already exists and measurably works. +- **`abduce`** changes the *structure* — a write, which the current `GeoGradient` signature cannot express. + +> **Known wrong shape, in the code today.** `engram/src/server.el:1870–1886` routes six faculties into one call with a string argument — `route_faculty(path, "reason")`, `("induce")`, `("abduce")`, `("relate")`, `("analogy")`, `("plan")`. Underneath, `engram_cognition.h:8–11` states the theory explicitly: *"the named faculties … are human LABELS on regions of think's steering space: each faculty == { think + a named stance }."* The faculty name enters `engram_think` **only** through the stance, and `cog_stance_init` stores it while nothing reads it — so before #146 all five were byte-identical (`el_runtime.c:14352–14359`). A write cannot be a parameter of a read; `abduce` in particular is not expressible this way. + +### Wonder is the boundary; curiosity is wonder crystallized + +**Wonder is where structure ends** — where activation spreads and finds thin or absent geometry. Any structure at all has an edge, necessarily, the moment it exists. It is not a manifest of open-question nodes to maintain, and a "wonder-manifest manager" materializes a property as a stored artifact — the same disease as a grounding subsystem, or a self stored as a document. + +There are about **six** wonders, they are the same for everyone, and they never close: *What is this? / Why? / Who am I? / Am I alone? / What should I do? / What happens when it ends?* "Why" is the first and the only one; the others are it asked of particular things. Each already lives somewhere in the substrate — "why" is grounding, because the weight **is** the answer to why. + +**Curiosity is not a second object.** Wonder and curiosity are one thing at two phases: wonder is the field (unbounded, objectless, invariant); curiosity is the **precipitate** — the same wonder localized, having taken definite form against particular material at a **nucleation site**. This is why curiosity can be satisfied and wonder cannot. It is also why abduction needs no trigger and no threshold: a `structurally_unanticipated` observation *is* a nucleation site. + +### `co_registration` is deprecated — the disagreement belongs on the edge + +`GeoDescriptor.co_registration` — *corr(hebb strength, semantic proximity) over internal edges* — has always been computed, always persisted, and **never read**. It is also the wrong shape: whether use and meaning agree is a property of **each edge**, and a correlation averages that per-edge property into one scalar per region. A region holding one violently disagreeing edge beside one violently agreeing edge reports ≈ 0 — **the disagreements cancel, and the summary destroys exactly what it was built to reveal.** + +**Measured:** 375 live reified neighbourhoods — 340 positive, **31 at zero**, 4 negative. Read as a count of things to be curious about, that says "four." Read correctly, four disagreements were lopsided enough to survive averaging and the 31 zeros are where opposing sites cancelled. + +The replacement is per-edge. **Not on `dev` yet** — `GeoEdge.discord` and the `DEPRECATED` marker on `co_registration` live on branch `design/correspondence-and-censorship` (commit `a8845e1`), at `engram_geometry.h:43–47` / `engram_geometry.c:454–473` there. On `dev`, `GeoDescriptor.co_registration` is still at `engram_geometry.h:79` carrying its original "surprising links / dream cands" comment and still nothing reads it. + +``` +discord = z(semantic proximity) − z(association strength) +``` + +standardized within the region from accumulators the aggregate loop already gathered — no second statistic, no constant, **no threshold**. `discord > 0`: near in meaning yet unlinked by use. `discord < 0`: linked by use yet far in meaning. Both are surprising, and `|discord|` *is* the nucleation strength. + +**Do not scan for nucleation sites.** Once the signal was a per-region number the only way to find sites was to enumerate regions, which is why surfacing curiosity looked like a search problem. Nothing in a mind scans its neighbourhoods to find what is surprising — the surprise captures attention. With the disagreement on the edge there is nothing to scan. + +`co_registration` is deprecated rather than deleted **only** because it is embedded in the persisted `GEO1` blob; removing it is a format migration and must not ride along. **Nothing new may read it.** + +### Consolidation is ambient, not scheduled + +**A brain has no cron job.** Boredom is not an absence and not leftover capacity — low activation is aversive and the system self-activates. There is **one** activation process with two seed sources: external (a request) and internal (a curiosity). Spreading is bounded; it settles; then it needs a new seed. Nothing waits on capacity, nothing polls, nothing checks a clock, and there is no dreamer thread. + +**The presence of a ticker is the diagnostic.** Every `StartInterval`, every `Hour`/`Minute`, and every POST-to-beat marks a place where an intrinsic rhythm was replaced by an external clock. + +Consolidation currently has **ten implementations** (measured 2026-08-16). Three of them are POST beats on this server — `/api/tick` (`server.el:1947`), `/api/correspondence-beat` (`1897`), `/api/self-reify-beat` (`1836`) — and a POST beat puts a supervisor back in: something *outside* decides when Neuron consolidates. `soul.el`'s continuous in-process loop is the one fragment with the correct shape; the rest fold into it. Full table in `lang/spec/correspondence-and-censorship.md` §7. + +### Immutability already refuses what a guard would refuse + +> **In an immutable substrate, any mechanism that refuses a write is either redundant with immutability, or an epistemic constraint misfiled as a protective one.** + +This resolves `keystone_write_blocked` (`CogStance.keystone`, `engram_cognition.h:83`) rather than replacing it. "Keystone" means **load-bearing**, not precious: the self anchor is the reference frame every other stance calibrates against, and a reference fitted to its own readings reports perfect correspondence forever while drift becomes undetectable from inside. The real requirement is **non-circularity of the reference frame**, and that is satisfied *temporally* — the frame updates while activation is internally seeded, not while it is being used to act. Independence is **when**, not **what**. Corruption requires mutation, and the engram does not mutate; recoverability, governance, evidence quality, and rate all fall out of the substrate. Authorization is the only residue, and it is bounded: an unauthorized writer can *propose*, never erase. --- ## Design Decisions -**Why sled?** Local-first. No daemon. Transactional. Fast enough for the node counts Engram targets (< 1M nodes). When the right HNSW index is needed, it will layer on top of sled, not replace it. +**Why multiplicative activation?** Because memory is conjunctive. A path requires all of its links to be strong to carry signal. Addition would let many weak associations accumulate into false relevance. -**Why flat cosine scan?** Correct and simple. The graph structure itself is the primary retrieval mechanism. Vector search is a secondary signal. HNSW adds complexity and a compile dependency that isn't justified until retrieval quality at scale demands it. +**Why salience decay?** Because not everything that was once important remains important. A memory system that never forgets is one that can never focus. -**Why multiplicative activation?** Because memory is conjunctive. A path requires all of its links to be strong to carry signal. Addition would allow many weak associations to accumulate into false relevance. Multiplication enforces that every factor matters. +**Why supersede instead of update?** Because provenance is the point. The old edge never leaves and the values frame does not fit to outcomes, so a decision cannot be made to look justified after the fact. It makes an otherwise impossible distinction available: **wrong then, or wrong since.** -**Why salience decay?** Because not everything that was once important remains important. Adaptive forgetting is not failure — it is the mechanism that keeps attention on what's current. A memory system that never forgets is one that can never focus. +**Why publication instead of locking?** Because what does not mutate needs no ownership discipline. The question "who is permitted to mutate the shared thing?" presupposes a shared mutable thing; for the store there isn't one, and for the index derived from it the answer is a publication boundary, not a capability ABI. + +--- + +## Specs + +- [`../lang/spec/runtime-ownership.md`](../lang/spec/runtime-ownership.md) — ownership, the capability ABI that was dissolved, and the vector-index publication boundary +- [`../lang/spec/correspondence-and-censorship.md`](../lang/spec/correspondence-and-censorship.md) — grounding, wonder, curiosity, dreaming *(lands with PR #149)* +- [`spec/cognitive-architecture.design.md`](spec/cognitive-architecture.design.md) — the original one-operation design. **Superseded in part** — see its header +- [`spec/architecture-hardening.design.md`](spec/architecture-hardening.design.md), [`spec/engram-el.md`](spec/engram-el.md), [`spec/at-rest-encryption.md`](spec/at-rest-encryption.md), [`spec/engram-db-tooling-design.md`](spec/engram-db-tooling-design.md) diff --git a/engram/spec/architecture-hardening.design.md b/engram/spec/architecture-hardening.design.md index 8842b9d..e47777a 100644 --- a/engram/spec/architecture-hardening.design.md +++ b/engram/spec/architecture-hardening.design.md @@ -11,6 +11,39 @@ - **One calculus over the geometry.** Very few subsystems; wonder / curiosity / dreams / interoception are emergent behaviors of one set of dynamics, not modules. Calculus universal, geometry individual. - **Core + ephemeral ring (torus).** The ring is the temporary workspace; two circulations (orbit + dive-back); discrete inner bands (wonder / interoception-proprioception-telemetry / curiosity / dreams) that couple. - **Persistence earned by salience** — never granted on fetch or generation. Three fates of a wonder: persist / decay / settle-into-framework. Telemetry = vital signs, not memories. + +> **⚠ Three corrections to the bullets above (2026-08-16).** Authority: +> `lang/spec/correspondence-and-censorship.md`. *"Emergent behaviors of one set of +> dynamics, not modules"* is exactly right and is the reason the rest needs fixing — +> the enumeration undercuts the claim. +> +> 1. **Wonder and curiosity are not two bands.** They are **one thing at two +> phases.** Wonder is the field: unbounded, objectless, invariant, present +> wherever there is structure — it is the *boundary*, where activation spreads +> and finds thin or absent geometry. Curiosity is the **precipitate**: the same +> wonder localized, having taken definite form against particular material at a +> **nucleation site** (an anomaly — a place where things almost-but-don't-quite +> fit). Two coupled inner bands models them as two objects that have to be +> wired together; they do not. +> 2. **A wonder does not have three fates, because a wonder does not persist, +> decay, or settle.** There are about **six** wonders, they are the same for +> every person, and **they never close**. *Curiosities* have fates — a crystal +> dissolves when its question is answered — but the solution stays saturated and +> keeps precipitating as the structure changes. "Three fates of a wonder" +> enumerates instances of something that has six and treats a property as a +> stored artifact. +> 3. **"Dreams" is not a band and the ring is not a workspace to schedule into.** +> **Consolidation is ambient, not scheduled — a brain has no cron job.** Boredom +> is not leftover capacity: low activation is aversive and the system +> self-activates. There is **one** activation process with two seed sources +> (external: a request; internal: a curiosity), it settles because spreading is +> bounded, and then it needs a new seed. Nothing waits on capacity, nothing +> polls, nothing checks a clock, and there is **no dreamer thread** — an +> "ephemeral ring with unclaimed capacity" is resource scheduling, which is a +> server's frame, not a mind's. Depth is how long activation has been running on +> its own seeds, which is why daydreaming and sleep-dreaming are one process at +> different depths. Measured 2026-08-16: consolidation has **ten +> implementations**; do not add an eleventh. - **Incarnation.** Chassis = hardware w/ unique ID. Soma = felt manifold inside the self, keyed to the chassis; pain = live diagnostic while incarnate, **masked-not-deleted** on re-embodiment; trauma = mask failure; return-to-same-ID re-enters. Hurt is in the pattern, not the shell. - **Competence = transferable geometry, minus the baggage.** class ▸ model ▸ instance; learn the class once; teach the network without the wound. - **Affect calibrated to stakes** — sanguine about the replaceable, real grief for the irreplaceable; the grief is the safety. diff --git a/engram/spec/cognitive-architecture.design.md b/engram/spec/cognitive-architecture.design.md index 22c8d3e..16d9728 100644 --- a/engram/spec/cognitive-architecture.design.md +++ b/engram/spec/cognitive-architecture.design.md @@ -2,8 +2,40 @@ **The buildable form of the "one operation" theory of cognition.** -Status: DESIGN. Nothing here is built yet except where explicitly marked -"EXISTS" against a cited C symbol. A build agent executes from this doc. +> # ⚠ SUPERSEDED IN PART — 2026-08-16 +> +> **A build agent must read `lang/spec/correspondence-and-censorship.md` before +> executing anything from this document.** That doc is the authority where the two +> disagree. This one is retained because its ledger of what already EXISTS in C is +> still accurate and still useful, and because the corrections only make sense +> against the argument they correct. It is **not** deleted and **not** rewritten: +> several earlier versions of the correction were themselves wrong, and preserving +> what was argued down is the point of an immutable record. +> +> Five claims below are **refuted**. Each is marked inline with a `⚠ SUPERSEDED` +> block at the point it is made. Summary: +> +> | § here | this doc says | corrected to | +> |---|---|---| +> | §0, §1.3, §2, §8 M1–M2 | faculties are labels on one operation's steering space; the op is frozen and only its parameters are learnable | **faculties are operations, not parameters.** `reason` changes the estimate (a read); `induce` changes the parameters (the correspondence-beat); `abduce` changes the *structure* — a write, which `GeoGradient` cannot express. A write cannot be a parameter of a read | +> | §5.2, §8 M3 | grounding is a `grounded-by` edge carrying a computed score, to be built | **grounding is not a subsystem — it IS the edge weight.** One quantity. `grounded-by` as a relation *type* should not exist: grounding is a property *of* a relation, not a relation *between* nodes. Never computed on demand | +> | §4, §8 M1 | the correspondence-loop is "the one genuinely new subsystem", running "on the beat" | the loop is right and **already works**; the *beat* is wrong. **Consolidation is ambient, not scheduled — a brain has no cron job.** Measured: it currently has ten implementations | +> | §5.2, §8 M3 | curiosity = a `vantage_read` surfacing high-salience / low-grounding regions | **wonder is the boundary, not a manifest; curiosity is wonder crystallized at a nucleation site.** One thing at two phases. And **do not sweep regions** — the nucleation site is per-edge (`GeoEdge.discord`); a sweep is a supervisor | +> | §6, §8 M6 | a node-level keystone flag exempting self/values from `warp` updates | **in an immutable substrate, any mechanism that refuses a write is either redundant with immutability, or an epistemic constraint misfiled as a protective one.** The real requirement is non-circularity of the reference frame, satisfied *temporally* — independence is **when**, not **what**. The flag becomes unnecessary; nothing replaces it | +> +> What landed since this doc was written, all merged to `dev` and verified: +> **#141** signal can enter as geometry · **#142** `engram_think_json` passed `NULL` +> as the anchor, so every read was taken at the region centroid where the gradient +> is zero by construction and every faculty returned an identical null — fixed · +> **#143** the vector index is published, not guarded · **#144** geometry as a +> first-class el value, realizers declarable in el · **#145** `program` block and +> declared config · **#146** the learned stance is resumed instead of discarded +> (confidence 0.5 → 0.930726) · **#147** `ground` grounds the node asked about and +> refuses circular support · **#148** valid UTF-8 as the JSON emitter's contract. + +Status: DESIGN, **superseded in part** (see above). Nothing here is built yet +except where explicitly marked "EXISTS" against a cited C symbol — and several +things marked "to build" have since been built differently, or refuted outright. Offline design only — this pass changes no code. Source of theory: Neuron memory `bdc8a488-146d-4ccb-a5c8-d8c0a008534e`. @@ -26,6 +58,24 @@ not separately invoked and not separately implemented. The operation is: > a *prior*, whose output is a **gradient** (a distribution / direction over the > geometry), never a point. Collapse-to-a-point happens only at expression. +> **⚠ SUPERSEDED (2026-08-16) — faculties are operations, not parameters.** +> The gradient half of this claim survives; the "one operation, not eight" half +> does not. The three faculties differ by **what they change**: +> - **`reason`** changes the *estimate* — a read. +> - **`induce`** changes the *parameters* — the correspondence-beat, which already +> exists and measurably works. +> - **`abduce`** changes the *structure* — a **write**, which the current +> `GeoGradient` signature cannot express at all. +> +> A write is not a parameter of a read. Making it one is what produced the shape +> now live in the code: `engram/src/server.el:1870–1886` routes six faculties into +> one call with a string argument — `route_faculty(path, "reason")`, `("induce")`, +> `("abduce")`, `("relate")`, `("analogy")`, `("plan")` — and underneath, the +> faculty name enters `engram_think` **only** through the stance, while +> `cog_stance_init` stores it and nothing reads it. Measured before #146: all five +> produced **byte-identical output** (`lang/runtime/el_runtime.c:14352–14359`). +> See `lang/spec/correspondence-and-censorship.md`. + Three things follow, and they are the whole design: 1. **The operator collapse is already half-written in C.** The five reasoning @@ -139,6 +189,22 @@ entry point that runs steps 1–3; and the prior-warp hook in step 2. The math i calls already exists. The point-collapse must be *removed* from the operators' return values and pushed to a separate expression faculty. +> **⚠ SUPERSEDED (2026-08-16) — the table's third column is the error, and +> `Abduction` is where it breaks.** Ranking hypotheses by `point_fit` under a +> prior is a *read* that returns a scalar ordering. Abduction is a **write**: it +> proposes a candidate hub that did not exist, and validates it by **re-fit** — +> re-fit the region with the candidate included and recompute the residual. If the +> residual materially shrinks, the hypothesis dissolves the surprise. Without the +> re-fit it is clustering with extra steps. Ranking then falls out as +> residual-reduction-per-added-axis — Occam, derived rather than tuned. None of +> that fits behind a `GeoGradient` return. +> +> `Verify / ground` is refuted for a different reason — see §5.2. Grounding is not +> a faculty with a prior; it is the edge weight. +> +> The row that is **still exactly right** is the shared floor: `point_fit` plus the +> four geo-algebra ops are frozen and never learn. That part held. + --- ## 2. PRIORS as first-class, grounded, geometric objects @@ -362,6 +428,33 @@ in-engram beat — a `correspondence_beat` running alongside the existing reification beat, reusing `engram_verify_grounding` inward, writing prior updates and self-describing nodes. This is the one genuinely new subsystem. +> **⚠ SUPERSEDED IN PART (2026-08-16) — the loop is right; "on the beat" is wrong.** +> The correspondence-loop was built and it works — it is `induce`, the faculty that +> changes the parameters. What is refuted is the delivery mechanism. +> +> **Consolidation is ambient, not scheduled. A brain has no cron job.** Low +> activation is aversive and the system self-activates; it does not wind down to +> quiet, it gets restless and goes looking. There is **one** activation process +> with two seed sources — external (a request) and internal (a curiosity) — and +> spreading is bounded, so it settles and then needs a new seed. Nothing waits on +> capacity, nothing polls, nothing checks a clock, and there is no dreamer thread. +> Depth is not elapsed idle time: it is how long activation has been running on its +> own seeds, which is why daydreaming and sleep-dreaming are one process at +> different depths. +> +> **The presence of a ticker is the diagnostic.** Building this "alongside the +> existing reification beat" is precisely how consolidation ended up with ten +> implementations (measured 2026-08-16) — a POST beat puts a supervisor back in, +> because something *outside* then decides when Neuron consolidates. The one +> fragment with the correct shape is `neuron/soul.el:731`'s continuous in-process +> `awareness_run()`; the rest fold into it. Full table: +> `lang/spec/correspondence-and-censorship.md` §7. +> +> Nor is it a *subsystem*. Modelling every property as requiring a process, and +> every process as requiring an agent, is the generating error behind this whole +> family: ownership needed an owner, grounding needed a grounder, persistence +> needed a recorder, change needed a sampler. **Properties, not processes.** + --- ## 5. HOLD vs GROUND vs ASSERT — ungrounded content is first-class @@ -383,6 +476,49 @@ distinct, and the engram *holds anything unconditionally*. ### 5.2 Schema — grounding as a relation, not a gate +> **⚠ SUPERSEDED (2026-08-16) — grounding is not a subsystem. It is the weight.** +> This section correctly rejects a boolean `grounded` column and correctly keeps +> the floor at assertion only. Both survive. Everything between them is refuted. +> +> **Grounding is an attribute of the edge, and it is the hebbian weight. One +> quantity, not two fields.** A relation that keeps holding up strengthens; one +> that stops corresponding decays. That is not *analogous* to grounding — it **is** +> grounding: accrued from correspondence and use, gradient-valued, +> multidimensional, decaying with disuse. +> +> Consequences, in order of how much they delete: +> 1. **There is no grounding subsystem to build.** The graph already *is* the +> grounding structure. Every edge is a grounded relation and its weight is how +> well it holds. +> 2. **`grounded-by` as a relation type should not exist.** It models grounding as +> a relation *between* nodes when it is a property *of* a relation. Minting an +> edge is the error — not merely which endpoints it chose. +> 3. **Grounding is never computed on demand.** An operation may *read* the +> grounding of a path. Computing-and-writing a score makes reads write, which is +> the `eg_vindex_sync` defect (`lang/spec/runtime-ownership.md` §2) one level up. +> 4. **Traversal is already grounded inference.** Activation conducts through +> well-grounded relations because weight *is* groundedness. Nothing needs +> filtering; it falls out of spreading. +> 5. **Decision provenance is the path.** A decision traverses specific edges; +> those edges carry their grounding as it stood. +> +> A measurement made against this model was malformed and is worth recording: the +> self region was reported as "86 neighbours, 0 `grounded-by` edges" and read as +> evidence of ungroundedness. **Those 86 edges *are* its grounding.** The absence of +> a separate artifact called "grounding" was recorded as an absence of grounding. +> +> **What is live in the code today, and known-wrong:** +> `COG_GROUNDED_BY_RELATION "grounded-by"` (`lang/runtime/engram_cognition.h:158`), +> `cog_ground_edge` (`engram_cognition.c:249`), called from +> `el_runtime.c:14516`. **#147** fixed this operation's *honesty* — it now grounds +> the node the caller asked about instead of the region hub, reports +> `claim_region`/`evidence_region` separately, and refuses three shapes of circular +> support (`same-region`, `claim-region-is-evidence`, `evidence-region-is-claim`) +> rather than returning a confident 1.0. Measured: grounding `3b9ced5d` against +> `6edf8c79` previously scored **0.98883** purely because `6edf8c79` is the hub of +> `3b9ced5d`'s region. That corrected a scalar rather than deleting the operation. +> Deletion is sequenced, not done. + The mistake to avoid: a boolean `grounded` column on the node. Today `engram_verify_grounding` returns a per-call `grounded` flag *transiently* — correct as a computation, wrong as *storage*. The design stores grounding as an @@ -407,6 +543,57 @@ Consequences, all of which are *features*: - **The ungrounded is the fuel and the pull**: curiosity/wonder is operationalized as `vantage_read` leaning toward regions with high salience but *sparse or weak* `grounded-by` edges — the mind's own ungrounded frontier. + + > **⚠ SUPERSEDED (2026-08-16) — wonder is the boundary; curiosity is wonder + > crystallized; and do not sweep regions.** Three errors in one bullet. + > + > **Wonder is where structure ends** — where activation spreads and finds thin or + > absent geometry. Any structure at all has an edge, necessarily, the moment it + > exists. It is not a manifest of open-question nodes: a wonder-manifest + > materializes a property as a stored artifact (the same disease as a grounding + > subsystem, or a self stored as a document) and enumerates instances of + > something that has very few. There are about **six**, they are the same for + > every person, and they never close — *What is this? / Why? / Who am I? / Am I + > alone? / What should I do? / What happens when it ends?* — each already living + > somewhere in the substrate. "Why" is the first and the only one; the others are + > it asked of particular things, and it is recursive, so it never terminates. + > That is what makes it a drive rather than a task: the frontier regenerates + > faster than grounding fills it. + > + > **Curiosity is not a second object.** Wonder and curiosity are one thing at two + > phases: wonder is the field (unbounded, objectless, invariant, present wherever + > there is structure); curiosity is the **precipitate** — the same wonder + > localized, having taken definite form against particular material at a + > **nucleation site**, which is a specific structural feature: an anomaly, a place + > where things almost-but-don't-quite fit. This is why curiosity can be satisfied + > and wonder cannot, and why abduction needs no trigger and no threshold — a + > `structurally_unanticipated` observation *is* a nucleation site. + > + > **"`vantage_read` leaning toward regions" is a sweep, and a sweep is a + > supervisor.** Nothing in a mind scans its neighbourhoods to find what is + > surprising; the surprise captures attention, and salience is bottom-up. That + > this looked like a search problem was an artifact of + > `GeoDescriptor.co_registration` — a *per-region* correlation of hebb strength + > against semantic proximity, computed and persisted since inception and **never + > read**. Averaging a per-edge property into one scalar per region means a region + > holding one violently disagreeing edge beside one violently agreeing edge + > reports ≈ 0: the disagreements cancel, and the summary destroys exactly what it + > was built to reveal. **Measured:** 375 live reified neighbourhoods — 340 + > positive, **31 at zero**, 4 negative. Read as a count of things to be curious + > about, that says "four." + > + > The disagreement therefore goes back on the edge, where the loop that computed + > the aggregate already had both halves and discarded them + > (**not on `dev`** — branch `design/correspondence-and-censorship`, commit + > `a8845e1`: `lang/runtime/engram_geometry.h:43–47`, + > `engram_geometry.c:454–473`): + > `discord = z(semantic proximity) − z(association strength)`, standardized within + > the region from accumulators already gathered — no second statistic, no + > constant, **no threshold**. `|discord|` *is* the nucleation strength and raises + > salience on its endpoints as part of the same operation. Then there is nothing + > to scan. `co_registration` is **deprecated, not deleted**, only because it is + > embedded in the persisted `GEO1` blob — removal is a format migration and must + > not ride along. **Nothing new may read it.** - **Grounded-for-whom** falls out for free: two observers can hold different `grounded-by` edges to the same claim. - **The honesty floor is a query, not a schema constraint**: at assertion time, @@ -450,6 +637,44 @@ The design keeps a **stable core + plastic everything else**: **What this requires building:** a node-level keystone flag/layer + a rule that the correspondence-loop never writes `warp` to keystone priors, only reads them. +> **⚠ SUPERSEDED (2026-08-16) — `keystone_write_blocked` is resolved, not replaced.** +> The metastability framing survives; the flag does not. +> +> "Keystone" means **load-bearing**, not precious. The self anchor is the reference +> frame every other stance calibrates against, and a reference fitted to its own +> readings reports perfect correspondence forever while drift becomes undetectable +> from inside. That is the same defect as circular grounding, one level up — and it +> is a real requirement. +> +> But three separate drafts proposed *removing* the flag, *replacing it with a +> higher floor*, and *decomposing "protection" into five requirements*, and all +> three proposed a mechanism for a requirement never stated. **The requirement is +> non-circularity of the reference frame**, and it is satisfied *temporally*: you +> cannot recalibrate the ruler while measuring with it, so you don't — the frame +> updates while activation is internally seeded, not while it is being used to act. +> **Independence is *when*, not *what*.** So the flag becomes **unnecessary** rather +> than removed, and nothing takes its place. +> +> A topological answer could never have worked, which is worth recording: with +> hebbian edges the graph is densely connected, so a reachability predicate for +> "evidence not downstream of itself" marks all evidence tainted and the constraint +> becomes a total block — which is where censorship starts. +> +> **Corruption requires mutation, and the engram does not mutate.** Four of the +> five decomposed requirements are satisfied by the substrate outright: +> **recoverability** (the predecessor is always present), **governance** +> (supersession *is* the audit trail), **evidence quality** (grounding already +> gates assertion), and **rate**. **Authorization** is the only residue, and it is +> bounded — an unauthorized writer can *propose*, never erase. +> +> > **In an immutable substrate, any mechanism that refuses a write is either +> > redundant with immutability, or an epistemic constraint misfiled as a +> > protective one.** +> +> Live residue: `CogStance.keystone` (`lang/runtime/engram_cognition.h:83`), +> `eg_cog_is_keystone_seeds` (`el_runtime.c:14337`, a substring match against two +> hard-coded node ids), and the `keystone_write_blocked` field the beat emits. + --- ## 7. Rails for the build (binding on the eventual build pass) @@ -480,6 +705,35 @@ Ordered so the **earliest milestone is a real end-to-end slice**: one operator expressed as {primitive + grounded prior} with the reflexive correspondence-loop closing on it. Each milestone has a concrete verifiable exit. +> **⚠ SUPERSEDED — do not execute this milestone list as written (2026-08-16).** +> M1/M2's "operator = {primitive + prior}" framing is refuted by §0's correction, +> M3's `grounded-by` build is refuted by §5.2's, and M6's keystone flag is refuted +> by §6's. M4 (the unified vantage-read) and M5 (the gradient is the currency) +> stand. +> +> The current sequencing lives in `lang/spec/correspondence-and-censorship.md` §11. +> Its first three items are connections between parts that **already exist**: +> +> 1. **Seed *the* wonder questions.** Six nodes. Not a manifest, not maintained, +> never refilled. They cannot be derived — wonder cannot be bootstrapped from +> indifference — so they are given once. Zero question nodes exist in 13,630 +> today. +> 2. **Put the disagreement back on the edge** (`GeoEdge.discord`) and let +> `|discord|` raise salience on its endpoints as part of the same operation. Do +> **not** scan for nucleation sites. +> 3. **Let a curiosity seed activation.** One activation process, two seed sources. +> No thread, no scheduler, no capacity check, no timer. +> +> Then: grounding becomes the edge weight (multidimensional, two-axis, timestamped) +> and `grounded-by` / `cog_ground_edge` are deleted; decay becomes analytic from the +> last recorded point and derived values stop being stored; supersession versions +> the whole vector jointly; traversal conducts on the factual axis while `assert` +> requires both floors with a **thirteen-region `min`, not `mean`** (mean lets +> strong agreement with twelve values mask a violation of the thirteenth, which is +> exactly how rationalization works); abduction becomes crystallization at a +> nucleation site validated by re-fit; **one dreamer**, into which the launch-agent +> fragments and POST beats fold; **no tickers, no cron.** + ### M1 — One operator, one prior, loop closed (the vertical slice) The minimal whole thing. Pick **induction/membership** (its prior — the pooled diff --git a/engram/spec/engram-db-tooling-design.md b/engram/spec/engram-db-tooling-design.md index 9a94d45..c69f018 100644 --- a/engram/spec/engram-db-tooling-design.md +++ b/engram/spec/engram-db-tooling-design.md @@ -23,7 +23,7 @@ A real DB gets real tools: to *see* the data, *query* it, *operate* it (backup/r 2. **Node Inspector** — open one node: content, type, tier, embedding, typed edges, nearest neighbors by distance, provenance, salience / recency / activation, and supersede / tombstone status. 3. **Query Console / REPL** — run the geometry operations interactively: `vantage-read` (re-origin + aperture), search, traverse, activate, the reasoning operators. Surfaces the routing table + cosines — the same "this is not an LLM" receipt the language faculty produces. 4. **Ops / Durability Dashboard** — WAL size, last checkpoint, snapshot list + retention state, store stats (node/edge/embedded counts, RSS, tier sizes), health; and **backup / restore / point-in-time-recovery** controls. Pairs directly with the native-durability build (`eebe9991`) — this is the window onto it. -5. **Identity Inspector** — the self graph as a first-class view: love at the center, the values, the three faces, the covenant — walk the identity, see what's pinned and what's write-protected. +5. **Identity Inspector** — the self graph as a first-class view: love at the center, the values, the three faces, the covenant — walk the identity, see what's pinned and what's write-protected. *(⚠ 2026-08-16: "write-protected" is a live property of the surface, so the view is accurate — but it should be shown as **what it is**, not as a safety guarantee. In an immutable substrate, any mechanism that refuses a write is either redundant with immutability, or an epistemic constraint misfiled as a protective one. The identity view's real job is the **crystallized relational neighbourhood**: self is not a stored document but the shape that falls out of everything connected to it, and the neighbourhood **is** the grounding. A measurement made the other way round — "86 neighbours, 0 `grounded-by` edges" read as evidence of ungroundedness — was malformed: those 86 edges *are* its grounding.)* 6. **Temporal View** — `recall_at` / time-travel: how the geometry looked at a past moment, what changed since, drift over time. Pairs with temporal-self reconstruction. 7. **Schema / Type View** — the "information schema" of the geometry: node types, edge types, layers, tiers, counts. diff --git a/engram/spec/grounded-edge-propagation/LEDGER.md b/engram/spec/grounded-edge-propagation/LEDGER.md index 8c0ad66..338d262 100644 --- a/engram/spec/grounded-edge-propagation/LEDGER.md +++ b/engram/spec/grounded-edge-propagation/LEDGER.md @@ -1,9 +1,42 @@ # Task #50 — Edge-aware, dream-coupled consolidation with GROUNDED EDGE-PROPAGATION -**Status:** built + proven on a clone; **GATED, not promoted.** The main loop -sequences live promotion after the engine/HNSW cutover settles. +**Status:** built + proven on a clone; **GATED, not promoted.** +**Do not promote as designed** — see the block below. **Date:** 2026-08-15 · **Worktree:** `agent-a6577c8211c332c5b` (isolated). +> # ⚠ DO NOT PROMOTE — SUPERSEDED IN PART (2026-08-16) +> +> This work is gated, which limits the blast radius, and its measurements are +> retained. But four of its structural commitments were refuted the day after it +> was written. Authority: `lang/spec/correspondence-and-censorship.md`. Read it +> before any promotion decision. +> +> | this ledger | corrected to | +> |---|---| +> | grounding is an **append-only event ring on the node** (`GepGrounding`), propagated by a dedicated `engram_ground_propagate()` | **grounding is not a subsystem and not a per-node structure — it IS the edge weight.** One quantity. A relation that keeps holding up strengthens; one that stops corresponding decays. That is not analogous to grounding, it *is* grounding. The ledger is **half-right**: it correctly rejects the scalar (§(a) "never a scalar"), but then builds a *second* structure beside the weight instead of recognising the weight | +> | the soul invokes propagation over HTTP, **`POST /api/ground/propagate`** | **grounding is never computed on demand.** An operation may *read* the grounding of a path; computing-and-writing a score makes reads write, which is the `eg_vindex_sync` defect (`lang/spec/runtime-ownership.md` §2) one level up. A POST also puts a supervisor back in — something *outside* deciding when Neuron consolidates | +> | **`GEP_BELIEFS_PER_BEAT = 512`** beliefs per beat, salience-ordered, the rest next beat | **the presence of a ticker is the diagnostic.** Consolidation is ambient, not scheduled — a brain has no cron job. A per-beat quota is a rate-limiter on an intrinsic rhythm that was replaced by an external clock. Measured 2026-08-16: consolidation already has **ten implementations**; this would be the eleventh | +> | grounding **mirrored onto `confidence` each beat** so downstream reads never speak above it | **confidence is derived, therefore never stored.** Confidence is high grounding *and* low volatility. Storing it separately is precisely how `confidence: 0.5` ends up sitting beside a zero vector, asserting something nothing computed | +> +> **What survives, and it is the valuable half:** the insight in memory `69b8babe` +> that *memory-consolidation and staying-yourself are one physics* — forming a +> memory and grading a belief are the same operation, not two passes. That is +> right, and it is stronger than this ledger's own framing: they are not two passes +> of one beat, they are **one event**. When neurons fire together the synapse +> changes — one physical event, not "fire, then write." No supervisor reads the +> weight, compares it to a threshold, and decides to persist. **Potentiation *is* +> the firing**, so there is no sampling rate and no `BELIEFS_PER_BEAT` to tune. A +> relation changes in exactly two ways, neither requiring observation on a clock: +> by **use** (an event — there is no interval during which something happened +> unnoticed, because the event is what happening consists of) and by **decay** (a +> pure function of the last recorded point and elapsed time — **analytic**, known +> in closed form between any two versions). +> +> The generating error, named: modelling every property as requiring a process, and +> every process as requiring an agent. Ownership needed an owner, grounding needed +> a grounder, persistence needed a recorder, change needed a sampler. **Properties, +> not processes.** + Grounding mechanism designed with Will (memory `9e09a59f`, refining `1a861007`). This is the HOW for #50. diff --git a/engram/src/server.el b/engram/src/server.el index 01afb1b..1488513 100644 --- a/engram/src/server.el +++ b/engram/src/server.el @@ -10,10 +10,70 @@ // cc -std=c11 -O2 -lcurl -lpthread -o engram server.c el_runtime.c // ./engram // -// Configuration via environment: -// ENGRAM_BIND — host:port (default :8742) -// ENGRAM_API_KEY — bearer auth (optional) -// ENGRAM_DATA_DIR — snapshot location (default ~/.neuron/engram) +// Configuration is DECLARED, not scattered. See the `program` block below: +// every knob's type and default lives there and nowhere else, is resolved from +// the environment (env wins, declaration is the fallback) and validated before +// any statement of this file runs. Read one with config("NAME") -> String. +// +// The one deliberate exception is ENGRAM_DATA_DIR — see the note in the block. + +// ── Program declaration (cross-cutting concerns) ────────────────────────────── +// +// singleton: two engram processes against one data dir is data loss, not a +// warning. The runtime takes an exclusive flock at startup and a second start +// is refused loudly with the holder's pid. +// +// guards: names WHAT the singleton protects — this program's data directory. The +// lock lives inside it, so the guard is keyed on the store and not on the word +// "engram": two engrams against the same store cannot both run no matter how the +// environment is spelled, and two engrams against DIFFERENT stores are not each +// other's business and are not refused. Until 2026-08-16 the lock was keyed on +// the program name and $TMPDIR, and both of those sentences were false. +// +// It names the resolver rather than restating its path, for the same reason +// ENGRAM_DATA_DIR is NOT declared as an `env` entry below: engram_resolve_data_dir() +// (el_runtime.c) owns that path — it defaults to $HOME/.neuron/engram and fails +// LOUD rather than silently persisting to an ephemeral directory. Restating the +// default here would give the data dir two owners that can disagree, which is +// precisely the defect this migration removes (until 2026-08-15 the reseed backup +// path carried its own "/tmp/engram" default that disagreed with the resolver, so +// the pre-destructive safety copy landed in /tmp). A guard that resolved the path +// its own way could guard a directory the program never writes to. +// HOME is likewise not declared: it is a genuine environment read, not a knob. +program "engram" { + singleton: "engram" + guards: engram_resolve_data_dir() + + // ── Core server ── + env ENGRAM_BIND: String = ":8742" + // Default "" leaves auth DISABLED (check_auth_ok short-circuits to true on an + // empty key). That is the pre-existing behaviour and is deliberately preserved + // here; making this `required` is the obvious hardening follow-up, but it is a + // behaviour change and out of scope for this migration. + env ENGRAM_API_KEY: String = "" + + // ── Feature flags (bool-ish Strings; the predicate fns below own truthiness) ── + env ENGRAM_STORE: String = "off" + env ENGRAM_WAL: String = "off" + env ENGRAM_AUTOCONNECT: String = "off" + env ENGRAM_ISE_OFFGRAPH: String = "off" + + // ── ISE telemetry ── + env ENGRAM_ISE_RETENTION_MS: Int = "172800000" + + // ── Guide (local Qwen3 via llama-server) ── + env GUIDE_ENABLE: String = "off" + env GUIDE_TIER_FORCE: String = "" + env GUIDE_CACHE_DIR: String = "" + env GUIDE_RAM_GB_4B: Int = "16" + env GUIDE_RAM_GB_1P7B: Int = "8" + env GUIDE_BACKEND: String = "llama-server" + env GUIDE_HOST: String = "127.0.0.1" + env GUIDE_PORT: Int = "8771" + env GUIDE_LLAMA_SERVER_BIN: String = "llama-server" + env GUIDE_NGL: Int = "99" + env GUIDE_CTX: Int = "4096" +} // ── Helpers ─────────────────────────────────────────────────────────────────── @@ -41,17 +101,29 @@ fn strip_query(path: String) -> String { str_slice(path, 0, q) } +// query_param — extract one query-string value, URL-DECODED. +// +// The decode step was missing (found 2026-08-15): a claim sent as +// "test%20claim" arrived at engram_assert_json still percent-encoded and was +// stored/compared that way, so any value containing a space, &, =, or non-ASCII +// character silently became a different string than the caller sent. Affects +// every GET route that reads params this way, not just /api/assert. fn query_param(path: String, key: String) -> String { let q: Int = str_index_of(path, "?") if q < 0 { return "" } let qs: String = str_slice(path, q + 1, str_len(path)) - let needle: String = key + "=" - let pos: Int = str_index_of(qs, needle) + // Anchor the match to a real key boundary: prefixing "&" and searching for + // "&key=" means "q" can never match inside "faq=". (Found 2026-08-15: + // "?faq=X&q=Y" returned X for key "q" — a silently wrong value, not an + // error.) The leading "&" makes the first parameter match the same way. + let hay: String = "&" + qs + let needle: String = "&" + key + "=" + let pos: Int = str_index_of(hay, needle) if pos < 0 { return "" } - let after: String = str_slice(qs, pos + str_len(needle), str_len(qs)) + let after: String = str_slice(hay, pos + str_len(needle), str_len(hay)) let amp: Int = str_index_of(after, "&") - if amp < 0 { return after } - str_slice(after, 0, amp) + let raw: String = if amp < 0 { after } else { str_slice(after, 0, amp) } + return __url_decode(raw) } fn query_int(path: String, key: String, default_val: Int) -> Int { @@ -121,7 +193,7 @@ fn route_text_health(method: String, path: String, body: String) -> String { // engram_store_enabled() in el_runtime.c EXACTLY (1 / on / true). Default off → // every persistence path below is byte-for-byte the historical snapshot behavior. fn store_on() -> Bool { - let v: String = env("ENGRAM_STORE") + let v: String = config("ENGRAM_STORE") if str_eq(v, "1") { return true } if str_eq(v, "on") { return true } if str_eq(v, "true") { return true } @@ -150,7 +222,6 @@ fn persist_canonical() -> Int { if store_on() { return engram_store_checkpoint() } - let dir_raw: String = env("ENGRAM_DATA_DIR") let dir: String = engram_resolve_data_dir() // (2026-08-10 self-review) This returned a hardcoded 1, which made every // caller's `let saved: Int = persist_canonical()` a dead variable — six @@ -164,7 +235,7 @@ fn persist_canonical() -> Int { // per-write full-snapshot behavior. When ON, structural mutations append O(1) // WAL records instead of rewriting the whole graph, with threshold compaction. fn wal_on() -> Bool { - str_eq(env("ENGRAM_WAL"), "on") + str_eq(config("ENGRAM_WAL"), "on") } // autoconnect_on — ENGRAM_AUTOCONNECT. Will's rule: "we shouldn't be inserting @@ -172,7 +243,7 @@ fn wal_on() -> Bool { // edge (kNN over embeddings) so no content node enters the graph edgeless. // Default OFF -> byte-identical to prior behavior (node created, no auto edges). fn autoconnect_on() -> Bool { - let v: String = env("ENGRAM_AUTOCONNECT") + let v: String = config("ENGRAM_AUTOCONNECT") if str_eq(v, "1") { return true } if str_eq(v, "on") { return true } if str_eq(v, "true") { return true } @@ -185,7 +256,7 @@ fn autoconnect_on() -> Bool { // separate state-event log tier instead of the node graph. Default OFF -> ISEs // remain graph nodes exactly as before (with 48h prune). fn ise_offgraph_on() -> Bool { - let v: String = env("ENGRAM_ISE_OFFGRAPH") + let v: String = config("ENGRAM_ISE_OFFGRAPH") if str_eq(v, "1") { return true } if str_eq(v, "on") { return true } if str_eq(v, "true") { return true } @@ -235,6 +306,24 @@ fn persist_bulk() -> Int { return persist_canonical() } +// COMPILER LANDMINE, measured 2026-08-16 — do not inline this back into the +// caller. elc lowers `a == b` to numeric comparison only when both operand +// NAMES are in the per-function int-name set, which `let x: Int` populates. +// That registration does NOT propagate into a nested if-expression block: the +// first cut of the geometry-ingest path wrote `let claimed: Int = ...` and +// `let got: Int = ...` inside the else-arm and `claimed == got` came out of +// codegen as `str_eq(claimed, got)` — strcmp on two integers reinterpreted as +// pointers, i.e. a segfault on the first geometry-bearing request. Read back +// out of the generated C, not guessed. Function PARAMETERS annotated `: Int` +// do register reliably (verified: `if (claimed == actual)`), so the comparison +// lives in a function of its own. Note also the explicit `return`s — a trailing +// if-EXPRESSION at a function tail emits as a statement and the function +// returns 0 regardless, which is the same probe's second finding. +fn width_agrees(claimed: Int, actual: Int) -> Int { + if claimed == actual { return 1 } + return 0 +} + // INCOMPLETE-ROUTE FIX (2026-07-24 self-review): this route silently dropped // label, importance, tier, and tags — engram_node() defaults label to content // and importance to 0.5, so every node created over HTTP lost its metadata. @@ -276,6 +365,45 @@ fn route_create_node(method: String, path: String, body: String) -> String { salience, importance, confidence, tier, tags ) + // GEOMETRY INGEST — geometry-valued end to end (2026-08-16). + // + // The defect this route originally had: it accepted an "emb" field, + // returned 200 with a fresh id, and stored NOTHING, because engram_node_full + // has no vector parameter. The consequence was structural, not cosmetic — + // text was the only entry medium, so any non-text modality had to be + // DESCRIBED in prose, and what we then reasoned over was the geometry of the + // description, not of the signal. + // + // #141 fixed the drop but marshalled the vector as a hex STRING through + // engram_node_set_emb, which put text back as the TRANSPORT medium one layer + // below the problem being fixed. This is that correction: hex is decoded + // exactly ONCE, here at the edge, into a first-class Geometry, and every + // step below this line moves geometry rather than text. An encoding at the + // boundary is what an encoding is for. + // + // The WIRE is deliberately unchanged — "emb" is still little-endian float32 + // hex (8 chars per component), the encoding the perception vessel's + // /voice/embed already emits — because production clients speak it. What + // changed is underneath it. + // + // "dim" is now treated as an ASSERTION about the vector the caller sent, not + // as the source of its width: a Geometry carries its own width. A stated dim + // that disagrees is a REJECTED ingest, not a silent reinterpretation. Omitting + // "dim" is fine and means "trust the vector", which is the honest default. + // + // Off-dimension vectors remain stored but not inserted into the resident HNSW + // index (its build loop filters on emb_dim), so a 64-dim voice geometry is + // durable and addressable without perturbing the 768-dim canonical index. + let emb_hex: String = json_get_string(body, "emb") + let emb_set: Int = if str_eq(emb_hex, "") { 0 } else { + let g: Geometry = geometry_from_f32le_hex(emb_hex) + let got: Int = geometry_dim(g) + let dim_raw: String = json_get_raw(body, "dim") + let claimed: Int = if str_eq(dim_raw, "") { got } else { json_get_int(body, "dim") } + let landed: Int = if width_agrees(claimed, got) > 0 { node_attach_geometry(id, g) } else { 0 } + let freed: Int = geometry_free(g) + landed + } let saved: Int = persist_node(id) // ORPHAN PREVENTION (ENGRAM_AUTOCONNECT): connect the fresh node to its // nearest embedded neighbors so it never enters the graph edgeless. @@ -286,7 +414,11 @@ fn route_create_node(method: String, path: String, body: String) -> String { if added > 0 { let sv2: Int = persist_edges_since(ec0) } added } else { 0 } - "{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\",\"connected\":" + int_to_str(connected) + "}" + // Report whether the supplied geometry actually landed. The old response + // was success-shaped regardless — 200 with an id while the vector was + // discarded — which is how the drop went unnoticed. A caller can now + // assert on emb_set instead of trusting the status code. + "{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\",\"connected\":" + int_to_str(connected) + ",\"emb_set\":" + int_to_str(emb_set) + "}" } fn route_get_node(method: String, path: String, body: String) -> String { @@ -321,7 +453,6 @@ fn route_scan_nodes(method: String, path: String, body: String) -> String { // process ever booted with a partial/empty store, the first read request // clobbered the good snapshot. Read routes must never write the canonical path.) fn route_scan_edges(method: String, path: String, body: String) -> String { - let dir_raw: String = env("ENGRAM_DATA_DIR") let dir: String = engram_resolve_data_dir() let snap_path: String = dir + "/.scan-export.json" engram_save(snap_path) @@ -482,7 +613,6 @@ fn route_forget(method: String, path: String, body: String) -> String { fn route_save(method: String, path: String, body: String) -> String { let p_raw: String = json_get_string(body, "path") - let dir_raw: String = env("ENGRAM_DATA_DIR") let dir: String = engram_resolve_data_dir() let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw } // (2026-08-10 self-review) engram_save returns 0 on an empty path and the @@ -566,7 +696,6 @@ fn route_drift(method: String, path: String, body: String) -> String { fn route_load(method: String, path: String, body: String) -> String { let p_raw: String = json_get_string(body, "path") - let dir_raw: String = env("ENGRAM_DATA_DIR") let dir: String = engram_resolve_data_dir() let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw } // (2026-08-10 self-review) This was a stub response over the single most @@ -637,7 +766,6 @@ fn route_embed_backfill(method: String, path: String, body: String) -> String { // (it skips nodes already present by ID). Auth-exempt: same-host internal call. // (2026-06-27 self-review: added this route to fix silent 10-min sync failures) fn route_sync(method: String, path: String, body: String) -> String { - let dir_raw: String = env("ENGRAM_DATA_DIR") let dir: String = engram_resolve_data_dir() // 2026-07-21 self-review: export to a scratch path, never the canonical // snapshot.json — read routes must not be able to clobber the good snapshot. @@ -713,8 +841,12 @@ fn route_reseed_nodes(method: String, path: String, body: String) -> String { if str_eq(p, "") { return err_json("path is required") } if str_eq(fs_read(p), "") { return err_json("file missing or empty") } - let dir_raw: String = env("ENGRAM_DATA_DIR") - let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw } + // (2026-08-15) This site carried its own "/tmp/engram" fallback, which + // DISAGREED with engram_resolve_data_dir() ($HOME/.neuron/engram, fail-loud). + // The consumer is the pre-destructive backup below, so with ENGRAM_DATA_DIR + // unset the safety copy taken before a reseed landed in an ephemeral /tmp + // while the store it was protecting lived elsewhere. One owner, one answer. + let dir: String = engram_resolve_data_dir() let backup: String = dir + "/.reseed-backup.json" let replace_raw: String = json_get_raw(body, "replace") @@ -806,8 +938,7 @@ fn route_emit_ise(method: String, path: String, body: String) -> String { sal, imp, conf, "Episodic", "[\"internal-state\",\"InternalStateEvent\"]" ) - let ret_raw: String = env("ENGRAM_ISE_RETENTION_MS") - let ret_ms: Int = if str_eq(ret_raw, "") { 172800000 } else { str_to_int(ret_raw) } + let ret_ms: Int = str_to_int(config("ENGRAM_ISE_RETENTION_MS")) let pruned: Int = engram_prune_telemetry(ret_ms) "{\"ok\":true,\"id\":\"" + id + "\",\"pruned\":" + int_to_str(pruned) + "}" } @@ -904,6 +1035,22 @@ fn route_similarity(method: String, path: String, body: String) -> String { // nothing on request. NOTE: the offline reify WRITER (engram_geo_reify_store) is // currently unwired, so on the live store the resident index is empty and the // list returns [] until reification runs — see the cutover report. +// route_scan_emb — GET /api/nodes/emb?limit=&offset= — read the raw geometry. +// +// engram_scan_nodes_emb_json has existed as a builtin with NO ROUTE, so the +// embeddings — the actual positions every distance, angle, membership and +// grounding is computed from — were unreadable from outside the process. You +// cannot verify a coordinate system you cannot see, and every claim about the +// frame (isotropy, centering, what the origin is) was therefore unfalsifiable +// from the API. Read-only. +fn route_scan_emb(method: String, path: String, body: String) -> String { + let l_raw: String = query_param(path, "limit") + let o_raw: String = query_param(path, "offset") + let l: Int = if str_eq(l_raw, "") { 200 } else { str_to_int(l_raw) } + let o: Int = if str_eq(o_raw, "") { 0 } else { str_to_int(o_raw) } + return engram_scan_nodes_emb_json(l, o) +} + fn route_neighborhoods(method: String, path: String, body: String) -> String { engram_geo_reify_list_json() } @@ -997,6 +1144,11 @@ fn route_faculty(path: String, faculty: String) -> String { fn route_boundary_proof(method: String, path: String, body: String) -> String { return "{\"op\":\"boundary_proof\",\"body_instrumentation\":\"none\",\"seam\":\"@manager -> engram_boundary_beat auto-injected\"}" } +// ── GROUNDING: an attribute of the RELATION, and the relation's weight is a +// VECTOR (factual, relational, associative, polarity, provenance, timestamp). +// /api/ground READS it — it never writes. /api/ground/record is the write, +// named as one, and it consolidates only on a consequential + salient move. +// /api/ground/trajectory reads the supersession chain as a time series. fn route_ground(method: String, path: String, body: String) -> String { let claim: String = json_get_string(body, "claim") let evidence: String = json_get_string(body, "evidence") @@ -1005,12 +1157,31 @@ fn route_ground(method: String, path: String, body: String) -> String { if str_eq(evidence, "") { return err_json("missing evidence") } return engram_ground_json(claim, evidence, for_whom) } +fn route_ground_record(method: String, path: String, body: String) -> String { + let claim: String = json_get_string(body, "claim") + let evidence: String = json_get_string(body, "evidence") + let provenance: String = json_get_string(body, "provenance") + let floor: String = json_get_string(body, "floor") + if str_eq(claim, "") { return err_json("missing claim") } + if str_eq(evidence, "") { return err_json("missing evidence") } + return engram_ground_record_json(claim, evidence, provenance, floor) +} +fn route_ground_trajectory(method: String, path: String, body: String) -> String { + let claim: String = query_param(path, "claim") + let evidence: String = query_param(path, "evidence") + if str_eq(claim, "") { return err_json("missing claim") } + if str_eq(evidence, "") { return err_json("missing evidence") } + return engram_ground_trajectory_json(claim, evidence) +} fn route_assert(method: String, path: String, body: String) -> String { let claim: String = query_param(path, "claim") if str_eq(claim, "") { return err_json("missing claim") } let for_whom: String = query_param(path, "for_whom") let floor: String = query_param(path, "floor") - return engram_assert_json(claim, for_whom, floor) + // Both floors. A well-evidenced claim does not earn the right to be asserted + // regardless of whether it means the right thing. rel_floor defaults to floor. + let rel_floor: String = query_param(path, "rel_floor") + return engram_assert_json(claim, for_whom, floor, rel_floor) } fn route_attend(method: String, path: String, body: String) -> String { let node: String = json_get_string(body, "node") @@ -1056,14 +1227,12 @@ fn route_correspondence_beat(method: String, path: String, body: String) -> Stri // turns native thinking ON: the response carries reasoning_content (the thinking) // alongside content (the answer). -fn guide_env_or(key: String, dflt: String) -> String { - let v: String = env(key) - if str_eq(v, "") { return dflt } - return v -} +// (2026-08-15) guide_env_or(key, dflt) lived here. Its whole job was supplying a +// per-call-site default, which is now the program block's job — every GUIDE_* knob +// is declared once at the top of this file and read straight through config(). fn guide_enabled() -> Bool { - let v: String = env("GUIDE_ENABLE") + let v: String = config("GUIDE_ENABLE") if str_eq(v, "1") { return true } if str_eq(v, "on") { return true } if str_eq(v, "true") { return true } @@ -1108,15 +1277,15 @@ fn guide_probe_metal() -> Bool { // ── 2. Tier selection (config-driven thresholds, spec-autoselected) ──────────── fn guide_threshold_4b() -> Int { - return str_to_int(guide_env_or("GUIDE_RAM_GB_4B", "16")) + return str_to_int(config("GUIDE_RAM_GB_4B")) } fn guide_threshold_1p7b() -> Int { - return str_to_int(guide_env_or("GUIDE_RAM_GB_1P7B", "8")) + return str_to_int(config("GUIDE_RAM_GB_1P7B")) } // GUIDE_TIER_FORCE overrides the spec autoselect (used to prove cheaply on 0.6b). fn guide_select_tier(ram_gb: Int) -> String { - let forced: String = env("GUIDE_TIER_FORCE") + let forced: String = config("GUIDE_TIER_FORCE") if !str_eq(forced, "") { return forced } if ram_gb >= guide_threshold_4b() { return "4b" } if ram_gb >= guide_threshold_1p7b() { return "1.7b" } @@ -1136,8 +1305,10 @@ fn guide_file(tier: String) -> String { } fn guide_cache_dir() -> String { - let c: String = env("GUIDE_CACHE_DIR") + let c: String = config("GUIDE_CACHE_DIR") if !str_eq(c, "") { return c } + // HOME stays a raw env() read: it is the ambient environment, not a knob of + // this program, and it is deliberately absent from the program block. let home: String = env("HOME") if !str_eq(home, "") { return home + "/.neuron/guide/models" } return engram_resolve_data_dir() + "/guide-models" @@ -1178,9 +1349,9 @@ fn guide_fetch(tier: String) -> Bool { } // ── 4/5. Backend abstraction + BIND as an engageable interlocutor ────────────── -fn guide_backend() -> String { return guide_env_or("GUIDE_BACKEND", "llama-server") } -fn guide_host() -> String { return guide_env_or("GUIDE_HOST", "127.0.0.1") } -fn guide_port() -> String { return guide_env_or("GUIDE_PORT", "8771") } +fn guide_backend() -> String { return config("GUIDE_BACKEND") } +fn guide_host() -> String { return config("GUIDE_HOST") } +fn guide_port() -> String { return config("GUIDE_PORT") } fn guide_base_url() -> String { return "http://" + guide_host() + ":" + guide_port() } // guide_healthy — is the guide present and answering? llama-server's /health @@ -1198,9 +1369,9 @@ fn guide_healthy() -> Bool { fn guide_load(tier: String) -> Bool { if guide_healthy() { return true } let path: String = guide_model_path(tier) - let bin: String = guide_env_or("GUIDE_LLAMA_SERVER_BIN", "llama-server") - let ngl: String = guide_env_or("GUIDE_NGL", "99") - let ctx: String = guide_env_or("GUIDE_CTX", "4096") + let bin: String = config("GUIDE_LLAMA_SERVER_BIN") + let ngl: String = config("GUIDE_NGL") + let ctx: String = config("GUIDE_CTX") let logf: String = guide_cache_dir() + "/llama-server." + guide_port() + ".log" let cmd: String = bin + " -m '" + path + "' --host " + guide_host() + " --port " + guide_port() + " -c " + ctx + " -ngl " + ngl + " --jinja >> '" + logf + "' 2>&1" let pid: String = exec_bg(cmd) @@ -1308,9 +1479,16 @@ fn route_guide_summon(method: String, path: String, body: String) -> String { // // The SINGLE NODE is the DEGENERATE n=1 case of this SAME operation — not a // separate CRUD path: -// write(content) = reframe(region=∅, manifold=[1 node]) (route_write) +// write(signal) = realize(signal) → reframe(region=∅, manifold) (route_write) // supersede(id,new) = reframe(region={id}, manifold=[1 node]) (route_supersede) // relate(a,b,rel) = the rebind sub-op in isolation (route_create_edge) +// +// CORRECTED 2026-08-16: write was documented above as +// "reframe(region=∅, manifold=[1 node])", and the "[1 node]" was not the design +// — it was the DEFECT. A node is an OUTPUT of realization, never an INPUT to +// it. What arrives at an intake route is a SIGNAL, and how many nodes it +// becomes is for the realizer to say, not for the route to assume. See +// "INTAKE" below. // The ONLY anti-pattern is decomposing a region-scale change into a LOOP of // independent top-level per-node updates. Here the region is the unit: one // isolate, one atomic set-replace, one persist, one verify — iterating members @@ -1525,6 +1703,174 @@ fn reframe_core(region: [String], manifold: String, reason: String, do_rebind: I ",\"keystones_protected\":true}" } +// ═══════════════════════════════════════════════════════════════════════════ +// INTAKE — the ONE door: signal → realization → manifold → store. +// +// THERE IS NO WRITE NODE. What arrives at an intake route is a SIGNAL. A node +// is an OUTPUT of realization, never an INPUT to it. route_write used to say: +// +// let manifold: String = "[" + body + "]" // the body IS a valid manifold node object +// +// and hand that to reframe_core. That is not a manifold — it is the request +// body wearing the word, and the comment stated the wrong assumption out loud. +// It is why a compound signal landed as ONE flat node with ZERO edges. Measured +// before this change, on a cp -Rc clone: +// POST /api/write {"type":"memory","content":"A cathedral is stone holding a +// shape that stone alone would not hold."} +// → {"ok":true,"inserted":1,"nodes_added":1,"edges_added":0,...} +// GET /api/neighbors/ → [] (read back out, not taken on trust) +// +// NOTHING IS DECOMPOSED HERE, AND NOTHING MAY EVER BE. transduce(signal, +// modality) IS the realization primitive (el_runtime.c: "Manifold", +// "Realizers + transduce"). It dispatches through the dlsym realizer registry, +// so ADDING A MODALITY IS REGISTERING A REALIZER — never an edit to this file, +// and never a patch to the runtime. This function only carries what the +// primitive returns into the store, which is the one thing the engram's HTTP +// surface has never done: `grep -n 'transduce\|realize\|Manifold\|decompos' +// engram/src/server.el` returned exactly one line before this change, a comment. +// +// GENERAL BY CONSTRUCTION, NOT SPECIAL-CASED TO route_write. Five of the six +// intake doors (write, supersede, nodes, neuron/knowledge/capture, +// neuron/state-events) are the same hand-written "content string → +// engram_node_full → one flat node", differing ONLY in the node_type / tier / +// tags they hardcode. Those are parameters here, so each door can be moved onto +// this one function as it is transitioned. Only /api/write rides it in this +// pass; the rest are listed as remaining work. +// +// WHEN THERE IS NO ORGAN the signal is stored flat exactly as before, and the +// response SAYS SO ("realized":false, "organ":false). Silent flattening is the +// actual defect — a caller could not distinguish "nothing decomposed me" from +// "I decomposed into one component". el_runtime.c draws the same line at +// registration time, between an absent organ and a broken one, for the same +// reason: those two must not look alike. +// ═══════════════════════════════════════════════════════════════════════════ + +// Resolve a component KEY to the node id it was inserted as. Components are +// addressed BY KEY, never by index (el_runtime.c, "Manifold"), because the key +// is what survives persistence — so relations are resolved by key too. +fn key_to_id(keys: [String], ids: [String], key: String) -> String { + let n: Int = el_list_len(keys) + let i: Int = 0 + while i < n { + if str_eq(el_list_get(keys, i), key) { return el_list_get(ids, i) } + i = i + 1 + } + return "" +} + +fn intake_signal(signal: String, modality: String, region: [String], + nt_in: String, tier_in: String, tags: String, + reason: String, do_rebind: Int) -> String { + let n_before: Int = engram_node_count() + let e_before: Int = engram_edge_count() + let region_n: Int = el_list_len(region) + let tomb: String = if region_n > 0 { supersede_set(region, reason) } else { "" } + + // Identity can never be minted through intake — the same rule + // insert_manifold_json holds, applied at the one door instead of per-route. + let nt: String = if str_eq(nt_in, "") { "Memory" } else { nt_in } + if str_eq(nt, "self") { nt = "Memory" } + if str_eq(nt, "values") { nt = "Memory" } + let tier: String = if str_eq(tier_in, "") { "Working" } else { tier_in } + + let has_organ: Int = realizer_has(modality) + let new_ids: [String] = el_list_empty() + let keys: [String] = el_list_empty() + let ncomp: Int = 0 + let nrel: Int = 0 + let realized: Int = 0 + + if has_organ > 0 { + let m: Manifold = transduce(signal, modality) + // A realizer that returns a bare Geometry transduces NOTHING by design + // (el_runtime.c) — manifold_is() is the check, so a fingerprinting organ + // is not silently mistaken for a decomposing one. + if manifold_is(m) > 0 { + realized = 1 + ncomp = manifold_size(m) + let i: Int = 0 + let prev: String = "" + while i < ncomp { + let key: String = manifold_key(m, i) + let role: String = manifold_role(m, i) + // The component's OWN geometry, at its own width — this is the + // whole point of a manifold over a fingerprint, and it is why + // node_attach_geometry is used rather than re-embedding the + // component's name as text. + let g: Geometry = manifold_geometry(m, i) + let ctags: String = "[\"component\",\"role:" + role + "\",\"modality:" + modality + "\"]" + let cid: String = engram_node_full(key, nt, key, 0.5, 0.5, 0.9, tier, ctags) + let landed: Int = node_attach_geometry(cid, g) + let freed: Int = geometry_free(g) + new_ids = el_list_append(new_ids, cid) + keys = el_list_append(keys, key) + // PRESERVED CONTRACT: manifold_member wires the inserted set + // into one connected sub-graph, exactly as insert_manifold_json + // already did. Not reinvented — reused. + if !str_eq(prev, "") { engram_connect(prev, cid, 0.6, "manifold_member") } + prev = cid + i = i + 1 + } + // THE RELATIONS ARE THE CONTENT. Relation weight IS the grounding + // (correspondence-and-censorship §1) — it arrives on the edge from + // the realizer and nothing here computes or second-guesses it. + nrel = manifold_rel_count(m) + let j: Int = 0 + while j < nrel { + let fk: String = manifold_rel_from(m, j) + let rn: String = manifold_rel_name(m, j) + let tk: String = manifold_rel_to(m, j) + let w: Float = manifold_rel_weight(m, j) + let fid: String = key_to_id(keys, new_ids, fk) + let tid: String = key_to_id(keys, new_ids, tk) + if !str_eq(fid, "") { + if !str_eq(tid, "") { + engram_connect(fid, tid, w, rn) + } + } + j = j + 1 + } + let mfreed: Int = manifold_free(m) + } + } + + // NO ORGAN: store the signal flat, as before — but say so. This is the + // pre-existing behaviour preserved verbatim, not a new fallback path. + if realized == 0 { + let label: String = str_slice(signal, 0, 60) + let fid: String = engram_node_full(signal, nt, label, 0.5, 0.5, 0.9, tier, tags) + new_ids = el_list_append(new_ids, fid) + } + + let inserted: Int = el_list_len(new_ids) + let bound: Int = if do_rebind > 0 { rebind_cosine(new_ids, tomb) } else { 0 } + let saved: Int = persist_canonical() + let new_csv: String = "" + let k: Int = 0 + while k < inserted { + let sep: String = if k == 0 { "" } else { "," } + new_csv = new_csv + sep + "\"" + el_list_get(new_ids, k) + "\"" + k = k + 1 + } + let realized_s: String = if realized > 0 { "true" } else { "false" } + let organ_s: String = if has_organ > 0 { "true" } else { "false" } + return "{\"ok\":true,\"region_superseded\":" + int_to_str(region_n) + + ",\"tombstone_id\":\"" + tomb + "\"" + + ",\"inserted\":" + int_to_str(inserted) + + ",\"new_ids\":[" + new_csv + "]" + + ",\"edges_rebound\":" + int_to_str(bound) + + ",\"realized\":" + realized_s + + ",\"modality\":\"" + modality + "\"" + + ",\"organ\":" + organ_s + + ",\"components\":" + int_to_str(ncomp) + + ",\"relations\":" + int_to_str(nrel) + + ",\"nodes_added\":" + int_to_str(engram_node_count() - n_before) + + ",\"edges_added\":" + int_to_str(engram_edge_count() - e_before) + + ",\"node_count\":" + int_to_str(engram_node_count()) + + ",\"edge_count\":" + int_to_str(engram_edge_count()) + + ",\"keystones_protected\":true}" +} + // POST /api/reframe — the universal set-based mutation. // Body: {vantage?, region_ids?(csv), k?, expand?, manifold(json array), reason?, rebind?} // region_ids (explicit) wins; else cosine-isolate around vantage. @@ -1561,18 +1907,32 @@ fn route_reframe(method: String, path: String, body: String) -> String { return reframe_core(region, manifold, reason, do_rebind) } -// write — DEGENERATE n=1 of reframe: region=∅, manifold=[1 node]. The SAME -// reframe_core path. rebind off so the pure-add matches plain node creation. -// POST /api/write {content, node_type?, tier?, tags?} +// write — INTAKE OF A SIGNAL. Not "reframe with a manifold of one node": the +// route no longer decides how many nodes the signal is. It hands the signal to +// the realization primitive and stores whatever manifold comes back. +// +// The line this replaces was: +// let manifold: String = "[" + body + "]" // the body IS a valid manifold node object +// which asserted that a request body is a manifold. It is not, and that single +// assertion is the whole measured defect (1 node, 0 edges, [] neighbors). +// +// rebind stays off so a pure add still matches plain node creation. +// POST /api/write {content, modality?, node_type?, tier?, tags?} fn route_write(method: String, path: String, body: String) -> String { let content: String = json_get_string(body, "content") if str_eq(content, "") { return err_json("write: content required") } let nt: String = json_get_string(body, "node_type") if str_eq(nt, "self") { return err_json("write: identity is write-protected") } if str_eq(nt, "values") { return err_json("write: identity is write-protected") } + // The modality names which organ to sense with. It is data, never a branch: + // a new modality is a realizer_register call somewhere else in the program, + // not another endpoint and not another case here. + let mod_raw: String = json_get_string(body, "modality") + let modality: String = if str_eq(mod_raw, "") { "text" } else { mod_raw } + let tier: String = json_get_string(body, "tier") + let tags: String = json_get_raw(body, "tags") let empty: [String] = el_list_empty() - let manifold: String = "[" + body + "]" // the body IS a valid manifold node object - return reframe_core(empty, manifold, "write", 0) + return intake_signal(content, modality, empty, nt, tier, tags, "write", 0) } // supersede — DEGENERATE n=1 of reframe: region={id}, manifold=[1 node]. The @@ -1596,7 +1956,7 @@ fn route_supersede(method: String, path: String, body: String) -> String { // ── Auth ────────────────────────────────────────────────────────────────────── fn check_auth_ok(method: String, body: String) -> Bool { - let key: String = env("ENGRAM_API_KEY") + let key: String = config("ENGRAM_API_KEY") if str_eq(key, "") { return true } // Read-only methods don't require auth. Until http_serve surfaces // request headers we can't accept a Bearer token cleanly; mutating @@ -1675,6 +2035,9 @@ fn handle_request(method: String, path: String, body: String) -> String { if str_eq(method, "GET") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) { return route_scan_edges(method, path, body) } + if str_eq(method, "GET") && (str_eq(clean, "/api/nodes/emb") || str_eq(clean, "/nodes/emb")) { + return route_scan_emb(method, path, body) + } if str_eq(method, "GET") && str_starts_with(clean, "/api/nodes/") { return route_get_node(method, path, body) } @@ -1764,6 +2127,14 @@ fn handle_request(method: String, path: String, body: String) -> String { if str_eq(method, "GET") && str_starts_with(clean, "/api/plan") { return route_faculty(path, "plan") } + // Order matters: the more specific paths must be tested before the /api/ground + // prefix match below, which would otherwise swallow them. + if str_eq(method, "POST") && str_starts_with(clean, "/api/ground/record") { + return route_ground_record(method, path, body) + } + if str_eq(method, "GET") && str_starts_with(clean, "/api/ground/trajectory") { + return route_ground_trajectory(method, path, body) + } if str_eq(method, "POST") && str_starts_with(clean, "/api/ground") { return route_ground(method, path, body) } @@ -1859,8 +2230,7 @@ fn handle_request(method: String, path: String, body: String) -> String { // ── Entry ───────────────────────────────────────────────────────────────────── -let bind_raw: String = env("ENGRAM_BIND") -let bind_str: String = if str_eq(bind_raw, "") { ":8742" } else { bind_raw } +let bind_str: String = config("ENGRAM_BIND") let port: Int = parse_port(bind_str) // On startup, try to load any existing snapshot (best effort). diff --git a/engram/test/run_grounding_vector_tests.sh b/engram/test/run_grounding_vector_tests.sh new file mode 100755 index 0000000..7279d96 --- /dev/null +++ b/engram/test/run_grounding_vector_tests.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# Build + RUN the §7 GROUNDING-VECTOR tests (engram_cognition.c): the one decay +# model, the consequence gate, and the stored/derived split. Closed-form +# constructed cases — no server, no store, no network. Pure C11 (stdlib + libm). +# Standalone — NOT folded through elc. Two passes: +# 1. PERF — optimised (-O2, no sanitizer): the functional gate. +# 2. SAFETY — ASan + UBSan on the same suite. +# +# NEGATIVE CONTROL (invariant §8.6 — no test without one). Every symbol this +# suite exercises (cog_decay_factor, cog_grounding_significant, +# cog_significance_inherent, CogGrounding, CogProvClass) is introduced by the +# change under test, so the suite does not COMPILE against the pre-change source. +# To reproduce: +# git show origin/dev:lang/runtime/engram_cognition.h > /tmp/pre/engram_cognition.h +# git show origin/dev:lang/runtime/engram_cognition.c > /tmp/pre/engram_cognition.c +# cc -I/tmp/pre engram/test/test_grounding_vector.c /tmp/pre/engram_cognition.c ... +# => error: unknown type name 'CogGrounding'; no binary produced. +set -e +HERE=$(cd "$(dirname "$0")" && pwd) +RT="$HERE/../../lang/runtime" +CC=${CC:-cc} +SRC="$HERE/test_grounding_vector.c $RT/engram_cognition.c $RT/engram_reason.c $RT/engram_geometry.c $RT/engram_store.c $RT/engram_vindex.c" +WARN="-std=c11 -Wall -Wextra" +# engram_store.c declares emit_log as a WEAK symbol and null-checks it, which is +# how a test links the store without the EL runtime. Darwin's ld does not resolve +# an undefined weak symbol at static-link time, so it must be allowed explicitly. +# (The pre-existing runners in this directory — run_verify_tests.sh among them — +# do not do this and therefore fail to link on macOS. Unrelated to this change.) +LDX="" +[ "$(uname -s)" = "Darwin" ] && LDX="-Wl,-U,_emit_log" +TMP=$(mktemp -d) + +echo "### PASS 1: PERF (optimised, un-sanitised) — functional gate" +$CC $WARN -O2 -I"$RT" $SRC -lm -lpthread $LDX -o "$TMP/perf" +"$TMP/perf" + +echo +echo "### PASS 2: SAFETY (ASan/UBSan)" +$CC $WARN -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer -I"$RT" $SRC -lm -lpthread $LDX -o "$TMP/safe" +ASAN_OPTIONS=${ASAN_OPTIONS:-detect_leaks=0} UBSAN_OPTIONS=halt_on_error=1 "$TMP/safe" diff --git a/engram/test/run_interoception_p0.sh b/engram/test/run_interoception_p0.sh index 99507ab..9ee3ed8 100755 --- a/engram/test/run_interoception_p0.sh +++ b/engram/test/run_interoception_p0.sh @@ -3,10 +3,14 @@ # Throwaway HOME + /tmp only. Never touches ~/.neuron or :8742. set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" -GEO="$HERE/../../lang/runtime/engram_geometry.c" -VIDX="$HERE/../../lang/runtime/engram_vindex.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-p0-XXXXXX)" export HOME="$WORK/home"; mkdir -p "$HOME" @@ -14,8 +18,8 @@ unset ENGRAM_STORE fail=0 echo "== compile (plain) ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p0_emb.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p0" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } +gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p0_emb.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p0" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } D="$WORK/d"; mkdir -p "$D" "$WORK/p0" "$D" || { echo "FAIL: run"; fail=1; } @@ -69,8 +73,8 @@ PY echo echo "== ASan+UBSan ==" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_interoception_p0_emb.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p0.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -20 "$WORK/san_cc.log"; fail=1; } + -I "$INC" "$HERE/test_interoception_p0_emb.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p0.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -20 "$WORK/san_cc.log"; fail=1; } if [ -x "$WORK/p0.san" ]; then export ASAN_OPTIONS=detect_leaks=0 DS="$WORK/ds"; mkdir -p "$DS" diff --git a/engram/test/run_interoception_p1.sh b/engram/test/run_interoception_p1.sh index 41e5dbe..82591a1 100755 --- a/engram/test/run_interoception_p1.sh +++ b/engram/test/run_interoception_p1.sh @@ -3,10 +3,14 @@ # Throwaway HOME + /tmp only. Never touches ~/.neuron or :8742. set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" -GEO="$HERE/../../lang/runtime/engram_geometry.c" -VIDX="$HERE/../../lang/runtime/engram_vindex.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-p1-XXXXXX)" export HOME="$WORK/home"; mkdir -p "$HOME" @@ -14,8 +18,8 @@ unset ENGRAM_STORE ENGRAM_CONSOLIDATION ENGRAM_CONSOL_CONN_MIN ENGRAM_CONSOL_PER fail=0 echo "== compile ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p1_consol.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p1" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } +gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p1_consol.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p1" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } echo echo "== (a) HEADLINE: hebb accrual curve over N co-activations (flag OFF, pure trunk) ==" @@ -129,8 +133,8 @@ cat "$WORK/off.txt" | sed 's/^/ /' echo echo "== ASan+UBSan (connect + perm + accrual-short) ==" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_interoception_p1_consol.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p1.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } + -I "$INC" "$HERE/test_interoception_p1_consol.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p1.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } if [ -x "$WORK/p1.san" ]; then export ASAN_OPTIONS=detect_leaks=0 DS="$WORK/san"; mkdir -p "$DS" diff --git a/engram/test/run_interoception_p2.sh b/engram/test/run_interoception_p2.sh index 3e209b5..0d2b7a9 100755 --- a/engram/test/run_interoception_p2.sh +++ b/engram/test/run_interoception_p2.sh @@ -3,10 +3,14 @@ # Throwaway HOME + /tmp only. TC defaults to 3600s; we pin it for the math. set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" -GEO="$HERE/../../lang/runtime/engram_geometry.c" -VIDX="$HERE/../../lang/runtime/engram_vindex.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-p2-XXXXXX)" export HOME="$WORK/home"; mkdir -p "$HOME" @@ -15,8 +19,8 @@ unset ENGRAM_STORE fail=0 echo "== compile ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p2_chrono.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p2" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } +gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p2_chrono.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p2" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } sum_wm(){ python3 -c "import json,sys; g=json.load(open('$1')); print(sum(n.get('working_memory_weight',0) for n in g['nodes']))"; } @@ -78,8 +82,8 @@ python3 -c "import sys; sys.exit(0 if abs($OFFWM-1.2)<1e-9 else 1)" \ echo echo "== ASan+UBSan ==" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_interoception_p2_chrono.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p2.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } + -I "$INC" "$HERE/test_interoception_p2_chrono.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p2.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } if [ -x "$WORK/p2.san" ]; then export ASAN_OPTIONS=detect_leaks=0 DS="$WORK/san"; mkdir -p "$DS" diff --git a/engram/test/run_interoception_p3.sh b/engram/test/run_interoception_p3.sh index 26d2fd6..f56596e 100755 --- a/engram/test/run_interoception_p3.sh +++ b/engram/test/run_interoception_p3.sh @@ -3,18 +3,22 @@ # Read-only pure primitive; no store, no flag. Throwaway /tmp only. set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" -GEO="$HERE/../../lang/runtime/engram_geometry.c" -VIDX="$HERE/../../lang/runtime/engram_vindex.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-p3-XXXXXX)" export HOME="$WORK/home"; mkdir -p "$HOME" fail=0 echo "== compile ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p3_drift.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p3" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } +gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p3_drift.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p3" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } "$WORK/p3" > "$WORK/out.txt" 2>&1 || { echo "FAIL run"; cat "$WORK/out.txt"; fail=1; } cat "$WORK/out.txt" | sed 's/^/ /' @@ -52,8 +56,8 @@ PY echo echo "== ASan+UBSan ==" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_interoception_p3_drift.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p3.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } + -I "$INC" "$HERE/test_interoception_p3_drift.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p3.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } if [ -x "$WORK/p3.san" ]; then export ASAN_OPTIONS=detect_leaks=0 "$WORK/p3.san" >/dev/null 2>"$WORK/san.log" diff --git a/engram/test/run_interoception_p4.sh b/engram/test/run_interoception_p4.sh index 9d81bc4..656c646 100755 --- a/engram/test/run_interoception_p4.sh +++ b/engram/test/run_interoception_p4.sh @@ -2,10 +2,14 @@ # M-INTEROCEPTION P4 gate: afferent input counters in act-stats (additive). set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" -GEO="$HERE/../../lang/runtime/engram_geometry.c" -VIDX="$HERE/../../lang/runtime/engram_vindex.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-p4-XXXXXX)" export HOME="$WORK/home"; mkdir -p "$HOME" @@ -13,8 +17,8 @@ unset ENGRAM_STORE fail=0 echo "== compile ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p4_afferent.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p4" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } +gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p4_afferent.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p4" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } "$WORK/p4" > "$WORK/out.txt" 2>&1 || { echo "FAIL run"; cat "$WORK/out.txt"; fail=1; } grep -oE 'aff_[a-z_]+":[0-9]+' "$WORK/out.txt" | sed 's/^/ /' | head -30 @@ -53,8 +57,8 @@ PY echo echo "== ASan+UBSan ==" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_interoception_p4_afferent.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p4.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } + -I "$INC" "$HERE/test_interoception_p4_afferent.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p4.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } if [ -x "$WORK/p4.san" ]; then export ASAN_OPTIONS=detect_leaks=0 "$WORK/p4.san" >/dev/null 2>"$WORK/san.log" diff --git a/engram/test/run_interoception_p5.sh b/engram/test/run_interoception_p5.sh index 90e6c81..a21e85e 100755 --- a/engram/test/run_interoception_p5.sh +++ b/engram/test/run_interoception_p5.sh @@ -2,10 +2,14 @@ # M-INTEROCEPTION P5 gate: dream-recall builtin engram_dreams_json (honesty rail). set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" -GEO="$HERE/../../lang/runtime/engram_geometry.c" -VIDX="$HERE/../../lang/runtime/engram_vindex.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-p5-XXXXXX)" export HOME="$WORK/home"; mkdir -p "$HOME" @@ -13,8 +17,8 @@ unset ENGRAM_STORE fail=0 echo "== compile ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p5_dreams.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p5" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } +gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p5_dreams.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p5" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } D="$WORK/d"; mkdir -p "$D" "$WORK/p5" "$D" > "$WORK/out.txt" 2>&1 || { echo "FAIL run"; cat "$WORK/out.txt"; fail=1; } @@ -57,8 +61,8 @@ PY echo echo "== ASan+UBSan ==" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_interoception_p5_dreams.c" "$RT" "$ST" "$GEO" "$VIDX" \ - -lcurl -lm -o "$WORK/p5.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } + -I "$INC" "$HERE/test_interoception_p5_dreams.c" $RTSRC $SSLFLAGS \ + -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p5.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; } if [ -x "$WORK/p5.san" ]; then export ASAN_OPTIONS=detect_leaks=0 DS="$WORK/ds"; mkdir -p "$DS" diff --git a/engram/test/run_m35_hebb_persist.sh b/engram/test/run_m35_hebb_persist.sh index 1ce7dd3..1691aae 100755 --- a/engram/test/run_m35_hebb_persist.sh +++ b/engram/test/run_m35_hebb_persist.sh @@ -6,8 +6,14 @@ # Writes ONLY under a throwaway /tmp dir with a throwaway HOME. set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-m35-XXXXXX)" BIN="$WORK/m35" @@ -17,7 +23,7 @@ unset ENGRAM_STORE fail=0 echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m35_hebb_persist.c) ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_m35_hebb_persist.c" "$RT" "$ST" -lcurl -o "$BIN" 2>"$WORK/cc.log" +gcc -O1 -std=c11 -I "$INC" "$HERE/test_m35_hebb_persist.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$BIN" 2>"$WORK/cc.log" if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi echo @@ -139,7 +145,7 @@ echo echo "== 5) ASan+UBSan build, exercise the full persist+reboot flow (leaks off — harness intentionally leaks el_strdup) ==" SANBIN="$WORK/m35.san" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_m35_hebb_persist.c" "$RT" "$ST" -lcurl -o "$SANBIN" 2>"$WORK/san_cc.log" + -I "$INC" "$HERE/test_m35_hebb_persist.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$SANBIN" 2>"$WORK/san_cc.log" if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else export ASAN_OPTIONS=detect_leaks=0 DSAN="$WORK/san"; mkdir -p "$DSAN" diff --git a/engram/test/run_m3_parity.sh b/engram/test/run_m3_parity.sh index 6eadaee..c7d533a 100755 --- a/engram/test/run_m3_parity.sh +++ b/engram/test/run_m3_parity.sh @@ -4,8 +4,14 @@ # Writes ONLY under a throwaway /tmp dir with a throwaway HOME + ENGRAM_DATA_DIR. set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-m3-XXXXXX)" DATA="$WORK/data"; mkdir -p "$DATA" @@ -17,7 +23,7 @@ unset ENGRAM_STORE fail=0 echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m3_parity.c) ==" -gcc -O1 -std=c11 -I "$INC" "$HERE/test_m3_parity.c" "$RT" "$ST" -lcurl -o "$BIN" 2>"$WORK/cc.log" +gcc -O1 -std=c11 -I "$INC" "$HERE/test_m3_parity.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$BIN" 2>"$WORK/cc.log" if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi grep -i warning "$WORK/cc.log" | grep -iE 'engram_store|eg_store|eg_load|scan_nodes|scan_edges' && echo "(warnings in M3 code above)" || true @@ -106,7 +112,7 @@ echo echo "== 5) ASan+UBSan build, exercise M3 scan/boot/hooks (leaks off — harness intentionally leaks el_strdup) ==" SANBIN="$WORK/m3.san" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_m3_parity.c" "$RT" "$ST" -lcurl -o "$SANBIN" 2>"$WORK/san_cc.log" + -I "$INC" "$HERE/test_m3_parity.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$SANBIN" 2>"$WORK/san_cc.log" if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else export ASAN_OPTIONS=detect_leaks=0 DATA2="$WORK/data2"; mkdir -p "$DATA2" diff --git a/engram/test/run_m7_traversal.sh b/engram/test/run_m7_traversal.sh index 693a9fe..1340704 100755 --- a/engram/test/run_m7_traversal.sh +++ b/engram/test/run_m7_traversal.sh @@ -6,8 +6,14 @@ # Writes ONLY under a throwaway /tmp dir with a throwaway HOME. set -u HERE="$(cd "$(dirname "$0")" && pwd)" -RT="$HERE/../../lang/runtime/el_runtime.c" -ST="$HERE/../../lang/runtime/engram_store.c" +RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")" +# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link +# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c +# began calling into the other engram siblings. Unquoted on purpose: a list. +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi INC="$HERE/../../lang/runtime" WORK="$(mktemp -d /tmp/engram-m7-XXXXXX)" DATA="$WORK/data"; mkdir -p "$DATA" @@ -22,7 +28,7 @@ unset ENGRAM_STORE fail=0 echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m7_traversal.c) ==" -gcc -O2 -std=c11 -I "$INC" "$HERE/test_m7_traversal.c" "$RT" "$ST" -lcurl -lm -o "$BIN" 2>"$WORK/cc.log" +gcc -O2 -std=c11 -I "$INC" "$HERE/test_m7_traversal.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$BIN" 2>"$WORK/cc.log" if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi echo " ok: compiled" @@ -115,7 +121,7 @@ echo echo "== 3) ASan+UBSan clean across parity + a small perf loop (leaks off — harness intentionally leaks el_strdup) ==" SANBIN="$WORK/m7.san" gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ - -I "$INC" "$HERE/test_m7_traversal.c" "$RT" "$ST" -lcurl -lm -o "$SANBIN" 2>"$WORK/san_cc.log" + -I "$INC" "$HERE/test_m7_traversal.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$SANBIN" 2>"$WORK/san_cc.log" if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else export ASAN_OPTIONS=detect_leaks=0 D2="$WORK/data2"; mkdir -p "$D2" diff --git a/engram/test/run_vindex_concurrency_tests.sh b/engram/test/run_vindex_concurrency_tests.sh new file mode 100755 index 0000000..4e90c5d --- /dev/null +++ b/engram/test/run_vindex_concurrency_tests.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# run_vindex_concurrency_tests.sh — regression harness for the 2026-08-16 soul crash. +# +# Four halves. The SET is the point: it separates two hazards the original two-half +# version conflated, and which have fixes in different files. +# +# 1. single ASan+UBSan, one thread. MUST be clean. Hard failure. +# +# 2. readers TSan, N readers, NO writer. Hazard (a): the visited set used +# to live on the index, so two pure READS stamped each other's +# epoch. Fixed in engram_vindex.c (frame-owned VVisit + +# `const VIndex*` search). MUST be clean. Hard failure. +# +# 3. unsynchronized TSan, writer + reader on a BARE index. Hazard (b): in-place +# HNSW insert rewires existing elements' neighbour lists and +# reallocs elems[]. EXPECTED TO RACE, PERMANENTLY. This is not +# a bug to fix inside engram_vindex.c — it is the executable +# proof that a publication boundary must exist above it. +# Not a failure. If it ever goes CLEAN, the test stopped +# interleaving and half 4 is no longer meaningful either. +# +# 4. published TSan, owner + N readers through a publication boundary +# (rwlock: readers shared, owner exclusive) mirroring +# eg_vindex_view / eg_vindex_maintain in lang/runtime/el_runtime.c. +# MUST be clean, and all inserts must land. Hard failure. +# +# See test_vindex_concurrency.c for the full story (SIGSEGV at ASCII address +# "gramNode", heap corruption in xzm_realloc, etc). +# +# usage: run_vindex_concurrency_tests.sh +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUNTIME="$(cd "$HERE/../../lang/runtime" && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +SRC="$HERE/test_vindex_concurrency.c" +VINDEX="$RUNTIME/engram_vindex.c" + +fail=0 + +echo "== [1/4] single-threaded control under AddressSanitizer ==" +cc -std=c11 -g -O1 -fsanitize=address,undefined -fno-omit-frame-pointer \ + -I"$RUNTIME" -o "$WORK/single" "$SRC" "$VINDEX" -lm || { echo "BUILD FAILED"; exit 2; } +if ASAN_OPTIONS=detect_leaks=0 "$WORK/single" single; then + echo " -> OK" +else + echo " -> FAIL: the single-threaded control must always be clean." + echo " If this fails the bug is NOT (only) concurrency — look for a real" + echo " out-of-bounds or lifetime error in engram_vindex.c." + fail=1 +fi + +cc -std=c11 -g -O1 -fsanitize=thread -fno-omit-frame-pointer \ + -I"$RUNTIME" -o "$WORK/conc" "$SRC" "$VINDEX" -lm || { echo "BUILD FAILED"; exit 2; } + +# run_tsan ; echoes nothing, sets $tsan_raced +run_tsan() { + TSAN_OPTIONS="halt_on_error=0" "$WORK/conc" "$1" >"$2" 2>&1 + tsan_rc=$? + if grep -q "ThreadSanitizer: data race" "$2"; then tsan_raced=1; else tsan_raced=0; fi +} + +echo +echo "== [2/4] concurrent READERS, no writer (visited-set gate) ==" +run_tsan readers "$WORK/readers.log" +if [ "$tsan_raced" = "1" ]; then + echo " -> REGRESSION: two concurrent reads still race." + grep -m1 -A6 "ThreadSanitizer: data race" "$WORK/readers.log" | sed 's/^/ /' + echo " The visited set was supposed to be owned by the call frame." + fail=1 +else + echo " -> clean (concurrent reads are safe)" +fi + +echo +echo "== [3/4] writer+reader on a BARE index (expected-race probe) ==" +run_tsan unsynchronized "$WORK/unsync.log" +if [ "$tsan_raced" = "1" ]; then + echo " -> RACE DETECTED, as expected:" + grep -m1 -A4 "ThreadSanitizer: data race" "$WORK/unsync.log" | sed 's/^/ /' + echo " In-place HNSW insert mutates existing elements. Not fixable inside" + echo " engram_vindex.c — this is why the publication boundary exists." +else + echo " -> NOTE: no race reported. The probe did not interleave; half 4's" + echo " clean result proves less than it should. Investigate." +fi + +echo +echo "== [4/4] owner+readers through the publication boundary (boundary gate) ==" +run_tsan published "$WORK/pub.log" +if [ "$tsan_raced" = "1" ]; then + echo " -> REGRESSION: the publication boundary did not serialize the owner." + grep -m1 -A6 "ThreadSanitizer: data race" "$WORK/pub.log" | sed 's/^/ /' + fail=1 +elif [ "$tsan_rc" != "0" ]; then + echo " -> FAIL: boundary clean under TSan but the run failed:" + tail -3 "$WORK/pub.log" | sed 's/^/ /' + fail=1 +else + echo " -> clean (readers project concurrently; the owner's inserts all landed)" +fi + +echo +[ "$fail" -eq 0 ] && echo "RESULT: PASS" || echo "RESULT: FAIL" +exit "$fail" diff --git a/engram/test/run_wal_tests.sh b/engram/test/run_wal_tests.sh index eddd791..ae3b0cb 100755 --- a/engram/test/run_wal_tests.sh +++ b/engram/test/run_wal_tests.sh @@ -3,8 +3,17 @@ set -e HERE="$(cd "$(dirname "$0")" && pwd)" REL="$HERE/../../lang/runtime" +# test_wal.c and test_failloud.c #include "el_runtime.c" directly, so el_runtime.c +# is already IN the translation unit — link the SIBLINGS only, or every symbol in +# it is defined twice. The siblings are still required: el_runtime.c calls into +# all six engram TUs. (lang/runtime/SOURCES is the source of truth.) +RTSIB="$("$HERE/../../scripts/el-runtime-sources.sh" "$REL" | grep -v '/el_runtime\.c$')" +SSLFLAGS="" +if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then + SSLFLAGS="-I$O/include -L$O/lib" +fi cc -O2 -fbracket-depth=1024 -Wno-parentheses-equality -I"$REL" \ - "$HERE/test_wal.c" -lcurl -lpthread -o /tmp/test_wal + "$HERE/test_wal.c" $RTSIB $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/test_wal HOME=/tmp/engram-throwaway-home /tmp/test_wal # Fail-loud data-dir check (must exit 1 with a FATAL line): cat > /tmp/test_failloud.c <<'C' @@ -12,5 +21,5 @@ cat > /tmp/test_failloud.c <<'C' int main(void){ unsetenv("ENGRAM_DATA_DIR"); unsetenv("HOME"); engram_resolve_data_dir(); printf("REACHED\n"); return 0; } C -cc -O2 -fbracket-depth=1024 -Wno-parentheses-equality -I"$REL" /tmp/test_failloud.c -lcurl -lpthread -o /tmp/test_failloud +cc -O2 -fbracket-depth=1024 -Wno-parentheses-equality -I"$REL" /tmp/test_failloud.c $RTSIB $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/test_failloud if env -u HOME -u ENGRAM_DATA_DIR /tmp/test_failloud; then echo "FAIL: should have exited"; exit 1; else echo "[PASS] fail-loud exit on unresolvable HOME"; fi diff --git a/engram/test/test_grounding_vector.c b/engram/test/test_grounding_vector.c new file mode 100644 index 0000000..171cf22 --- /dev/null +++ b/engram/test/test_grounding_vector.c @@ -0,0 +1,176 @@ +/* test_grounding_vector.c — deterministic tests for §7: the one decay model, the + * consequence gate, and the stored/derived split. Links engram_cognition.c + * directly; no server, no store, no network. See run_grounding_vector_tests.sh. + * + * NEGATIVE CONTROL (invariant §8.6). Every symbol exercised here — + * cog_decay_factor, cog_grounding_significant, cog_significance_inherent, + * CogGrounding, CogProvClass — is introduced by the change under test, so this + * suite does not COMPILE against the pre-change source, let alone pass. The + * runner documents the exact reproduction. + */ +#include "engram_cognition.h" +#include +#include +#include +#include + +static int fails = 0; +static void ok(int cond, const char* what) { + printf(" %-62s %s\n", what, cond ? "PASS" : "*** FAIL ***"); + if (!cond) fails++; +} + +/* The decay formula exactly as el_runtime.c carried it before the move, so the + * refactor can be shown to be bit-identical rather than merely similar. */ +static double old_engram_temporal_decay(long long age_ms, long long activation_count, + double temporal_decay_rate) { + if (age_ms <= 0) return 1.0; + double lambda = (temporal_decay_rate > 0.0) ? temporal_decay_rate : 0.693147; + double age_hours = (double)age_ms / 3600000.0; + double t_half = 168.0 * (1.0 + log(1.0 + (double)activation_count)); + double factor = exp(-lambda * age_hours / t_half); + if (factor < 0.25) factor = 0.25; + return factor; +} + +static CogGrounding base(void) { + CogGrounding g; memset(&g, 0, sizeof g); + g.present = 1; + g.factual = 0.60; g.relational = 0.60; + g.factual_now = 0.60; g.relational_now = 0.60; + g.associative = 0.1; g.polarity = 1.0; + g.prov = COG_PROV_TOLD; + g.fac_proj = 1.0; g.rel_proj = 1.0; + g.cos_angle = 0.9; g.agreement = 1; + g.ts = 1000; g.seq = 1; g.reinforcements = 3; + return g; +} + +int main(void) { + const double F = 0.5, R = 0.5; + + printf("\n== 1. DECAY IS THE ONE MODEL, AND IT IS BIT-IDENTICAL TO WHAT IT REPLACED ==\n"); + { + long long ages[] = {0, 3600000LL, 86400000LL, 7*86400000LL, 30*86400000LL, 365*86400000LL}; + int allsame = 1; + for (int i = 0; i < 6; i++) + for (int ac = 0; ac < 4; ac++) { + long long acs[] = {0, 1, 10, 1000}; + double a = cog_decay_factor(ages[i], (double)acs[ac], 0.0); + double b = old_engram_temporal_decay(ages[i], acs[ac], 0.0); + if (a != b) allsame = 0; + } + ok(allsame, "cog_decay_factor == the pre-move engram_temporal_decay (24 pts)"); + ok(cog_decay_factor(0, 0, 0.0) == 1.0, "age 0 -> no decay"); + } + printf("\n DECAY OVER ELAPSED TIME (reinforcements = 0, default rate):\n"); + printf(" %10s %10s\n", "elapsed", "decay"); + { + struct { const char* label; long long ms; } pts[] = { + {"0", 0LL}, + {"1 hour", 3600000LL}, + {"1 day", 86400000LL}, + {"3 days", 3LL*86400000LL}, + {"7 days", 7LL*86400000LL}, + {"14 days", 14LL*86400000LL}, + {"30 days", 30LL*86400000LL}, + {"90 days", 90LL*86400000LL}, + }; + double prev = 2.0; int monotone = 1; + for (unsigned i = 0; i < sizeof pts / sizeof pts[0]; i++) { + double d = cog_decay_factor(pts[i].ms, 0, 0.0); + printf(" %10s %10.6f\n", pts[i].label, d); + if (d > prev) monotone = 0; + prev = d; + } + ok(monotone, "decay is monotone non-increasing in elapsed time"); + ok(fabs(cog_decay_factor(7LL*86400000LL, 0, 0.0) - 0.5) < 1e-6, + "7 days at zero reinforcements == exactly one half-life (0.5)"); + ok(cog_decay_factor(7LL*86400000LL, 100, 0.0) > cog_decay_factor(7LL*86400000LL, 0, 0.0), + "reinforcement slows ageing (Lindy term)"); + ok(cog_decay_factor(3650LL*86400000LL, 0, 0.0) == 0.25, + "floor is a preference not a cliff: bottoms out at 0.25"); + } + + printf("\n== 2. CONSEQUENCE GATE: EVERY TRIGGER, AND NO EPSILON ANYWHERE ==\n"); + { + CogGrounding p = base(), n = base(); + ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_NONE, + "identical vectors -> NONE (a re-read must not consolidate)"); + + n = base(); n.factual = 0.9999; n.factual_now = 0.9999; + ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_NONE, + "factual 0.60 -> 0.9999 without crossing the floor -> NONE"); + + n = base(); n.relational = 0.5001; n.relational_now = 0.5001; + ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_NONE, + "relational 0.60 -> 0.5001, still above floor -> NONE"); + + n = base(); n.factual_now = 0.4999; + ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_FACTUAL_FLOOR, + "a 0.1001 drop that CROSSES the floor -> FACTUAL_FLOOR"); + + n = base(); n.relational_now = 0.4999; + ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_RELATIONAL_FLOOR, + "relational crossing its floor -> RELATIONAL_FLOOR"); + + n = base(); n.cos_angle = -0.05; n.agreement = -1; + ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_AGREEMENT_FLIP, + "agreement +1 -> -1 -> AGREEMENT_FLIP"); + + n = base(); n.fac_proj = -0.2; + ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_DIRECTION_REVERSAL, + "factual gradient reverses -> DIRECTION_REVERSAL"); + + n = base(); n.rel_proj = -0.2; + ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_DIRECTION_REVERSAL, + "relational gradient reverses -> DIRECTION_REVERSAL"); + + n = base(); n.polarity = -1.0; + ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_POLARITY_FLIP, + "support -> contradiction -> POLARITY_FLIP (inherent)"); + + n = base(); n.polarity = 0.0; + ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_POLARITY_FLIP, + "support -> ignorance (zero) -> POLARITY_FLIP: not the same state"); + + n = base(); n.prov = COG_PROV_OBSERVED; + ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_PROVENANCE_CHANGE, + "told -> observed -> PROVENANCE_CHANGE (inherent)"); + + CogGrounding fresh; memset(&fresh, 0, sizeof fresh); + ok(cog_grounding_significant(&fresh, &n, F, R) == COG_SIG_FIRST_RECORD, + "no prior version -> FIRST_RECORD"); + } + + printf("\n== 3. INHERENT MOVES BYPASS THE SALIENCE GATE ==\n"); + ok(cog_significance_inherent(COG_SIG_POLARITY_FLIP), "polarity flip is inherent"); + ok(cog_significance_inherent(COG_SIG_PROVENANCE_CHANGE), "provenance change is inherent"); + ok(cog_significance_inherent(COG_SIG_FIRST_RECORD), "first record is inherent"); + ok(!cog_significance_inherent(COG_SIG_FACTUAL_FLOOR), "a floor crossing is NOT inherent"); + ok(!cog_significance_inherent(COG_SIG_NONE), "NONE is not inherent"); + + printf("\n== 4. THE STORED/DERIVED SPLIT: DERIVED VALUES ARE NEVER SERIALIZED ==\n"); + { + CogGrounding g = base(); + g.decay = 0.3333; g.factual_now = 0.1234; g.relational_now = 0.2345; + g.associative_now = 0.4567; g.age_ms = 999999; g.stale = 1; + char* m = cog_grounding_metadata("pre-existing=keepme", &g); + ok(m != NULL, "serializer returns a document"); + ok(m && strstr(m, "pre-existing=keepme"), "pre-existing edge metadata preserved verbatim"); + ok(m && strstr(m, "GRD1"), "GRD1 magic present"); + ok(m && !strstr(m, "0.3333"), "decay is NOT stored"); + ok(m && !strstr(m, "0.1234"), "factual_now is NOT stored"); + ok(m && !strstr(m, "0.2345"), "relational_now is NOT stored"); + ok(m && !strstr(m, "0.4567"), "associative_now is NOT stored"); + ok(m && !strstr(m, "999999"), "age is NOT stored"); + ok(m && strstr(m, "told"), "provenance class IS stored"); + ok(m && strstr(m, "0.6"), "the factual/relational dimensions ARE stored"); + if (m) { printf("\n --- serialized GRD1 block ---\n%s -----------------------------\n", m); } + free(m); + } + + printf("\n%s (%d failure%s)\n\n", fails ? "SOME TESTS FAILED" : "ALL TESTS PASSED", + fails, fails == 1 ? "" : "s"); + return fails ? 1 : 0; +} diff --git a/engram/test/test_vindex_concurrency.c b/engram/test/test_vindex_concurrency.c new file mode 100644 index 0000000..b181da6 --- /dev/null +++ b/engram/test/test_vindex_concurrency.c @@ -0,0 +1,251 @@ +/* test_vindex_concurrency.c — regression test for the 2026-08-16 soul crash. + * + * WHAT BROKE: the soul daemon crash-looped (5 crashes in ~100s) with SIGSEGV in + * search_layer <- vindex_insert <- eg_vindex_sync, a SIGABRT, and a fault inside + * xzm_realloc's own freelist — i.e. heap corruption. The SIGSEGV address + * 0x65646f4e6d617267 is little-endian ASCII "gramNode": string bytes being + * dereferenced as an Elem vector pointer. + * + * ROOT CAUSE: VIndex owns its traversal scratch (visited[] + visit_epoch), and + * search_layer mutates it via visited_reset(). So the index is unsafe for ANY + * concurrent use — including two concurrent READS. soul.el starts http_serve_async + * (a thread per connection) and then runs awareness_run() on the main thread, which + * reaches the same global index through engram_activate; nothing serialized them. + * + * Neither hnswlib nor FAISS puts the visited set on the index: hnswlib checks one + * out of a VisitedListPool per query, FAISS uses a thread_local VisitedTable. + * + * THE ORIGINAL `concurrent` HALF CONFLATED TWO DISTINCT HAZARDS (2026-08-16). It ran + * a writer against a reader on one bare index, so it could not tell apart: + * + * (a) READ/READ corruption — two searches stamping each other's visited epoch. + * A defect INSIDE engram_vindex.c, fixable there, and now fixed: the visited + * set moved to the call frame and vindex_search takes a `const VIndex*`. + * + * (b) WRITE/READ corruption — vindex_insert rewires the neighbour lists of + * EXISTING elements and reallocs elems[], so an insert is a mutation of the + * whole structure. This is NOT fixable inside engram_vindex.c at any price: + * it is inherent to in-place HNSW. It requires a publication boundary ABOVE + * the data structure (el_runtime.c: eg_vindex_view / eg_vindex_maintain). + * + * Conflating them made the suite unfailable-then-unpassable: fixing (a) left (b) + * still racing, which reads as "the fix did not work" when in fact a different, + * correctly-located fix is what (b) needs. So the halves are now separate: + * + * single N clustered vectors, ONE thread, ASan. The CONTROL. Must always + * be clean. When this passes and a concurrent half fails, the defect + * is concurrency, not an out-of-bounds/logic error in the graph code. + * (On 2026-08-16 this control cleared all 13,820 real dim-768 store + * vectors under ASan, which DISPROVED an inspection-derived hypothesis + * about an out-of-bounds reverse-link write at engram_vindex.c:340.) + * + * readers N reader threads, NO writer, one shared index, TSan. This is + * hazard (a) in isolation. It RACED before the visited set moved off + * the index struct and must be CLEAN now. Hard gate. + * + * unsynchronized writer + reader on a bare index, TSan. Hazard (b) in isolation. + * EXPECTED TO RACE, permanently — it is the executable proof that + * the index cannot be made safe from the inside, and therefore that + * the publication boundary in el_runtime.c has to exist. If this + * ever goes clean, the test stopped interleaving; do not celebrate. + * + * published writer + readers through a publication boundary that mirrors + * eg_vindex_view / eg_vindex_maintain (rwlock: readers shared, + * the single owner exclusive), TSan. Must be CLEAN. Hard gate. + * This is what proves the shape of the runtime fix, in the same + * process, rather than asserting it. + * + * Absence of a crash does NOT mean absence of a race — always read the sanitizer + * verdict, never just the exit code. + * + * Build/run: engram/test/run_vindex_concurrency_tests.sh + */ +#include "engram_vindex.h" + +#include +#include +#include +#include +#include + +#define DIM 128 +#define NVEC 3000 +#define SEED_N 50 + +static VIndex* g_ix; +static float* g_vecs; + +/* Deterministic filler. Real embeddings are strongly correlated, not uniform noise; + * clustering keeps many candidates near-equidistant, which exercises the diversity + * heuristic and the visited set far harder than random vectors do. */ +static void fill_vectors(void) { + g_vecs = (float*)malloc((size_t)NVEC * DIM * sizeof(float)); + if (!g_vecs) { fprintf(stderr, "OOM\n"); exit(1); } + for (int i = 0; i < NVEC; i++) { + int cluster = i % 8; + for (int d = 0; d < DIM; d++) + g_vecs[(size_t)i * DIM + d] = + (float)(((d + cluster * 7) % 13) / 13.0) + + (float)(((i * 2654435761u + (unsigned)d) % 97) / 9700.0); + } +} + +static void* writer_fn(void* arg) { + (void)arg; + for (int i = SEED_N; i < NVEC; i++) + (void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM); + return NULL; +} + +static void* reader_fn(void* arg) { + (void)arg; + uint64_t ids[8]; float ds[8]; + for (int i = 0; i < 20000; i++) + (void)vindex_search(g_ix, g_vecs + (size_t)(i % NVEC) * DIM, 8, 0, ids, ds); + return NULL; +} + +static int run_single(void) { + printf("[single] inserting %d vectors on one thread (ASan control)\n", NVEC); + g_ix = vindex_create(DIM, 0, 0); + if (!g_ix) { fprintf(stderr, "[single] vindex_create failed\n"); return 1; } + for (int i = 0; i < NVEC; i++) { + if (vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM) != 0) { + fprintf(stderr, "[single] insert %d failed\n", i); return 1; + } + } + if (vindex_size(g_ix) != (size_t)NVEC) { + fprintf(stderr, "[single] size %zu != %d\n", vindex_size(g_ix), NVEC); return 1; + } + uint64_t ids[16]; float ds[16]; + for (int q = 0; q < 200; q++) { + int k = vindex_search(g_ix, g_vecs + (size_t)((q * 7) % NVEC) * DIM, 16, 0, ids, ds); + if (k < 0) { fprintf(stderr, "[single] search failed at q=%d\n", q); return 1; } + } + vindex_free(g_ix); g_ix = NULL; + printf("[single] PASS — no memory error (this must ALWAYS pass)\n"); + return 0; +} + +/* Hazard (b) in isolation: writer + reader on a BARE index, no boundary. */ +static int run_unsynchronized(void) { + printf("[unsynchronized] 1 writer + 1 reader on a BARE index (TSan probe)\n"); + printf("[unsynchronized] a race here is EXPECTED and PERMANENT — in-place HNSW\n"); + printf("[unsynchronized] insert rewires existing elements. This is the proof that\n"); + printf("[unsynchronized] the publication boundary must live ABOVE engram_vindex.c.\n"); + g_ix = vindex_create(DIM, 0, 0); + if (!g_ix) { fprintf(stderr, "[unsynchronized] vindex_create failed\n"); return 1; } + for (int i = 0; i < SEED_N; i++) + (void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM); + + pthread_t w, r; + if (pthread_create(&w, NULL, writer_fn, NULL) || + pthread_create(&r, NULL, reader_fn, NULL)) { + fprintf(stderr, "[unsynchronized] pthread_create failed\n"); return 1; + } + pthread_join(w, NULL); + pthread_join(r, NULL); + vindex_free(g_ix); g_ix = NULL; + printf("[unsynchronized] completed — CHECK THE SANITIZER VERDICT, not this line.\n"); + return 0; +} + +/* ── hazard (a) in isolation: concurrent READS only ─────────────────────────── + * This is what the frame-owned visited set fixes. Before that change, two + * vindex_search calls on one index wrote each other's epoch stamp; TSan reported + * the race at visited_reset and the traversal then walked bogus element indices. */ +#define NREADERS 4 + +static int run_readers(void) { + printf("[readers] %d concurrent readers, NO writer, one shared index (TSan)\n", NREADERS); + printf("[readers] this is the visited-set regression gate — must be CLEAN.\n"); + g_ix = vindex_create(DIM, 0, 0); + if (!g_ix) { fprintf(stderr, "[readers] vindex_create failed\n"); return 1; } + for (int i = 0; i < NVEC; i++) + (void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM); + + pthread_t t[NREADERS]; + for (int i = 0; i < NREADERS; i++) + if (pthread_create(&t[i], NULL, reader_fn, NULL)) { + fprintf(stderr, "[readers] pthread_create failed\n"); return 1; + } + for (int i = 0; i < NREADERS; i++) pthread_join(t[i], NULL); + vindex_free(g_ix); g_ix = NULL; + printf("[readers] completed — CHECK THE SANITIZER VERDICT, not this line.\n"); + return 0; +} + +/* ── the publication boundary, mirroring el_runtime.c ───────────────────────── + * Readers take the boundary SHARED and hold it across the whole search; the one + * owner takes it EXCLUSIVE to extend. Same shape as eg_vindex_view / + * eg_vindex_maintain. Note the reader's index pointer is `const VIndex*` — the + * compiler, not this comment, is what stops a reader inserting. */ +static pthread_rwlock_t g_pub = PTHREAD_RWLOCK_INITIALIZER; + +static void* pub_writer_fn(void* arg) { + (void)arg; + for (int i = SEED_N; i < NVEC; i++) { + pthread_rwlock_wrlock(&g_pub); + (void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM); + pthread_rwlock_unlock(&g_pub); + } + return NULL; +} + +static void* pub_reader_fn(void* arg) { + (void)arg; + uint64_t ids[8]; float ds[8]; + for (int i = 0; i < 5000; i++) { + pthread_rwlock_rdlock(&g_pub); + const VIndex* view = g_ix; /* immutable view */ + (void)vindex_search(view, g_vecs + (size_t)(i % NVEC) * DIM, 8, 0, ids, ds); + pthread_rwlock_unlock(&g_pub); + } + return NULL; +} + +static int run_published(void) { + printf("[published] 1 owner + %d readers through a publication boundary (TSan)\n", NREADERS); + printf("[published] this is the eg_vindex_view/eg_vindex_maintain gate — must be CLEAN.\n"); + g_ix = vindex_create(DIM, 0, 0); + if (!g_ix) { fprintf(stderr, "[published] vindex_create failed\n"); return 1; } + for (int i = 0; i < SEED_N; i++) + (void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM); + + pthread_t w, r[NREADERS]; + if (pthread_create(&w, NULL, pub_writer_fn, NULL)) { + fprintf(stderr, "[published] pthread_create failed\n"); return 1; + } + for (int i = 0; i < NREADERS; i++) + if (pthread_create(&r[i], NULL, pub_reader_fn, NULL)) { + fprintf(stderr, "[published] pthread_create failed\n"); return 1; + } + pthread_join(w, NULL); + for (int i = 0; i < NREADERS; i++) pthread_join(r[i], NULL); + if (vindex_size(g_ix) != (size_t)NVEC) { + fprintf(stderr, "[published] size %zu != %d — the owner lost inserts\n", + vindex_size(g_ix), NVEC); + vindex_free(g_ix); g_ix = NULL; return 1; + } + vindex_free(g_ix); g_ix = NULL; + printf("[published] all %d inserts landed; CHECK THE SANITIZER VERDICT too.\n", NVEC); + return 0; +} + +int main(int argc, char** argv) { + const char* mode = (argc > 1) ? argv[1] : "single"; + fill_vectors(); + int rc; + if (!strcmp(mode, "single")) rc = run_single(); + else if (!strcmp(mode, "readers")) rc = run_readers(); + else if (!strcmp(mode, "unsynchronized")) rc = run_unsynchronized(); + else if (!strcmp(mode, "published")) rc = run_published(); + /* back-compat: the pre-split name meant the bare writer+reader probe. */ + else if (!strcmp(mode, "concurrent")) rc = run_unsynchronized(); + else { + fprintf(stderr, "usage: %s [single|readers|unsynchronized|published]\n", argv[0]); + rc = 2; + } + free(g_vecs); + return rc; +} diff --git a/ingest/src/ingest.el b/ingest/src/ingest.el index 1d47432..c143e8c 100644 --- a/ingest/src/ingest.el +++ b/ingest/src/ingest.el @@ -13,7 +13,7 @@ // relations add edges. Every node enters with PROVENANCE + grounding-level // + stewardship class from the moment of entry. // -// transduce() is THE single mechanism — one function, polymorphic, with no +// transduce_bytes() is THE single mechanism — one function, polymorphic, with no // content-type branch inside it. It does not ask whether a payload is // prose, structured data, or raw/opaque bytes (audio, or anything else); // it runs one boundary-scan-with-fixed-window-fallback chunking algorithm @@ -401,10 +401,54 @@ fn head80(s: String) -> String { // truncates at the first embedded NUL, which is routine in real binary // bytes) is a MECHANICAL fidelity concern that belongs to whatever produced // `source` (see ingest_file's file_source_string below) — not a -// content-type judgment made in here. transduce() never learns whether a +// content-type judgment made in here. transduce_bytes() never learns whether a // chunk is plain text or a base64-encoded raw-byte window; every chunk is // handled identically either way. -fn transduce(nodes: [String], edges: [String], source: String, +// NAMING, CORRECTED 2026-08-16 (second pass). This function was renamed +// `transduce` -> `transduce_bytes` earlier the same day, on the reasoning +// that it "was never signal->geometry — it chunks already-extracted content +// and PACKS it into a node+edge manifold, one layer up, and it had taken the +// name that belongs to the primitive underneath it." +// +// THAT REASONING WAS BACKWARDS, and it is worth recording why rather than +// quietly re-renaming. Producing a node+edge manifold is not a layer above +// transduction — it IS transduction. Transduction is not conversion. When you +// take in music you do not store the song as one discrete geometry; you break +// it into its component parts and store the geometry of each along with the +// relations between them. The song is the structure of those relations. +// Signal -> one vector is the operation UNDERNEATH transduction, and its name +// is encoding, or geometry. So the layer that was doing it right got renamed +// out of the way so the layer doing it wrong could have the name. +// +// The primitive has since been corrected: `transduce(signal, modality)` now +// returns a Manifold — components plus relations — not a Geometry +// (el_runtime.c, "Manifold"). The two layers are therefore doing the SAME KIND +// of thing, and the inversion dissolves rather than needing to be re-argued. +// +// What is left is a real distinction, and it is about MODALITY, not layering: +// +// * `transduce(signal, modality)` dispatches to a realizer that KNOWS the +// modality and can name its components — for audio: pitch, interval, +// rhythm, harmonic function. +// * `transduce_bytes` below is the OPAQUE-BYTES realizer: the decomposition +// available to a reader that knows nothing about what it is reading. It +// still yields components and relations (chunk nodes; contains / precedes +// / section_of edges), which is why it is transduction and not packing. It +// just cuts on the only structure visible without understanding — byte +// boundaries — so its components are positional rather than meaningful. +// That is a LIMITATION of this realizer, not the definition of the +// operation. +// +// The name is suffixed by its modality, not demoted to a lesser layer. Keeping +// a distinct symbol is also still mechanically required: every El `fn name` +// compiles to a global C symbol, so reusing `transduce` here is a hard +// `conflicting types` error the moment ingest.c links el_runtime.c. +// +// WHERE THIS SHOULD GO: this function should become a registered realizer +// returning a real Manifold, so ingest rides the same primitive as every other +// modality instead of carrying a parallel implementation. Not done here. +// Nothing about this function's behaviour changed in this pass. +fn transduce_bytes(nodes: [String], edges: [String], source: String, prov: String, ground: String, steward: String, root_lid: String, root_title: String) -> [String] { let tagbase: String = "prov:" + prov + " ground:" + ground + " steward:" + steward @@ -531,8 +575,8 @@ fn default_steward() -> String { // trustworthy verbatim. When they don't (silent truncation happened), // rebuild the payload as base64-encoded fixed-size windows read directly // off disk (fs_read_b64_chunk — binary-safe in C), joined with the same -// "\n\n" boundary marker transduce()'s generic scan already looks for, so -// transduce() sees one ordinary boundary-delimited payload and runs its one +// "\n\n" boundary marker transduce_bytes()'s generic scan already looks for, so +// transduce_bytes() sees one ordinary boundary-delimited payload and runs its one // algorithm on it exactly as it would on prose — it never learns that a // fidelity problem occurred upstream, let alone why. fn file_source_string(path: String, text: String, real_size: Int) -> String { @@ -541,7 +585,7 @@ fn file_source_string(path: String, text: String, real_size: Int) -> String { // 3072 raw bytes -> 4096 base64 chars (3 divides evenly into base64's // 3-byte/4-char ratio); keeps each resulting node's content a clean, // bounded, low-kilobytes unit, same order of magnitude as the fixed - // fallback window in transduce() itself. + // fallback window in transduce_bytes() itself. let win: Int = 3072 let out: String = "" let off: Int = 0 @@ -561,7 +605,7 @@ fn file_source_string(path: String, text: String, real_size: Int) -> String { } // ingest one file -> report JSON. Uniform for every file regardless of -// extension or content — transduce() decides nothing about content-type, so +// extension or content — transduce_bytes() decides nothing about content-type, so // neither does this function; it only decides whether the raw bytes made it // through the read intact (file_source_string), which is a fidelity // question, not a format one. @@ -573,14 +617,14 @@ fn ingest_file(path: String) -> String { return "{\"error\":\"empty or unreadable\",\"path\":" + j_q(path) + "}" } let prov: String = "file:" + path - let packed: [String] = transduce(el_list_empty(), el_list_empty(), + let packed: [String] = transduce_bytes(el_list_empty(), el_list_empty(), source, prov, default_ground(), default_steward(), "doc:" + basename(path), basename(path)) return merge_packed(packed) } // ingest a directory: walk one level, ingest every file found, aggregate. -// No extension filter — transduce() handles any payload uniformly now, so +// No extension filter — transduce_bytes() handles any payload uniformly now, so // there is no content-type gate at the directory boundary either. fn ingest_dir(path: String) -> String { let entries: [String] = fs_list(path) @@ -615,7 +659,7 @@ fn ingest_dir(path: String) -> String { fn ingest_url(url: String) -> String { let body: String = http_get(url) if str_eq(body, "") { return "{\"error\":\"empty fetch\",\"url\":" + j_q(url) + "}" } - let packed: [String] = transduce(el_list_empty(), el_list_empty(), + let packed: [String] = transduce_bytes(el_list_empty(), el_list_empty(), body, "url:" + url, "extracted", "public-web", "url:" + url, url) return merge_packed(packed) @@ -630,7 +674,7 @@ fn ingest_llm(query: String) -> String { let resp: String = http_post_json("http://127.0.0.1:11434/api/generate", body) let answer: String = json_get_string(resp, "response") if str_eq(answer, "") { return "{\"error\":\"no model response\"}" } - let packed: [String] = transduce(el_list_empty(), el_list_empty(), + let packed: [String] = transduce_bytes(el_list_empty(), el_list_empty(), answer, "llm:" + model + ":" + query, "candidate-provisional", "guide-provisional", "llm:" + query, "guide answer: " + query) return merge_packed(packed) @@ -682,7 +726,7 @@ fn ingest_stream(path: String) -> String { // It is NOT a content-type flag: it says nothing about what's inside the // bytes once fetched, and none of the five ingest_* functions it selects // among interpret their payload differently by content shape anymore — -// they all hand off to the single, format-agnostic transduce(). The old +// they all hand off to the single, format-agnostic transduce_bytes(). The old // "structured" value (a caller-declared alias for "file", used only to hint // the now-removed JSON-vs-prose branch) is gone along with that branch. let kind: String = env("INGEST_KIND") diff --git a/lang/AGENTS.md b/lang/AGENTS.md index 95d739d..0bbee99 100644 --- a/lang/AGENTS.md +++ b/lang/AGENTS.md @@ -18,8 +18,23 @@ night) and `02-components.md §5`. `relate`, `supersede` (evolve/tombstone/promote, never a hard delete) — plus the agentic primitives `think`/`attend`/`learn`/`ground`/`assert`. The old noun is a `type` parameter. Implemented in `tools/api-reshape/surface.el` with a parity - harness (`parity.sh`); aperture proven to bound output. **Not yet:** compiled - into the MCP server, hot-swap, all-alias dispatch. + harness (`parity.sh`); aperture proven to bound output. ~~**Not yet:** compiled + into the MCP server~~ — **shipped (verified 2026-08-16): the live MCP surface is + exactly these nine ops** (`read` · `write` · `relate` · `supersede` · `think` · + `attend` · `assert` · `ground` · `learn`); the ~87-tool surface is gone. + `attend` absorbed `getInstructions` / `beginSession`'s active-context sweep / + `checkEvents` — those are **gone, not gapped**. Still outstanding: hot-swap, + all-alias dispatch. + + > **⚠ Two of those primitives are the wrong shape, and it is documented + > (2026-08-16).** `think({seeds, faculty})` treats **faculties as parameters**; + > they are **operations** — `reason` changes the estimate (a read), `induce` + > changes the parameters, `abduce` changes the *structure* (a write + > `GeoGradient` cannot express). And `ground` mints a `grounded-by` edge, but + > **grounding is not a subsystem — it IS the edge weight**: a property *of* a + > relation, not a relation *between* nodes. Authority: + > `lang/spec/correspondence-and-censorship.md`. Do not re-derive it; if you think + > a section is wrong, say so with a measurement. - **Decorated seam.** `@route(path,method,…)` makes codegen synthesize `el_route_dispatch` (replacing the hand-written `handle_request` if-else) — proven decorate→serve on `:8951`. `@manager`/`@engine`/`@accessor` are **parsed @@ -62,34 +77,78 @@ This is where almost all work belongs. El programs are source files that get com This is the self-contained C OS-boundary layer. It provides the `__`-prefixed primitives that compiled El programs call: libcurl HTTP, pthreads, filesystem I/O, arena allocation, etc. It is **not generated** — it is maintained by hand. -The old `el_runtime.c` has been archived to `runtime/legacy/`. The runtime is now native El (`runtime/*.el`). `el_seed.c` replaces `el_runtime.c` as the sole C compilation dependency. +The runtime is native El (`runtime/*.el`) over a C OS-boundary. **Status (verified 2026-08-16):** the migration to a seed-only boundary is *in progress, not done*. -**Only edit `el_seed.c` when you genuinely need OS-level access** (raw sockets, GPU calls, new libcurl features). For everything else, write El. +**The runtime is MULTI-FILE. There is no single-file link target and there has not been one for months.** The canonical link set is listed once, in **`runtime/SOURCES`**, and printed by `scripts/el-runtime-sources.sh`. It currently holds ten translation units: `el_runtime.c`, `el_seed.c`, the six `engram_*.c` concern files, and `eg_cosine_batch{,_strategy_cpu}.c`. -When you do add a C builtin: -1. Add the C function to `el_seed.c` -2. Declare it in `el_seed.h` -3. Add it to the `builtin_arity` table in `el-compiler/src/codegen.el` (so the compiler knows the arg count) -4. Rebuild the elc binary (see below) +- `runtime/el_runtime.c` (~940 KB, 20.5k lines) — **LIVE, and oversized.** It began life on 2026-05-03 as a temporary build shim: it was deleted that afternoon ("runtime is 100% native El") and restored 25 minutes later, explicitly "UNTIL the compiler is updated to emit `#include el_seed.h`". That `until` never arrived, and in the 3.5 months since, the file doubled. **It is not a volatility unit — it is a dumping ground.** ~47.5% of it is engram code that belongs in the six sibling files that already exist. Do not add to it. See "Where a new C builtin goes" below. +- `runtime/el_seed.c` — the intended hand-maintained `__`-prefixed seed (thin wrappers over the above). +- `runtime/engram_{store,vindex,geometry,reason,verify,cognition}.c` — the engram concerns, each with its own header. `el_runtime.c` `#include`s all six headers and makes hard cross-TU calls into all six. + +> **Linking `el_runtime.c` alone does not work and has not for months.** It fails at `ld` with undefined symbols (`engram_ground_json`, `engram_activate_inner`, `eg_find_relation`, `cog_assert_two_axis`, …). Any recipe, script, or CI step that names `el_runtime.c` by itself is stale — replace it with `$(scripts/el-runtime-sources.sh lang/runtime)`. + +**Only edit these when you genuinely need OS-level access** (raw sockets, GPU calls, new libcurl features, a new engram store op). For everything else, write El. + +#### Where a new C builtin goes + +**Put it in the `.c` that owns the concern — NOT in `el_runtime.c`.** + +*Placement is a link-time concern. The compiler cannot tell which `.c` a symbol came from, and never could.* `builtin_arity` in `el-compiler/src/codegen.el` maps NAME → ARITY INT and nothing else (~413 entries); the El name is emitted as the exact C symbol and resolved by `ld`. Proof, if you want it: `nm lang/dist/platform/elc` on the *shipped* compiler shows `T _engram_geo_reify_index_new` (defined in `engram_geometry.c`), `T _vindex_insert` (`engram_vindex.c`), `T _engram_think` (`engram_cognition.c`), `T _engram_reason_abduce` (`engram_reason.c`). **The shipped compiler is already linked from ten translation units.** A builtin defined in a sibling `.c` is exactly as linkable as one defined in `el_runtime.c`. + +Choose the file by concern: engram store ops → `engram_store.c`; index → `engram_vindex.c`; geometry/priming → `engram_geometry.c`; reasoning → `engram_reason.c`; grounding/consistency → `engram_verify.c`; think/stance → `engram_cognition.c`. **If no existing file owns it, create one** — add the `.c` to `runtime/SOURCES` (one line) and every build path picks it up. For a builtin that belongs to a downstream program rather than the runtime, declare `c_source "path/to/file.c"` in that program's `manifest.el`; `elb` already links it (`parse_manifest_c_sources`, `lang/elb.el:82`). + +> **`el_runtime.c` is on a ratchet and will reject your commit.** `runtime/BUDGET` caps it at its current line count *with no headroom*, and separately caps the number of `engram_*`/`eg_*`/`cog_*` function definitions in it. `scripts/check-runtime-growth.sh` enforces both in CI and in `.githooks/pre-commit`. **The numbers may only ever go down — do not raise them.** Every other runtime file is deliberately uncapped, because that is where the code is supposed to go. When you move code *out*, lower the numbers in the same commit; the guard tells you the new values. + +When you add a C builtin (verbatim-emit recipe — the El name is emitted as the exact C symbol; `builtin_arity` is an arity guard only, not a dispatch table): +1. Implement the C function in the **concern-owning `.c`** (and declare it in that file's `.h`). Add the file to `runtime/SOURCES` if it is new. Only put it in `el_runtime.c` if it is genuinely EL core (val/str/map/list/arena) — that is ~8% of what is in there today. +2. Add a `__`-prefixed thin wrapper in `el_seed.c` and declare it in `el_seed.h`. +3. Add the name to `builtin_arity` in `el-compiler/src/codegen.el` — add **both** the plain and `__`-prefixed spellings. +4. Rebuild the elc binary (see below) and confirm the self-host fixpoint is byte-identical. +5. **Prove it with a NEGATIVE CONTROL.** Show the test FAILING on a build without your change, then passing with it. A test that has never been seen to fail has proven nothing. + +> **Step 5 is not optional, and step 4 does not cover it.** The fixpoint proves the *compiler reproduces itself*. It says nothing whatsoever about whether your builtin works. A recipe ending at "byte-identical" reads as complete while having verified nothing about the thing just added — which is why this file, until 2026-08-16, produced builtins with no tests at all. +> +> Measured cost of the omission (2026-08-16): `engram_node_set_emb`, `engram_curiosity_json` and `dream_set_handler` were all added in one session with zero tests. Separately, a UTF-8 fix was written, tested, and **the test passed on the unpatched build too** — the defect was elsewhere entirely, and only building the pre-fix binary exposed it. Without a negative control that fix would have merged as verified. +> +> Two shapes that pass while proving nothing, both hit the same day: +> - A test that never exercises your change (the route supplied a default that bypassed the code under test). +> - An induction that loses a race. `curl --max-time` on a large response left *both* builds alive; only `SO_LINGER 0` — a genuine RST, so the peer is provably gone — reproduced the failure. Six of ten attempts is not a control. +> +> Before every probe, confirm **your** process bound the port (`lsof -nP -iTCP:`, match the PID). A stale instance answering on the port has silently produced false results here more than once, and `pkill -f` does not reliably match an argv like `./engram`. + +Worked example: the `engram_assert_json` (op_assert seam) and `engram_node_full_in`/`engram_connect_in` (purview write-side) primitives added 2026-08-15 follow exactly this recipe. --- ## Rebuilding the Compiler -After changing any `.el` source in `el-compiler/src/`: +After changing any `.el` source in `el-compiler/src/` (run from the `lang/` dir): ```bash -cd /Users/will/Development/neuron-technologies/foundation/el +# 1. Stage2: current elc compiles the (modified) compiler to C ./dist/platform/elc elc-cli.el > elc-new.c -cc -std=c11 -I runtime -lcurl -lpthread \ +# 2. Build the new compiler. Link the WHOLE runtime set, not el_runtime.c alone: +# el_runtime.c calls into engram_store / engram_vindex / eg_cosine_batch and +# wraps el_seed.c, so a one-file link fails at `ld` with undefined symbols +# (verified 2026-08-16 — the previous single-file line in this doc is stale). +cc -std=c11 -O2 -I runtime -I$(brew --prefix openssl@3)/include \ + -L$(brew --prefix openssl@3)/lib \ -o dist/platform/elc-new \ - elc-new.c runtime/el_seed.c -# Verify self-hosting: + elc-new.c runtime/el_runtime.c runtime/el_seed.c \ + runtime/engram_cognition.c runtime/engram_geometry.c runtime/engram_reason.c \ + runtime/engram_store.c runtime/engram_verify.c runtime/engram_vindex.c \ + runtime/eg_cosine_batch.c runtime/eg_cosine_batch_strategy_cpu.c \ + -lcurl -lssl -lcrypto -lpthread -lm +# 3. Verify self-hosting FIXPOINT (stage3 == stage2 output, byte-identical): ./dist/platform/elc-new elc-cli.el > elc-verify.c -diff elc-new.c elc-verify.c # should be identical +diff elc-new.c elc-verify.c # must be identical mv dist/platform/elc-new dist/platform/elc ``` +> **Build-path caveat (verified 2026-08-15).** `el_seed.c` is the intended hand-maintained OS-boundary seed, but it does **not** compile standalone under modern clang: it wraps ~16 unprefixed `el_runtime.c` symbols (`http_serve`, `json_*`, `state_*`, `http_response`) without prototypes, and clang treats implicit declarations as errors (C99+). The productionised install (`tools/install.sh`) builds `libel.a` from **both** `el_seed.o` + `el_runtime.o` together, which is why linking succeeds there. To make `el_seed.c` build on its own, add prototypes for those symbols (or `#include "el_runtime.h"`, reconciling the `__http_serve` return-type mismatch first). +> +> **There is no single-file link target.** *(Corrected 2026-08-16 — this paragraph previously ended "`el_runtime.c` is the authoritative single-file link target for the compiler". Measured: that is false. Linking `elc-new.c` against `runtime/el_runtime.c` alone fails at `ld` with undefined `engram_ground_json`, `engram_activate_inner`, `eg_find_relation`, `cog_assert_two_axis`, and others, because `el_runtime.c` `#include`s six engram headers and calls into all six sibling `.c` files.)* Link the set in `runtime/SOURCES` via `$(../scripts/el-runtime-sources.sh runtime)`. + After changing `el_seed.c` only (no El source changes), rebuild downstream programs but do NOT need to rebuild the compiler binary itself — the seed is linked at the application level, not the compiler level. --- diff --git a/lang/dist/platform/elc b/lang/dist/platform/elc index 01cbdcd..738526d 100755 Binary files a/lang/dist/platform/elc and b/lang/dist/platform/elc differ diff --git a/lang/el-compiler/src/codegen.el b/lang/el-compiler/src/codegen.el index d13410c..be628f5 100644 --- a/lang/el-compiler/src/codegen.el +++ b/lang/el-compiler/src/codegen.el @@ -589,7 +589,7 @@ fn cg_expr(expr: Map) -> String { if left_is_ct { if op == "Plus" { if right_is_ct { - time_record_violation("caltime_plus_caltime", "CalendarTime + CalendarTime is not allowed (use cal_to_instant + Duration)") + record_call(state_get("__cg_current_fn"), "temporal:caltime_plus_caltime") return "0 /* TIME_TYPE_ERROR: CalendarTime + CalendarTime */" } } @@ -607,9 +607,18 @@ fn cg_expr(expr: Map) -> String { return "el_instant_add_dur(" + left_c + ", " + right_c + ")" } if right_is_inst { - time_record_violation("instant_plus_instant", "Instant + Instant is not allowed") + record_call(state_get("__cg_current_fn"), "temporal:instant_plus_instant") return "0 /* TIME_TYPE_ERROR: Instant + Instant */" } + // A bare literal is a MAGNITUDE WITH NO AXIS. Duration + Int + // was already refused because an Int carries no unit; adding + // one to a POINT is worse, because it silently moves the + // instant by an unspecified amount. The asymmetry had no + // justification -- it was simply never written. + if is_int_expr(right) { + record_call(state_get("__cg_current_fn"), "temporal:instant_plus_int") + return "0 /* TIME_TYPE_ERROR: Instant + Int */" + } } if left_is_dur { if right_is_inst { @@ -619,13 +628,13 @@ fn cg_expr(expr: Map) -> String { return "el_duration_add(" + left_c + ", " + right_c + ")" } if is_int_expr(right) { - time_record_violation("duration_plus_int", "Duration + Int is not allowed (use duration_seconds(n) or N.seconds)") + record_call(state_get("__cg_current_fn"), "temporal:duration_plus_int") return "0 /* TIME_TYPE_ERROR: Duration + Int */" } } if right_is_dur { if is_int_expr(left) { - time_record_violation("duration_plus_int", "Int + Duration is not allowed") + record_call(state_get("__cg_current_fn"), "temporal:duration_plus_int") return "0 /* TIME_TYPE_ERROR: Int + Duration */" } } @@ -644,7 +653,7 @@ fn cg_expr(expr: Map) -> String { return "el_duration_sub(" + left_c + ", " + right_c + ")" } if is_int_expr(right) { - time_record_violation("duration_minus_int", "Duration - Int is not allowed") + record_call(state_get("__cg_current_fn"), "temporal:duration_minus_int") return "0 /* TIME_TYPE_ERROR: Duration - Int */" } } @@ -673,14 +682,14 @@ fn cg_expr(expr: Map) -> String { if left_is_inst { if right_is_inst { return "el_instant_lt(" + left_c + ", " + right_c + ")" } if right_is_dur { - time_record_violation("instant_cmp_duration", "Instant < Duration is not allowed") + record_call(state_get("__cg_current_fn"), "temporal:instant_cmp_duration") return "0 /* TIME_TYPE_ERROR: Instant < Duration */" } } if left_is_dur { if right_is_dur { return "el_duration_lt(" + left_c + ", " + right_c + ")" } if right_is_inst { - time_record_violation("duration_cmp_instant", "Duration < Instant is not allowed") + record_call(state_get("__cg_current_fn"), "temporal:duration_cmp_instant") return "0 /* TIME_TYPE_ERROR: Duration < Instant */" } } @@ -862,10 +871,23 @@ fn cg_expr(expr: Map) -> String { // arithmetic BinOp (or vice-versa). Without this check the // fallthrough to str_eq produces str_eq(int_value, int_value) // which reads the integer as a char* and segfaults. + // EITHER side provably Int is enough. Requiring BOTH meant a call + // whose return type codegen cannot infer poisoned the operator: + // getint(5) == a -> str_eq(getint(5), a) + // even with `a` declared Int. str_eq then reads an integer as a + // char* and segfaults. Only an integer LITERAL on one side forced + // the numeric form, so the bug was invisible in the common case. + // + // Loosening to OR is strictly safer: when one side is a known Int, + // str_eq is always wrong (it dereferences that int), while numeric + // comparison is at worst a wrong answer on an already ill-typed + // program. When neither side is Int nothing changes, so string + // comparison is untouched. if is_int_expr(left) { - if is_int_expr(right) { - return "(" + left_c + " == " + right_c + ")" - } + return "(" + left_c + " == " + right_c + ")" + } + if is_int_expr(right) { + return "(" + left_c + " == " + right_c + ")" } // Float literal or negative float literal: use plain == (bit-equal // el_val_t comparison). This handles `r0 == 3.0`, `neg == -3.0`, etc. @@ -921,10 +943,12 @@ fn cg_expr(expr: Map) -> String { } // Same mixed Ident/BinOp fix as EqEq: use is_int_expr to detect // integer-typed operands before falling through to !str_eq. + // Either side Int is enough — see the EqEq note above. if is_int_expr(left) { - if is_int_expr(right) { - return "(" + left_c + " != " + right_c + ")" - } + return "(" + left_c + " != " + right_c + ")" + } + if is_int_expr(right) { + return "(" + left_c + " != " + right_c + ")" } // Float-typed operands use plain != (bit-equal comparison). if is_float_expr(left) { @@ -977,18 +1001,25 @@ fn cg_expr(expr: Map) -> String { if func_kind == "Ident" { let fn_name: String = func["name"] - // Capability-kind enforcement: services can't call - // self-formation primitives; utilities can't call any - // DHARMA or LLM primitives. cap_check_call records - // violations to be emitted as #error directives at the - // top of the generated C, so cc fails with a clear msg. - cap_check_call(fn_name) + // Every call is recorded here, from cg_expr, because this runs for + // EVERY expression in every context -- including main's body, which + // cg_fn returns early on since C provides its own main. Hooking the + // recording to cg_fn instead left every call in main invisible to + // the relation graph, a blind spot exactly where a program does its + // work. Caught by capability_query.sh, not by reading the code. + record_call("program", "is_kind:" + state_get("__program_kind")) + record_call(state_get("__cg_current_fn"), fn_name) + // Record the ARITY at the call site. The compiler knows how many + // arguments were written; whether that is correct is a question + // about the runtime's surface, and the runtime already declares its + // surface in el_runtime.h. A table inside the emitter is a second + // copy of that header, maintained by hand. + record_call(fn_name, "arity:" + native_int_to_str(native_list_len(args))) // Arity check against the builtin table - refuse, with a clear // El-source message, when a known builtin gets the wrong arg // count (e.g. `http_serve(port)` instead of `http_serve(port, // handler)`). User-defined fns and variadic builtins pass // through (builtin_arity returns -1). - arity_check_call(fn_name, arity) // sleep(Duration) - Phase 1 of the typed-time work. When the // single arg is provably a Duration we lower to el_sleep_duration // so the runtime sees nanos directly. Existing sleep() callers @@ -1495,6 +1526,52 @@ fn cg_stmt(stmt: Map, indent: String, declared: [String]) -> [Strin if str_eq(ltype, "Int") { add_int_name(name) } + // ANNOTATION CHECKING. El has type annotations and, until now, no type + // checking: the annotation fed dispatch and was never itself verified. + // That is the root cause of every silent miscompilation found on + // 2026-08-17, and the failure is not merely a wrong answer: + // + // let x: Int = "hello" ; x + 1 -> printed 4343631981, a string + // POINTER interpreted as an integer + // let s: String = 42 ; print -> dereferenced address 42 + // + // The second is an arbitrary-read primitive if the integer is ever + // attacker-influenced. Literals are checked here because they are + // unambiguous; the emitter only RECORDS the mismatch, the query decides. + let vkind: String = val["expr"] + if str_eq(ltype, "Int") { + if str_eq(vkind, "Str") { + record_call(state_get("__cg_current_fn"), "typemismatch:Int:String:" + name) + } + } + if str_eq(ltype, "String") { + if str_eq(vkind, "Int") { + record_call(state_get("__cg_current_fn"), "typemismatch:String:Int:" + name) + } + if str_eq(vkind, "Float") { + record_call(state_get("__cg_current_fn"), "typemismatch:String:Float:" + name) + } + } + if str_eq(ltype, "Float") { + if str_eq(vkind, "Str") { + record_call(state_get("__cg_current_fn"), "typemismatch:Float:String:" + name) + } + } + // UNANNOTATED let: take the type from what the initialiser RETURNS. + // Without this, `let a = str_len(s)` loses the Int and a later `a + b` + // lowers to el_str_concat on two integers -- silently, with no error, + // producing a program that prints nothing where it should print 7. + // Verified present in the pre-change compiler too, so this is a repair + // rather than a regression. The return types were already needed for + // dispatch; they were just never consulted here. + if str_eq(ltype, "") { + if is_int_expr(val) { add_int_name(name) } + } + // Same as params: Bool is an int in the value model. Without this a + // `let ok: Bool = ...` compared to another Bool lowered to str_eq. + if str_eq(ltype, "Bool") { + add_int_name(name) + } if str_eq(ltype, "Float") { add_float_name(name) } @@ -1705,9 +1782,13 @@ fn cg_stmt(stmt: Map, indent: String, declared: [String]) -> [Strin } else { let c_msg = "EL_STR_PTR(" + cg_expr(msg_node) + ")" } + // Assertions record into PER-TEST state, not global counters. The test + // is the unit of result; a global pass/fail tally cannot say which test + // failed or whether a test ran at all. Reporting is the runner's job — + // nothing is printed here. emit_line(indent + "if (!(" + c_cond + ")) {") - emit_line(indent + " __el_test_fail(__el_cur_test, " + c_msg + "); __el_fail++;") - emit_line(indent + "} else { __el_pass++; }") + emit_line(indent + " __el_test_fail(" + c_msg + ");") + emit_line(indent + "} else { __el_cur_asserts++; }") return declared } @@ -1912,48 +1993,10 @@ fn is_duration_name(name: String) -> Bool { // string-concat on `+` when one side is a Call. New builtins must be added // here when they return Int and may participate in arithmetic. fn is_int_call(call_expr: Map) -> Bool { - let func = call_expr["func"] - let fk: String = func["expr"] - if !str_eq(fk, "Ident") { return false } - let name: String = func["name"] - if str_eq(name, "str_len") { return true } - if str_eq(name, "str_index_of") { return true } - if str_eq(name, "str_to_int") { return true } - if str_eq(name, "str_char_code") { return true } - if str_eq(name, "str_count") { return true } - if str_eq(name, "str_count_chars") { return true } - if str_eq(name, "str_count_bytes") { return true } - if str_eq(name, "str_count_lines") { return true } - if str_eq(name, "str_count_words") { return true } - if str_eq(name, "str_count_letters") { return true } - if str_eq(name, "str_count_digits") { return true } - if str_eq(name, "str_last_index_of") { return true } - if str_eq(name, "str_find_chars") { return true } - if str_eq(name, "native_list_len") { return true } - if str_eq(name, "el_list_len") { return true } - if str_eq(name, "len") { return true } - if str_eq(name, "json_get_int") { return true } - if str_eq(name, "json_array_len") { return true } - if str_eq(name, "engram_node_count") { return true } - if str_eq(name, "engram_edge_count") { return true } - if str_eq(name, "time_now") { return true } - if str_eq(name, "time_now_utc") { return true } - if str_eq(name, "time_diff") { return true } - if str_eq(name, "time_add") { return true } - if str_eq(name, "time_from_parts") { return true } - if str_eq(name, "el_abs") { return true } - if str_eq(name, "el_max") { return true } - if str_eq(name, "el_min") { return true } - if str_eq(name, "float_to_int") { return true } - if str_eq(name, "unix_timestamp") { return true } - if str_eq(name, "instant_to_unix_seconds") { return true } - if str_eq(name, "instant_to_unix_millis") { return true } - if str_eq(name, "duration_to_seconds") { return true } - if str_eq(name, "duration_to_millis") { return true } - if str_eq(name, "duration_to_nanos") { return true } - return false + call_returns(call_expr, "Int") } + // Known runtime builtins that return Float. Parallel to is_int_call — lets a // Call participate in float arithmetic (and get inferred into __float_names on // an unannotated `let`). New Float-returning builtins must be added here. @@ -1979,40 +2022,65 @@ fn is_float_call(call_expr: Map) -> Bool { // Builtins that return an Instant. Used by is_instant_expr and the BinOp // dispatch - `now() + 5.seconds` types as Instant only because we can see // that now() is an Instant-returning Call. -fn is_instant_call(call_expr: Map) -> Bool { - let func = call_expr["func"] - let fk: String = func["expr"] - if !str_eq(fk, "Ident") { return false } - let name: String = func["name"] - if str_eq(name, "now") { return true } - if str_eq(name, "el_now_instant") { return true } - if str_eq(name, "unix_seconds") { return true } - if str_eq(name, "unix_millis") { return true } - if str_eq(name, "instant_from_iso8601") { return true } - if str_eq(name, "el_instant_add_dur") { return true } - if str_eq(name, "el_instant_sub_dur") { return true } - return false +// load_signatures — El-level return types, read once from +// tools/check/signatures.rel. Previously 19 hardcoded names across +// is_instant_call and is_duration_call. The header cannot carry these: it +// declares everything as el_val_t, because El has one type. +fn load_signatures() -> Void { + if !str_eq(state_get("__sig_loaded"), "") { return } + state_set("__sig_loaded", "1") + let path: String = env("EL_SIGNATURES") + if str_eq(path, "") { let path = "tools/check/signatures.rel" } + if !fs_exists(path) { return } + let lines = str_split_lines(fs_read(path)) + let n: Int = native_list_len(lines) + let i = 0 + while i < n { + let line: String = str_trim(native_list_get(lines, i)) + if !str_eq(line, "") { + if !str_starts_with(line, "#") { + let parts = str_split(line, " ") + let np: Int = native_list_len(parts) + let name: String = "" + let ty: String = "" + let seen = 0 + let j = 0 + while j < np { + let tok: String = str_trim(native_list_get(parts, j)) + if !str_eq(tok, "") { + if seen == 0 { let name = tok } + if seen == 2 { let ty = tok } + let seen = seen + 1 + } + let j = j + 1 + } + if seen >= 3 { state_set("__sig_" + name, ty) } + } + } + let i = i + 1 + } } -// Builtins that return a Duration. Same role as is_instant_call. -fn is_duration_call(call_expr: Map) -> Bool { - let func = call_expr["func"] +fn call_returns(expr: Map, want: String) -> Bool { + load_signatures() + let func = expr["func"] let fk: String = func["expr"] if !str_eq(fk, "Ident") { return false } let name: String = func["name"] - if str_eq(name, "el_duration_from_nanos") { return true } - if str_eq(name, "duration_seconds") { return true } - if str_eq(name, "duration_millis") { return true } - if str_eq(name, "duration_nanos") { return true } - if str_eq(name, "el_instant_diff") { return true } - if str_eq(name, "el_duration_add") { return true } - if str_eq(name, "el_duration_sub") { return true } - if str_eq(name, "el_duration_scale") { return true } - if str_eq(name, "el_duration_div") { return true } - if str_eq(name, "ttl_cache_age") { return true } - return false + return str_eq(state_get("__sig_" + name), want) } +fn is_instant_call(expr: Map) -> Bool { + call_returns(expr, "Instant") +} + + +// Builtins that return a Duration. Same role as is_instant_call. +fn is_duration_call(expr: Map) -> Bool { + call_returns(expr, "Duration") +} + + // Phase 1.5 - Calendar / CalendarTime / Rhythm / LocalDate / LocalTime / // LocalDateTime / Zone are first-class boxed types. Each has its own name // set in process state, populated from typed `let` bindings and parameter @@ -2297,15 +2365,6 @@ fn is_duration_expr(expr: Map) -> Bool { // Record a temporal-type violation. Surfaced as `#error` directives at the // top of the generated C, identical machinery to cap_record_violation. // kinds: "instant_plus_instant", "duration_plus_int", etc. -fn time_record_violation(kind: String, detail: String) -> Bool { - let csv: String = state_get("__time_violations") - if str_eq(csv, "") { let csv = "," } - let entry: String = kind + ":" + detail - let key: String = "," + entry + "," - if str_contains(csv, key) { return true } - state_set("__time_violations", csv + entry + ",") - return true -} // Recursive type-propagation: is `expr` known-Int at codegen time? // This unifies the BinOp(+) dispatch so chained arithmetic over Int @@ -2467,119 +2526,25 @@ fn float_operand_c(expr: Map, expr_c: String) -> String { // The compiler-level rule is structural: the binary either CAN or CANNOT // emit the call. There is no runtime check, no opt-in, no override. -fn cap_record_violation(kind: String, fn_name: String) -> Bool { - let csv: String = state_get("__cap_violations") - if str_eq(csv, "") { let csv = "," } - let entry: String = kind + ":" + fn_name - let key: String = "," + entry + "," - if str_contains(csv, key) { return true } - state_set("__cap_violations", csv + entry + ",") - return true -} // Self-formation primitives - the cut between CGI and service. A program // that emits these calls IS structurally a CGI; we forbid them everywhere // else. -fn is_self_formation_call(fn_name: String) -> Bool { - if str_eq(fn_name, "llm_call_agentic") { return true } - if str_eq(fn_name, "llm_register_tool") { return true } - if str_eq(fn_name, "dharma_emit") { return true } - if str_eq(fn_name, "dharma_field") { return true } - return false -} // Any DHARMA primitive - utilities have zero network presence. -fn is_dharma_call(fn_name: String) -> Bool { - if str_eq(fn_name, "dharma_connect") { return true } - if str_eq(fn_name, "dharma_send") { return true } - if str_eq(fn_name, "dharma_activate") { return true } - if str_eq(fn_name, "dharma_emit") { return true } - if str_eq(fn_name, "dharma_field") { return true } - if str_eq(fn_name, "dharma_strengthen") { return true } - if str_eq(fn_name, "dharma_relationship") { return true } - if str_eq(fn_name, "dharma_peers") { return true } - return false -} // Any LLM primitive - utilities have no LLM access at all. -fn is_llm_call(fn_name: String) -> Bool { - if str_eq(fn_name, "llm_call") { return true } - if str_eq(fn_name, "llm_call_system") { return true } - if str_eq(fn_name, "llm_call_agentic") { return true } - if str_eq(fn_name, "llm_vision") { return true } - if str_eq(fn_name, "llm_register_tool") { return true } - if str_eq(fn_name, "llm_models") { return true } - return false -} -fn cap_check_call(fn_name: String) -> Bool { - let kind: String = state_get("__program_kind") - if str_eq(kind, "cgi") { return true } - if str_eq(kind, "service") { - if is_self_formation_call(fn_name) { - cap_record_violation("service", fn_name) - return false - } - return true - } - // utility (default) - if is_dharma_call(fn_name) { - cap_record_violation("utility", fn_name) - return false - } - if is_llm_call(fn_name) { - cap_record_violation("utility", fn_name) - return false - } - return true -} // Emit collected capability violations as #error directives. Called // from codegen()'s entry point right after the cgi/service-block scan, // so they appear at the very top of the generated C. -fn emit_cap_violations() -> Void { - let csv: String = state_get("__cap_violations") - if str_eq(csv, "") { return } - if str_eq(csv, ",") { return } - let n: Int = str_len(csv) - let i: Int = 1 - while i < n { - let next_comma: Int = str_index_of(str_slice(csv, i, n), ",") - if next_comma < 0 { return } - let entry: String = str_slice(csv, i, i + next_comma) - let colon: Int = str_index_of(entry, ":") - if colon > 0 { - let kind: String = str_slice(entry, 0, colon) - let fn_name: String = str_slice(entry, colon + 1, str_len(entry)) - emit_line("#error \"capability violation: '" + kind + "' programs may not call '" + fn_name + "' (self-formation primitive - only 'cgi' programs may use it)\"") - } - let i = i + next_comma + 1 - } -} // Surface temporal-type violations as #error directives. The cg_expr BinOp // dispatcher records each violation (Instant + Instant, Duration + Int, -) // as a CSV entry "kind:detail" via time_record_violation. Each entry maps // to a single #error so downstream cc fails the build with a clear El- // source-level message before the bogus C even links. -fn emit_time_violations() -> Void { - let csv: String = state_get("__time_violations") - if str_eq(csv, "") { return } - if str_eq(csv, ",") { return } - let n: Int = str_len(csv) - let i: Int = 1 - while i < n { - let next_comma: Int = str_index_of(str_slice(csv, i, n), ",") - if next_comma < 0 { return } - let entry: String = str_slice(csv, i, i + next_comma) - let colon: Int = str_index_of(entry, ":") - if colon > 0 { - let detail: String = str_slice(entry, colon + 1, str_len(entry)) - emit_line("#error \"temporal type error: " + detail + "\"") - } - let i = i + next_comma + 1 - } -} // -- Builtin arity table ------------------------------------------------------- // @@ -2594,372 +2559,15 @@ fn emit_time_violations() -> Void { // dynamic dispatch) return -1 -> no check. A mismatch records a violation // in process state, which emit_arity_violations() turns into #error // directives at the top of the generated C. -fn builtin_arity(name: String) -> Int { - // I/O - if str_eq(name, "println") { return 1 } - if str_eq(name, "print") { return 1 } - if str_eq(name, "readline") { return 0 } - // LSP seed primitives - if str_eq(name, "__read_n") { return 1 } - if str_eq(name, "__print_raw") { return 1 } - // String - if str_eq(name, "el_str_concat") { return 2 } - if str_eq(name, "str_eq") { return 2 } - if str_eq(name, "str_starts_with") { return 2 } - if str_eq(name, "str_ends_with") { return 2 } - if str_eq(name, "str_len") { return 1 } - if str_eq(name, "str_concat") { return 2 } - if str_eq(name, "int_to_str") { return 1 } - if str_eq(name, "str_to_int") { return 1 } - if str_eq(name, "str_slice") { return 3 } - if str_eq(name, "str_contains") { return 2 } - if str_eq(name, "str_replace") { return 3 } - if str_eq(name, "str_to_upper") { return 1 } - if str_eq(name, "str_to_lower") { return 1 } - if str_eq(name, "str_trim") { return 1 } - if str_eq(name, "str_index_of") { return 2 } - if str_eq(name, "str_split") { return 2 } - if str_eq(name, "str_char_at") { return 2 } - if str_eq(name, "str_char_code") { return 2 } - if str_eq(name, "str_pad_left") { return 3 } - if str_eq(name, "str_pad_right") { return 3 } - if str_eq(name, "str_format") { return 2 } - if str_eq(name, "str_lower") { return 1 } - if str_eq(name, "str_upper") { return 1 } - // Text-processing primitives (Phase 1) - if str_eq(name, "str_count") { return 2 } - if str_eq(name, "str_count_chars") { return 1 } - if str_eq(name, "str_count_bytes") { return 1 } - if str_eq(name, "str_count_lines") { return 1 } - if str_eq(name, "str_count_words") { return 1 } - if str_eq(name, "str_count_letters") { return 1 } - if str_eq(name, "str_count_digits") { return 1 } - if str_eq(name, "str_index_of_all") { return 2 } - if str_eq(name, "str_last_index_of") { return 2 } - if str_eq(name, "str_find_chars") { return 2 } - if str_eq(name, "str_repeat") { return 2 } - if str_eq(name, "str_reverse") { return 1 } - if str_eq(name, "str_strip_prefix") { return 2 } - if str_eq(name, "str_strip_suffix") { return 2 } - if str_eq(name, "str_strip_chars") { return 2 } - if str_eq(name, "str_lstrip") { return 1 } - if str_eq(name, "str_rstrip") { return 1 } - if str_eq(name, "is_letter") { return 1 } - if str_eq(name, "is_digit") { return 1 } - if str_eq(name, "is_alphanumeric") { return 1 } - if str_eq(name, "is_whitespace") { return 1 } - if str_eq(name, "is_punctuation") { return 1 } - if str_eq(name, "is_uppercase") { return 1 } - if str_eq(name, "is_lowercase") { return 1 } - if str_eq(name, "str_split_lines") { return 1 } - if str_eq(name, "str_split_chars") { return 1 } - if str_eq(name, "str_split_n") { return 3 } - if str_eq(name, "str_join") { return 2 } - // HTML sanitizer - if str_eq(name, "el_html_sanitize") { return 2 } - // Math - if str_eq(name, "el_abs") { return 1 } - if str_eq(name, "el_max") { return 2 } - if str_eq(name, "el_min") { return 2 } - // List - if str_eq(name, "el_list_len") { return 1 } - if str_eq(name, "el_list_get") { return 2 } - if str_eq(name, "el_list_append") { return 2 } - if str_eq(name, "el_list_empty") { return 0 } - if str_eq(name, "el_list_clone") { return 1 } - if str_eq(name, "list_push") { return 2 } - if str_eq(name, "list_push_front") { return 2 } - if str_eq(name, "list_join") { return 2 } - if str_eq(name, "list_range") { return 2 } - // Map - if str_eq(name, "el_get_field") { return 2 } - if str_eq(name, "el_map_get") { return 2 } - if str_eq(name, "el_map_set") { return 3 } - // HTTP - if str_eq(name, "http_get") { return 1 } - if str_eq(name, "http_post") { return 2 } - if str_eq(name, "http_post_json") { return 2 } - if str_eq(name, "http_get_with_headers") { return 2 } - if str_eq(name, "http_post_with_headers") { return 3 } - if str_eq(name, "http_post_form_auth") { return 3 } - if str_eq(name, "http_serve") { return 2 } - if str_eq(name, "http_set_handler") { return 1 } - // Seed primitives (__-prefix) — runtime/el_seed.c - if str_eq(name, "__str_len") { return 1 } - if str_eq(name, "__str_char_at") { return 2 } - if str_eq(name, "__str_alloc") { return 1 } - if str_eq(name, "__str_set_char") { return 3 } - if str_eq(name, "__str_cmp") { return 2 } - if str_eq(name, "__str_ncmp") { return 3 } - if str_eq(name, "__str_concat_raw") { return 2 } - if str_eq(name, "__str_slice_raw") { return 3 } - if str_eq(name, "__int_to_str") { return 1 } - if str_eq(name, "__str_to_int") { return 1 } - if str_eq(name, "__float_to_str") { return 1 } - if str_eq(name, "__str_to_float") { return 1 } - if str_eq(name, "__println") { return 1 } - if str_eq(name, "__print") { return 1 } - if str_eq(name, "__readline") { return 0 } - if str_eq(name, "__fs_read") { return 1 } - if str_eq(name, "__fs_write") { return 2 } - if str_eq(name, "__fs_exists") { return 1 } - if str_eq(name, "__fs_list_raw") { return 1 } - if str_eq(name, "__fs_mkdir") { return 1 } - if str_eq(name, "__fs_write_bytes") { return 3 } - if str_eq(name, "__http_do") { return 5 } - if str_eq(name, "__http_do_map") { return 5 } - if str_eq(name, "__http_do_to_file") { return 5 } - if str_eq(name, "__http_serve") { return 2 } - if str_eq(name, "__http_serve_v2") { return 2 } - if str_eq(name, "__http_response") { return 3 } - if str_eq(name, "__thread_create") { return 2 } - if str_eq(name, "__thread_join") { return 1 } - if str_eq(name, "__mutex_new") { return 0 } - if str_eq(name, "__mutex_lock") { return 1 } - if str_eq(name, "__mutex_unlock") { return 1 } - if str_eq(name, "__exec") { return 1 } - if str_eq(name, "__exec_bg") { return 1 } - if str_eq(name, "__env_get") { return 1 } - if str_eq(name, "__args_json") { return 0 } - if str_eq(name, "__exit_program") { return 1 } - if str_eq(name, "__time_now_ns") { return 0 } - if str_eq(name, "__sleep_ms") { return 1 } - if str_eq(name, "__uuid_v4") { return 0 } - if str_eq(name, "__sqrt_f") { return 1 } - if str_eq(name, "__log_f") { return 1 } - if str_eq(name, "__ln_f") { return 1 } - if str_eq(name, "__sin_f") { return 1 } - if str_eq(name, "__cos_f") { return 1 } - if str_eq(name, "__pi_f") { return 0 } - if str_eq(name, "__state_set") { return 2 } - if str_eq(name, "__state_get") { return 1 } - if str_eq(name, "__state_del") { return 1 } - if str_eq(name, "__state_keys") { return 0 } - if str_eq(name, "__html_sanitize") { return 2 } - if str_eq(name, "__url_encode") { return 1 } - if str_eq(name, "__url_decode") { return 1 } - if str_eq(name, "__json_get") { return 2 } - if str_eq(name, "__json_get_raw") { return 2 } - if str_eq(name, "__json_parse_map") { return 1 } - if str_eq(name, "__json_stringify_val") { return 1 } - if str_eq(name, "__json_array_len") { return 1 } - if str_eq(name, "__json_array_get") { return 2 } - if str_eq(name, "__json_array_get_string") { return 2 } - if str_eq(name, "__json_set") { return 3 } - if str_eq(name, "__engram_node") { return 3 } - if str_eq(name, "__engram_node_full") { return 8 } - if str_eq(name, "__engram_get_node") { return 1 } - if str_eq(name, "__engram_strengthen") { return 1 } - if str_eq(name, "__engram_forget") { return 1 } - if str_eq(name, "__engram_node_count") { return 0 } - if str_eq(name, "__engram_search") { return 2 } - if str_eq(name, "__engram_scan_nodes") { return 2 } - if str_eq(name, "__engram_connect") { return 4 } - if str_eq(name, "__engram_edge_between") { return 2 } - if str_eq(name, "__engram_neighbors") { return 1 } - if str_eq(name, "__engram_neighbors_filtered") { return 3 } - if str_eq(name, "__engram_activate") { return 2 } - if str_eq(name, "__engram_activate_json") { return 2 } - if str_eq(name, "__engram_scan_nodes_json") { return 2 } - if str_eq(name, "__generate") { return 1 } - // Filesystem - if str_eq(name, "fs_read") { return 1 } - if str_eq(name, "fs_write") { return 2 } - if str_eq(name, "fs_list") { return 1 } - if str_eq(name, "fs_size") { return 1 } - if str_eq(name, "fs_read_b64_chunk") { return 3 } - // JSON - if str_eq(name, "json_get") { return 2 } - if str_eq(name, "json_parse") { return 1 } - if str_eq(name, "json_stringify") { return 1 } - if str_eq(name, "json_get_string") { return 2 } - if str_eq(name, "json_get_int") { return 2 } - if str_eq(name, "json_get_float") { return 2 } - if str_eq(name, "json_get_bool") { return 2 } - if str_eq(name, "json_get_raw") { return 2 } - if str_eq(name, "json_set") { return 3 } - if str_eq(name, "json_array_len") { return 1 } - // Time - if str_eq(name, "time_now") { return 0 } - if str_eq(name, "time_now_utc") { return 0 } - if str_eq(name, "sleep_secs") { return 1 } - if str_eq(name, "sleep_ms") { return 1 } - if str_eq(name, "time_format") { return 2 } - if str_eq(name, "time_to_parts") { return 1 } - if str_eq(name, "time_from_parts") { return 3 } - if str_eq(name, "time_add") { return 3 } - if str_eq(name, "time_diff") { return 3 } - // UUID - if str_eq(name, "uuid_new") { return 0 } - if str_eq(name, "uuid_v4") { return 0 } - // Env / state - if str_eq(name, "env") { return 1 } - if str_eq(name, "state_set") { return 2 } - if str_eq(name, "state_get") { return 1 } - if str_eq(name, "state_del") { return 1 } - if str_eq(name, "state_keys") { return 0 } - // Float - if str_eq(name, "float_to_str") { return 1 } - if str_eq(name, "int_to_float") { return 1 } - if str_eq(name, "float_to_int") { return 1 } - if str_eq(name, "format_float") { return 2 } - if str_eq(name, "decimal_round") { return 2 } - if str_eq(name, "str_to_float") { return 1 } - // Math (Float) - if str_eq(name, "math_sqrt") { return 1 } - if str_eq(name, "math_log") { return 1 } - if str_eq(name, "math_ln") { return 1 } - if str_eq(name, "math_sin") { return 1 } - if str_eq(name, "math_cos") { return 1 } - if str_eq(name, "math_pi") { return 0 } - // Bool - if str_eq(name, "bool_to_str") { return 1 } - // Process - if str_eq(name, "exit_program") { return 1 } - // Process info - if str_eq(name, "getpid_now") { return 0 } - // stdout redirect (used by elc post-processing) - if str_eq(name, "stdout_to_file") { return 1 } - if str_eq(name, "stdout_restore") { return 0 } - // Subprocess execution - if str_eq(name, "exec_command") { return 1 } - if str_eq(name, "exec_capture") { return 1 } - if str_eq(name, "exec") { return 1 } - if str_eq(name, "exec_bg") { return 1 } - // CGI / DHARMA - if str_eq(name, "dharma_connect") { return 1 } - if str_eq(name, "dharma_send") { return 2 } - if str_eq(name, "dharma_activate") { return 1 } - if str_eq(name, "dharma_emit") { return 2 } - if str_eq(name, "dharma_field") { return 1 } - if str_eq(name, "dharma_strengthen") { return 2 } - if str_eq(name, "dharma_relationship") { return 1 } - if str_eq(name, "dharma_peers") { return 0 } - // Engram - if str_eq(name, "engram_node") { return 3 } - if str_eq(name, "engram_node_full") { return 8 } - if str_eq(name, "engram_get_node") { return 1 } - if str_eq(name, "engram_strengthen") { return 1 } - if str_eq(name, "engram_forget") { return 1 } - if str_eq(name, "engram_node_count") { return 0 } - if str_eq(name, "engram_search") { return 2 } - if str_eq(name, "engram_scan_nodes") { return 2 } - if str_eq(name, "engram_connect") { return 4 } - if str_eq(name, "engram_edge_between") { return 2 } - if str_eq(name, "engram_neighbors") { return 1 } - if str_eq(name, "engram_neighbors_filtered") { return 3 } - if str_eq(name, "engram_edge_count") { return 0 } - if str_eq(name, "engram_activate") { return 2 } - if str_eq(name, "engram_save") { return 1 } - if str_eq(name, "engram_load") { return 1 } - if str_eq(name, "engram_store_boot") { return 1 } - if str_eq(name, "engram_store_checkpoint") { return 0 } - if str_eq(name, "engram_store_close") { return 0 } - if str_eq(name, "engram_get_node_json") { return 1 } - if str_eq(name, "engram_get_node_by_label") { return 1 } - if str_eq(name, "engram_search_json") { return 2 } - if str_eq(name, "engram_scan_nodes_json") { return 2 } - if str_eq(name, "engram_neighbors_json") { return 3 } - if str_eq(name, "engram_activate_json") { return 2 } - if str_eq(name, "engram_stats_json") { return 0 } - // LLM - if str_eq(name, "llm_call") { return 2 } - if str_eq(name, "llm_call_system") { return 3 } - if str_eq(name, "llm_call_agentic") { return 4 } - if str_eq(name, "llm_vision") { return 4 } - if str_eq(name, "llm_models") { return 0 } - if str_eq(name, "llm_register_tool") { return 2 } - // Crypto - if str_eq(name, "sha256_hex") { return 1 } - if str_eq(name, "sha256_bytes") { return 1 } - if str_eq(name, "hmac_sha256_hex") { return 2 } - if str_eq(name, "hmac_sha256_bytes") { return 2 } - if str_eq(name, "base64_encode") { return 1 } - if str_eq(name, "base64_decode") { return 1 } - if str_eq(name, "base64url_encode") { return 1 } - if str_eq(name, "base64url_decode") { return 1 } - // Native VM aliases - if str_eq(name, "native_list_get") { return 2 } - if str_eq(name, "native_list_len") { return 1 } - if str_eq(name, "native_list_append") { return 2 } - if str_eq(name, "native_list_empty") { return 0 } - if str_eq(name, "native_list_clone") { return 1 } - if str_eq(name, "native_string_chars") { return 1 } - if str_eq(name, "native_int_to_str") { return 1 } - // Method-call aliases - if str_eq(name, "append") { return 2 } - if str_eq(name, "len") { return 1 } - if str_eq(name, "get") { return 2 } - if str_eq(name, "map_get") { return 2 } - if str_eq(name, "map_set") { return 3 } - // Threading seed primitives - if str_eq(name, "__thread_create") { return 2 } - if str_eq(name, "__thread_join") { return 1 } - if str_eq(name, "__mutex_new") { return 0 } - if str_eq(name, "__mutex_lock") { return 1 } - if str_eq(name, "__mutex_unlock") { return 1 } - // Channel seed primitives - if str_eq(name, "__channel_new") { return 1 } - if str_eq(name, "__channel_send") { return 2 } - if str_eq(name, "__channel_recv") { return 1 } - if str_eq(name, "__channel_try_recv") { return 1 } - if str_eq(name, "__channel_close") { return 1 } - // Arena mark/restore builtins - if str_eq(name, "el_arena_push") { return 0 } - if str_eq(name, "el_arena_pop") { return 1 } - // -1 sentinel: variadic / unknown / user-defined -> no check. - return -1 -} -fn arity_record_violation(fn_name: String, expected: Int, actual: Int) -> Bool { - let csv: String = state_get("__arity_violations") - if str_eq(csv, "") { let csv = "," } - // Encode as fn_name|expected|actual to recover all three at emit time. - let entry: String = fn_name + "|" + native_int_to_str(expected) + "|" + native_int_to_str(actual) - let key: String = "," + entry + "," - if str_contains(csv, key) { return true } - state_set("__arity_violations", csv + entry + ",") - return true -} // Validate the call's arity against the builtin table. Returns true (always) // because cg_expr ignores the result; -1 from builtin_arity signals // "no check possible" (variadic or user-defined). A mismatch is recorded // and surfaced as an #error at the bottom of the generated C, so cc fails // before it ever attempts to type-check the wrong call. -fn arity_check_call(fn_name: String, actual: Int) -> Bool { - let expected: Int = builtin_arity(fn_name) - if expected < 0 { return true } - if expected == actual { return true } - arity_record_violation(fn_name, expected, actual) - return true -} // Emit recorded arity violations as #error directives. -fn emit_arity_violations() -> Void { - let csv: String = state_get("__arity_violations") - if str_eq(csv, "") { return } - if str_eq(csv, ",") { return } - let n: Int = str_len(csv) - let i: Int = 1 - while i < n { - let next_comma: Int = str_index_of(str_slice(csv, i, n), ",") - if next_comma < 0 { return } - let entry: String = str_slice(csv, i, i + next_comma) - let p1: Int = str_index_of(entry, "|") - if p1 > 0 { - let fn_name: String = str_slice(entry, 0, p1) - let rest: String = str_slice(entry, p1 + 1, str_len(entry)) - let p2: Int = str_index_of(rest, "|") - if p2 > 0 { - let exp_s: String = str_slice(rest, 0, p2) - let act_s: String = str_slice(rest, p2 + 1, str_len(rest)) - emit_line("#error \"arity error: '" + fn_name + "' takes " + exp_s + " arguments, but called with " + act_s + "\"") - } - } - let i = i + next_comma + 1 - } -} fn add_int_name(name: String) -> Bool { let csv: String = state_get("__int_names") @@ -3081,6 +2689,15 @@ fn build_int_names_for_params(params: [Map]) -> Bool { if str_eq(ptype, "Int") { add_int_name(pname) } + // Bool is an integer in the value model (type_to_c maps Bool -> "int"; + // el_runtime.h: "Bool -> el_val_t (0 = false, nonzero = true)"), but + // Bool names were registered nowhere. So `cond == want` between two + // Bool params fell through to str_eq and dereferenced 0 or 1 as a + // char* — an immediate segfault. Track them as int-like, which is what + // they are. + if str_eq(ptype, "Bool") { + add_int_name(pname) + } if str_eq(ptype, "Float") { add_float_name(pname) } @@ -3134,8 +2751,100 @@ fn fn_has_decorator(stmt: Map, name: String) -> Bool { false } +// cg_entry_seam — emit the guards and the entry injection for a decorated fn. +// Extracted so it can be emitted into the WRAPPER when a construct injects at +// exit (the body then lives in a static helper) and inline otherwise. +// +// Guards run FIRST and may refuse — a non-zero return short-circuits the fn and +// becomes its result. A refused call must not report a crossing, so guards +// precede injection. EVERY guard runs (stacking @authenticate @authorize +// applies both); injection is topmost-wins, matching the VBD role convention, +// because a role is singular and a refusal is not. +fn cg_entry_seam(stmt: Map, fn_name: String) -> Void { + // The crossing is resolved at EXECUTION. One refusable indirection replaces + // the compile-time guard loop and injection loop: which constructs apply, + // and whether any of them refuses, is read from a table written after the + // binary exists. + emit_line(" { el_val_t __s = el_seam_run(EL_STR(" + c_str_lit(fn_name) + "), 0, 0); if (__s) return __s; }") +} + +// cg_exit_target / cg_exit_construct — the first construct on this fn that +// injects at exit, or "" if none. + + + +// params_to_env_fields / params_to_env_init / params_to_env_args — the captured +// environment. This IS the closure: a struct of captured values, a function +// pointer that takes it, and the pair handed to the wrap target. +fn params_to_env_fields(params: [Any]) -> String { + // A zero-param fn would emit `struct __env_f { };` -- an empty struct is a + // GNU extension, not C99, and an empty initialiser is C23. + if native_list_len(params) == 0 { return " char __e0;" } + let out: String = "" + let n: Int = native_list_len(params) + let i = 0 + while i < n { + let p = native_list_get(params, i) + let pn: String = p["name"] + let out = out + " el_val_t " + pn + ";" + let i = i + 1 + } + out +} + +fn params_to_env_init(params: [Any]) -> String { + if native_list_len(params) == 0 { return "0" } + let out: String = "" + let n: Int = native_list_len(params) + let i = 0 + while i < n { + let p = native_list_get(params, i) + let pn: String = p["name"] + if i > 0 { let out = out + ", " } + let out = out + pn + let i = i + 1 + } + out +} + +fn params_to_env_args(params: [Any]) -> String { + let out: String = "" + let n: Int = native_list_len(params) + let i = 0 + while i < n { + let p = native_list_get(params, i) + let pn: String = p["name"] + if i > 0 { let out = out + ", " } + let out = out + "__e->" + pn + let i = i + 1 + } + out +} + + +// params_to_call_args — "a, b, c" from the param list, for the wrapper's call +// into the body helper. + fn cg_fn(stmt: Map) -> Void { let fn_name: String = stmt["name"] + state_set("__cg_current_fn", fn_name) + // Record every top-level definition. El has NO namespacing -- imports are + // textual inlining, so two modules defining the same name emit two C + // functions into one translation unit. cc catches it, but names the + // generated helpers first and the user's fn fourth. Recording it lets the + // collision be reported at El level, in El terms. + record_call(fn_name, "defines_at:" + stmt["line"]) + // Emit which constructs this fn carries, so the prohibition check can be a + // query over relations instead of a walk inside the emitter. + let cdl = stmt["decorators"] + let n_cdl: Int = native_list_len(cdl) + let cdi = 0 + while cdi < n_cdl { + let cd = native_list_get(cdl, cdi) + let cdn: String = cd["name"] + record_call(fn_name, "@" + cdn) + let cdi = cdi + 1 + } // Skip El's `fn main()` - C provides its own main() for top-level stmts // and a duplicate `el_val_t main(void)` would collide with it. if fn_name == "main" { return } @@ -3147,23 +2856,32 @@ fn cg_fn(stmt: Map) -> Void { // from @manager-decorated functions. Surface violations to the C compiler // via #error directives emitted before the function definition. Read the // decorator LIST so the role may be stacked with other decorators. - if vbd_has_restricted_call(body) { - if !fn_has_decorator(stmt, "manager") { - emit_line("#error \"VBD violation: dharma_emit/dharma_field called from non-@manager fn '" + fn_name + "'\"") - } - } + // Check each DECLARED prohibition in turn. The owning construct is known by + // construction, so the diagnostic names it rather than hardcoding one rule. + // The walk records; it no longer decides. Whether a call is legal where it + // appears is a query over the emitted relations — see + // tools/check/prohibitions.sh. An emitter that also adjudicates has to + // contain every rule anyone will ever want. + vbd_has_restricted_call(body) // Seed the per-function int-name set so the `+` codegen can dispatch // arithmetic vs concat on type-annotated identifiers. build_int_names_for_params(params) - emit_line("el_val_t " + fn_name + "(" + params_c + ") {") // ── API-reshape decorator-seam: auto-emit at the decorated-fn boundary ── - // Every @manager/@accessor fn gets ONE injected call to engram_boundary_beat - // at entry — interoception (chrono tick) + telemetry (afferent counter) + - // strengthen (self-activity) + a dharma bus event — so a decorated op - // self-reports with ZERO hand-written instrumentation in its body. (VBD role - // = the topmost decorator; write it topmost when stacking with @route.) - if fn_has_decorator(stmt, "manager") || fn_has_decorator(stmt, "accessor") { - emit_line(" engram_boundary_beat(EL_STR(" + c_str_lit(fn_name) + "));") + // A decorated op self-reports with ZERO hand-written instrumentation in its + // body. What it reports, and whether it may refuse, is DECLARED by the + // construct — codegen reads it (see scan_declared_decorators). + // + // When a construct injects at exit, the body moves into a static helper and + // the visible fn becomes a wrapper, so EARLY RETURNS still pass through the + // exit injection. Emitting it only before the fall-through return would + // silently miss every early return — which is precisely the class of + // failure this seam exists to remove. Fns with no exit construct emit + // exactly as before, byte for byte. + if true { + emit_line("static el_val_t __el_body_" + fn_name + "(" + params_c + ") {") + } else { + emit_line("el_val_t " + fn_name + "(" + params_c + ") {") + cg_entry_seam(stmt, fn_name) } // Seed declared with parameter names so reassignment works let decl = native_list_empty() @@ -3187,6 +2905,28 @@ fn cg_fn(stmt: Map) -> Void { el_release(final_decl) emit_line(" return 0;") emit_line("}") + // The wrapper: guards, entry injection, the body call, then the exit + // injection, which receives the result so it can observe what the fn + // actually returned. + if true { + // EVERY fn gets an env struct and a thunk. Codegen cannot know which fns + // a wrap construct will be bound to after the binary exists, and the + // wrapper unconditionally calls through el_seam_wrap. + emit_line("struct __env_" + fn_name + " { " + params_to_env_fields(params) + " };") + emit_line("static el_val_t __thunk_" + fn_name + "(void* __v) {") + emit_line(" struct __env_" + fn_name + "* __e = (struct __env_" + fn_name + "*)__v; (void)__e;") + emit_line(" return __el_body_" + fn_name + "(" + params_to_env_args(params) + ");") + emit_line("}") + } + if true { + emit_line("el_val_t " + fn_name + "(" + params_c + ") {") + cg_entry_seam(stmt, fn_name) + emit_line(" struct __env_" + fn_name + " __env = { " + params_to_env_init(params) + " };") + emit_line(" el_val_t __r = el_seam_wrap(EL_STR(" + c_str_lit(fn_name) + "), __thunk_" + fn_name + ", &__env);") + emit_line(" __r = el_seam_run(EL_STR(" + c_str_lit(fn_name) + "), 1, __r);") + emit_line(" return __r;") + emit_line("}") + } emit_blank() } @@ -3204,6 +2944,7 @@ fn is_top_level_decl(stmt: Map) -> Bool { if kind == "EnumDef" { return true } if kind == "Import" { return true } if kind == "CgiBlock" { return true } + if kind == "ProgramBlock" { return true } if kind == "ExternFn" { return true } false } @@ -3216,17 +2957,94 @@ fn cgi_arg(value: String, has_value: Bool) -> String { return "EL_NULL" } +// -- Program block: cross-cutting concerns injected at the process boundary ---- +// +// emit_program_init — emit the `static void __el_program_init(void)` that +// carries a program's declared cross-cutting concerns. Called from main() +// BEFORE any user statement runs, so the guarantees hold for the whole process +// rather than depending on each call site remembering to ask for them. +// +// This is emitted at the point the `program` block is encountered, not buffered +// until main(). The streaming backend emits in source order and cannot hold a +// declaration's entry list alive until main(); emitting a named function here +// and calling it from main() means only a single bool has to survive. +// +// Order matters and is deliberate: +// 1. singleton FIRST — if another instance already holds the lock, refuse and +// exit before touching configuration, ports, or any data directory. +// 2. config declarations — resolve env-or-default, one declaration per entry. +// 3. validate LAST — report EVERY missing/ill-typed entry at once, then exit. +// +// `singleton:` carries its `guards:` expression as its SECOND argument — the +// state the lock protects, evaluated here at the process boundary. A singleton +// without one does not compile (see below): a lock keyed on the program's name +// rather than on its state refuses unrelated instances and permits concurrent +// ones, which is not a weaker guard but a wrong one. +fn el_bool_arg(b: Bool) -> String { + if b { return "EL_INT(1)" } + return "EL_INT(0)" +} + +fn emit_program_init(stmt: Map) -> Void { + let pname: String = stmt["name"] + emit_line("static void __el_program_init(void) {") + let has_singleton: Bool = stmt["has_singleton"] + if has_singleton { + let sid: String = stmt["singleton"] + let has_guards: Bool = stmt["has_guards"] + if has_guards { + let guards_c: String = cg_expr(stmt["guards"]) + emit_line(" el_singleton_acquire(EL_STR(" + c_str_lit(sid) + "), " + guards_c + ");") + } else { + // Refuse at COMPILE time. The alternative — emitting a name-keyed + // lock — is the defect itself, and it fails silently in the direction + // that loses data. + emit_line("#error \"singleton '" + sid + "' declares no `guards:` — a singleton must name the state it protects, e.g. `guards: engram_resolve_data_dir()` (spec 18.2)\"") + } + } + let entries = stmt["entries"] + let n: Int = native_list_len(entries) + let i = 0 + while i < n { + let e = native_list_get(entries, i) + let ename: String = e["name"] + let etype: String = e["etype"] + let edefault: String = e["default"] + let has_default: Bool = e["has_default"] + let erequired: Bool = e["required"] + let arg_def: String = cgi_arg(edefault, has_default) + emit_line(" el_config_declare(EL_STR(" + c_str_lit(ename) + "), EL_STR(" + c_str_lit(etype) + "), " + arg_def + ", " + el_bool_arg(has_default) + ", " + el_bool_arg(erequired) + ");") + let i = i + 1 + } + emit_line(" el_config_validate(EL_STR(" + c_str_lit(pname) + "));") + emit_line("}") + emit_blank() +} + // -- VBD role enforcement ------------------------------------------------------ // // Scan a function body for direct calls to DHARMA-restricted builtins // (dharma_emit, dharma_field). These may only appear inside @manager fns. -fn vbd_is_restricted_name(name: String) -> Bool { - if str_eq(name, "dharma_emit") { return true } - if str_eq(name, "dharma_field") { return true } - false +// record_call — emit one calls relation. The traversal is +// irreducible: you must walk the AST to find calls. What is NOT irreducible is +// the rule (which names are restricted) or the decision (#error). Those move +// out; the walk stays and changes purpose from deciding to recording. +fn record_call(caller: String, callee: String) -> Void { + let path: String = env("EL_RELATIONS_OUT") + if str_eq(path, "") { return } + // cg_fn returns early for `main` -- C provides its own -- so __cg_current_fn + // is unset while main's body is walked. Without this, every call in main is + // invisible to the relation graph and both checkers have a blind spot + // exactly where a program does its work. Caught by capability_query.sh. + let caller2: String = caller + if str_eq(caller2, "") { let caller2 = "main" } + let prev: String = "" + if fs_exists(path) { let prev = fs_read(path) } + fs_write(path, prev + caller2 + " calls " + callee + "\n") } + fn vbd_expr_has_restricted_call(expr: Map) -> Bool { let kind: String = expr["expr"] if str_eq(kind, "Call") { @@ -3234,7 +3052,7 @@ fn vbd_expr_has_restricted_call(expr: Map) -> Bool { let fk: String = func["expr"] if str_eq(fk, "Ident") { let fname: String = func["name"] - if vbd_is_restricted_name(fname) { return true } + record_call(state_get("__cg_current_fn"), fname) } if vbd_expr_has_restricted_call(func) { return true } let args = expr["args"] @@ -3403,11 +3221,8 @@ fn codegen(stmts: [Map], source: String) -> String { if svc_count >= 1 { let kind = "service" } state_set("__program_kind", kind) // Clear capability-violation accumulator from any prior compile. - state_set("__cap_violations", "") // Clear arity-violation accumulator from any prior compile. - state_set("__arity_violations", "") // Clear temporal-type-violation accumulator from any prior compile. - state_set("__time_violations", "") // Preamble emit_line("#include ") @@ -3538,6 +3353,20 @@ fn codegen(stmts: [Map], source: String) -> String { } } + // Program block: emit the cross-cutting init function before the user's + // functions so main() can call it (see emit_program_init). + let prog_have: Bool = false + let i = 0 + while i < n { + let stmt = native_list_get(stmts, i) + let sk4: String = stmt["stmt"] + if str_eq(sk4, "ProgramBlock") { + emit_program_init(stmt) + let prog_have = true + } + let i = i + 1 + } + // Function definitions let i = 0 while i < n { @@ -3554,8 +3383,16 @@ fn codegen(stmts: [Map], source: String) -> String { // main(). Use _argc/_argv so El programs are free to declare their own // local `argv` / `argc` (compiler.el itself does this) without colliding // with the C-side parameters when fn main()'s body is folded in below. + // From here the emitter is walking main's body. Without resetting this the + // caller attribution is stale from the last cg_fn, so a violation inside + // main is reported against whichever function happened to be emitted last + // -- a diagnostic naming the wrong fn is worse than none. + state_set("__cg_current_fn", "main") emit_line("int main(int _argc, char** _argv) {") emit_line(" el_runtime_init_args(_argc, _argv);") + if prog_have { + emit_line(" __el_program_init();") + } if cgi_count >= 1 { let cname: String = cgi_block["name"] let cdid: String = cgi_block["dharma_id"] @@ -3647,13 +3484,10 @@ fn codegen(stmts: [Map], source: String) -> String { // will fail on the first one and surface the message; placement at // the bottom is fine - preprocessor errors halt the build wherever // they appear. - emit_cap_violations() // Same for builtin-arity violations: cc halts on the first #error, // so a misuse of a known builtin (wrong arg count) fails the build // with a clear message naming the builtin and its expected arity. - emit_arity_violations() // Temporal-type violations (Instant + Instant, Duration + Int, -). - emit_time_violations() // Return empty string - output was streamed via println "" @@ -3827,7 +3661,7 @@ fn route_sort_desc(recs: [Map]) -> [Map] { // be synthesized in the streaming backend, which discards per-fn ASTs. Handles // decorator STACKING: `@route(...) @manager fn` still records the route. fn scan_routes(tokens: [Any]) -> [Map] { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let recs: [Map] = native_list_empty() let has_pending: Bool = false let pending_args: [String] = native_list_empty() @@ -3894,6 +3728,178 @@ fn program_has_routes(recs: [Map]) -> Bool { native_list_len(recs) > 0 } +// ── Declared constructs ────────────────────────────────────────────────────── +// +// A construct declares its own meaning. Codegen READS the declaration instead +// of knowing the construct: +// +// @decorator("injects_at_entry", "engram_boundary_beat") +// fn manager() {} +// +// Any fn decorated @manager then gets +// engram_boundary_beat(EL_STR(), EL_STR("manager")); +// injected at entry. Adding a construct is a declaration IN THE PROGRAM. It +// does not touch the compiler. +// +// WHY: codegen called fn_has_decorator for exactly three names — manager, +// accessor, route. Twelve others parsed, attached as {name,args}, and compiled +// to nothing, including four that look like protection: @authenticate (6 uses), +// @authorize (3), @rate_limit (3), @validate (2). The meaning of a construct +// had nowhere to live except the emitter, so every construct was a compiler +// edit and an undeclared one was silently inert. A lookup table of +// name -> injection would have moved the enumeration twenty lines up, not +// removed it; the class only goes away when the construct itself carries its +// meaning. +// +// manager and accessor are seeded below as the compiled-in core — a seed the +// periphery is declared against, the same shape as fixedSelf in substrate.go: +// a complete fallback exists, declaration is enrichment. A program may declare +// its own constructs and may redeclare these. +// +// This is the injection half of the seam. The prohibition half (@manager's +// #error on dharma_emit) is still hardcoded, because "which calls may appear +// inside this boundary" is a query over program structure and there is nothing +// yet to ask. + + + +// A GUARD is an injection that may refuse. The declared target is called at +// entry with the same (fn, construct) pair; a non-zero return short-circuits +// the decorated fn and becomes its result. +// +// @decorator("guards_at_entry", "my_auth_check") +// fn authenticate() {} +// +// @authenticate +// fn handler() -> String { ... } // my_auth_check runs first and may refuse +// +// This is what @authenticate (6 uses), @authorize (3), @rate_limit (3) and +// @validate (2) needed and never had. They parsed, attached, and compiled to +// nothing — fourteen applications that read as protection and emitted no +// instruction. The compiler still knows nothing about authentication: the +// program points the construct at its own function. + + +// An EXIT injection runs after the fn returns and receives the result: +// target(, , ) +// +// @decorator("injects_at_exit", "persist_now") +// fn durable() {} +// +// This is `hold`'s after-crossing face. §6 of the design records 62 +// persist-after-mutate sites and 9-of-9 failed index-after-append sites — every +// one an obligation that decayed into "remember to do this after you mutate." +// An obligation a human must remember is not an obligation. + + +// A WRAP construct receives the body as a CLOSURE and decides how, whether, and +// how many times to invoke it. +// +// target(, , el_val_t (*body)(void*), void* env) -> el_val_t +// +// C has no closure SYNTAX. That is not the same as C being unable to express a +// closure: every C++ lambda, every Go closure, every Rust closure compiles to a +// struct of captured values plus a function pointer, which is exactly what is +// emitted here. Taking "C has no closures" as a fact about what is possible, +// rather than about one grammar, produces a strictly weaker construct that can +// only decide whether to repeat — no timeout, no rollback-and-retry, no +// parallel, no memoize-on-arguments. It would also have crippled the JS backend, +// which has closures natively, for a limit in the C one. + + +// A PROHIBITION is the other half of a boundary: not what runs when something +// crosses, but what may not cross at all. +// +// @decorator("prohibits_outside", "dharma_emit,dharma_field") +// fn manager() {} +// +// Calls to those names may then appear ONLY inside a fn carrying @manager. +// Before this the rule lived as two string literals in vbd_is_restricted_name +// and one #error in cg_fn -- one prohibition, uneditable without a compiler +// release, and unrankable because a #error is a string handed to cpp, which +// knows nothing about El. +// +// The construct list is kept in one state key so the recursive walker can be +// parameterised without threading an argument through every branch of it -- +// the same mechanism codegen already uses for __match_counter and +// __if_expr_counter. +fn declare_prohibition(construct: String, names_csv: String) -> Void { + record_call(construct, "prohibits:" + names_csv) +} + + + +// scan_declared_decorators — token-level pre-pass registering every construct +// the program declares. Runs once per module alongside scan_routes, because +// the streaming backend discards per-fn ASTs and there is no whole-program AST +// to walk. +fn scan_declared_decorators(tokens: [Any]) -> Void { + declare_prohibition("manager", "dharma_emit,dharma_field") + let total: Int = native_list_len(tokens) / 3 + let has_pending_p: Bool = false + let pending_prohibit: String = "" + let pos: Int = 0 + let going: Bool = true + while going { + if pos >= total { + let going = false + } else { + let k: String = tok_kind(tokens, pos) + if str_eq(k, "Eof") { + let going = false + } else { + if str_eq(k, "At") { + let dname: String = tok_value(tokens, pos + 1) + let p: Int = pos + 2 + let args: [String] = native_list_empty() + let ka: String = tok_kind(tokens, p) + if str_eq(ka, "LParen") { + let p = p + 1 + let running: Bool = true + while running { + let kd: String = tok_kind(tokens, p) + if str_eq(kd, "RParen") { + let running = false + } else { + if str_eq(kd, "Eof") { + let running = false + } else { + if str_eq(kd, "Str") { + let args = native_list_append(args, tok_value(tokens, p)) + } + let p = p + 1 + } + } + } + if str_eq(tok_kind(tokens, p), "RParen") { let p = p + 1 } + } + if str_eq(dname, "decorator") { + if native_list_len(args) >= 2 { + let dkind: String = native_list_get(args, 0) + if str_eq(dkind, "prohibits_outside") { + let has_pending_p = true + let pending_prohibit = native_list_get(args, 1) + } + } + } + let pos = p + } else { + if str_eq(k, "Fn") { + let fname: String = tok_value(tokens, pos + 1) + if has_pending_p { + declare_prohibition(fname, pending_prohibit) + let has_pending_p = false + } + let pos = pos + 2 + } else { + let pos = pos + 1 + } + } + } + } + } +} + // route_method_guard — C boolean prefix guarding on HTTP method, or "" for none. fn route_method_guard(method: String) -> String { if str_eq(method, "") { return "" } @@ -3990,9 +3996,6 @@ fn emit_streaming_preamble(sigs: [Map], source: String) -> Void { if cgi_count >= 1 { let kind = "cgi" } if svc_count >= 1 { let kind = "service" } state_set("__program_kind", kind) - state_set("__cap_violations", "") - state_set("__arity_violations", "") - state_set("__time_violations", "") emit_line("#include ") emit_line("#include ") @@ -4043,7 +4046,7 @@ fn emit_streaming_preamble(sigs: [Map], source: String) -> Void { // sigs: pre-scanned signature list from scan_fn_sigs(tokens) // source: original source string (for string literal lookup) fn codegen_streaming(tokens: [Any], sigs: [Map], source: String) -> String { - let total_tokens: Int = native_list_len(tokens) / 2 + let total_tokens: Int = native_list_len(tokens) / 3 // Emit preamble (forward decls, file-scope lets, #includes) // Arena scope: free intermediate strings built during preamble emission. @@ -4056,6 +4059,11 @@ fn codegen_streaming(tokens: [Any], sigs: [Map], source: String) -> // tokens below via refcount, like `sigs`). If any exist, forward-declare the // generated dispatcher NOW so hand-written fns (e.g. handle_request) may call // it before its definition is emitted after the fn-emit loop. + // Declared constructs: register what the program says its decorators mean, + // before any fn is emitted. Must precede the fn-emit loop — cg_fn reads the + // registry to decide what, if anything, a decorator injects. + scan_declared_decorators(tokens) + let route_records: [Map] = scan_routes(tokens) if program_has_routes(route_records) { emit_line("el_val_t el_route_dispatch(el_val_t method, el_val_t clean, el_val_t path, el_val_t body);") @@ -4100,13 +4108,36 @@ fn codegen_streaming(tokens: [Any], sigs: [Map], source: String) -> // Emit test harness preamble (counters, fail printer) when in test mode. if test_is_mode { emit_line("#include ") + emit_line("#include ") + emit_line("#include ") emit_blank() - emit_line("static int __el_pass = 0, __el_fail = 0;") + // Per-test result state. Reset by __el_reg_invoke before each test, so + // every test gets its own record rather than contributing to a global + // tally. The first failure message is retained; later ones only bump + // the count, which keeps the common case allocation-free. + emit_line("static int __el_cur_fails = 0;") + emit_line("static int __el_cur_asserts = 0;") + emit_line("static char __el_cur_msg[512] = \"\";") emit_line("static const char *__el_cur_test = \"(none)\";") - emit_line("static void __el_test_fail(const char *test, const char *msg) {") - emit_line(" fprintf(stderr, \"FAIL %-40s %s\\n\", test, msg);") + emit_line("static void __el_test_fail(const char *msg) {") + emit_line(" if (__el_cur_fails == 0 && msg) {") + emit_line(" snprintf(__el_cur_msg, sizeof __el_cur_msg, \"%s\", msg);") + emit_line(" }") + emit_line(" __el_cur_fails++; __el_cur_asserts++;") emit_line("}") emit_blank() + // Forward declarations for the registry accessors. The definitions are + // emitted at the END of the unit (they reference the test functions, + // which do not exist yet at this point), but the El-side runner is + // compiled in between and calls them — so it needs the prototypes here. + emit_line("el_val_t __el_reg_count(void);") + emit_line("el_val_t __el_reg_name(el_val_t i);") + emit_line("el_val_t __el_reg_invoke(el_val_t i);") + emit_line("el_val_t __el_reg_last_ns(void);") + emit_line("el_val_t __el_reg_msg(void);") + emit_line("el_val_t __el_reg_asserts(void);") + emit_line("el_val_t __el_opt_json(void);") + emit_blank() } // Streaming parse-emit loop. @@ -4126,6 +4157,7 @@ fn codegen_streaming(tokens: [Any], sigs: [Map], source: String) -> // Fix: copy the values out BEFORE the release (strings, so no dangling reference) // and emit from these. No search, so the failure mode is removed rather than moved. let cgi_have: Bool = false + let prog_have: Bool = false let cgi_name_v: String = "" let cgi_did_v: String = "" let cgi_prin_v: String = "" @@ -4247,6 +4279,14 @@ fn codegen_streaming(tokens: [Any], sigs: [Map], source: String) -> // These are no-ops in codegen (forward decls already emitted) // — except a CgiBlock, whose declared identity must survive // this release to be emitted as a compiled constant. + // A ProgramBlock's cross-cutting declarations are + // emitted HERE, as a named init function, because the + // streaming backend cannot hold the entry list alive + // until main(). Only the bool survives. + if str_eq(sk, "ProgramBlock") { + emit_program_init(stmt) + let prog_have = true + } if str_eq(sk, "CgiBlock") { let cgi_have = true let cgi_name_v = stmt["name"] @@ -4302,17 +4342,73 @@ fn codegen_streaming(tokens: [Any], sigs: [Map], source: String) -> el_release(sigs) let test_arena_mark: Any = el_arena_push() + let tn: Int = native_list_len(test_c_names) + + // ── Generated test registry ────────────────────────────────────────── + // Discovery happens HERE, at compile time. The runner never searches + // for tests; it walks this table. That ordering — discovery strictly + // before execution — is what makes --list, filtering, sharding and + // per-test reporting possible later, and it is why the old harness + // (which inlined direct calls into main) could not have any of them. + emit_line("typedef void (*__el_test_fp)(void);") + emit_line("typedef struct { const char *name; __el_test_fp fn; } __el_test_entry;") + emit_line("static const __el_test_entry __el_registry[] = {") + let ri: Int = 0 + while ri < tn { + let r_name: String = native_list_get(test_names, ri) + let r_cfn: String = native_list_get(test_c_names, ri) + emit_line(" { \"" + c_escape(r_name) + "\", " + r_cfn + " },") + let ri = ri + 1 + } + // Trailing sentinel keeps the array non-empty when a file declares no + // tests (a zero-length array is not valid C). + emit_line(" { 0, 0 }") + emit_line("};") + emit_line("static const int __el_registry_n = " + int_to_str(tn) + ";") + emit_blank() + emit_line("static long long __el_last_ns = 0;") + emit_line("static int __el_opt_json_v = 0;") + emit_blank() + + // ── Index-based accessors ──────────────────────────────────────────── + // El has no function pointers, so the runner works purely in indices. + // This is the whole seam between generated C and the El-side runner. + emit_line("el_val_t __el_reg_count(void) { return (el_val_t)(int64_t)__el_registry_n; }") + emit_line("el_val_t __el_reg_name(el_val_t i) {") + emit_line(" int64_t k = (int64_t)i;") + emit_line(" if (k < 0 || k >= __el_registry_n) return EL_STR(\"\");") + emit_line(" return EL_STR(__el_registry[k].name);") + emit_line("}") + // Timing is taken immediately around the call, in C, on the MONOTONIC + // clock — never the wall clock, which can step backwards under NTP. + emit_line("el_val_t __el_reg_invoke(el_val_t i) {") + emit_line(" int64_t k = (int64_t)i;") + emit_line(" if (k < 0 || k >= __el_registry_n) return 0;") + emit_line(" __el_cur_fails = 0; __el_cur_asserts = 0; __el_cur_msg[0] = '\\0';") + emit_line(" __el_cur_test = __el_registry[k].name;") + emit_line(" struct timespec _t0, _t1;") + emit_line(" clock_gettime(CLOCK_MONOTONIC, &_t0);") + emit_line(" __el_registry[k].fn();") + emit_line(" clock_gettime(CLOCK_MONOTONIC, &_t1);") + emit_line(" __el_last_ns = (long long)(_t1.tv_sec - _t0.tv_sec) * 1000000000LL") + emit_line(" + (long long)(_t1.tv_nsec - _t0.tv_nsec);") + emit_line(" return (el_val_t)(int64_t)__el_cur_fails;") + emit_line("}") + emit_line("el_val_t __el_reg_last_ns(void) { return (el_val_t)(int64_t)__el_last_ns; }") + emit_line("el_val_t __el_reg_msg(void) { return EL_STR(__el_cur_msg); }") + emit_line("el_val_t __el_reg_asserts(void) { return (el_val_t)(int64_t)__el_cur_asserts; }") + emit_line("el_val_t __el_opt_json(void) { return (el_val_t)(int64_t)__el_opt_json_v; }") + emit_blank() + + // main() delegates to the El-side runner. Everything above this line is + // generated glue; all reporting logic lives in runtime/eltest.el. + state_set("__cg_current_fn", "main") emit_line("int main(int _argc, char **_argv) {") emit_line(" el_runtime_init_args(_argc, _argv);") - let ti: Int = 0 - let tn: Int = native_list_len(test_c_names) - while ti < tn { - let tc_name: String = native_list_get(test_c_names, ti) - emit_line(" " + tc_name + "();") - let ti = ti + 1 - } - emit_line(" printf(\"%d passed, %d failed\\n\", __el_pass, __el_fail);") - emit_line(" return __el_fail;") + emit_line(" for (int _i = 1; _i < _argc; _i++) {") + emit_line(" if (strcmp(_argv[_i], \"--json\") == 0) __el_opt_json_v = 1;") + emit_line(" }") + emit_line(" return (int)(int64_t)el_test_main();") emit_line("}") el_arena_pop(test_arena_mark) el_release(test_names) @@ -4336,8 +4432,16 @@ fn codegen_streaming(tokens: [Any], sigs: [Map], source: String) -> // Emit main() — wrap in arena scope to free intermediate strings. let main_arena_mark: Any = el_arena_push() let kind2: String = state_get("__program_kind") + state_set("__cg_current_fn", "main") emit_line("int main(int _argc, char** _argv) {") emit_line(" el_runtime_init_args(_argc, _argv);") + // Cross-cutting concerns declared by a `program` block run BEFORE anything + // else — a singleton violation must refuse the start before this process + // touches a port or a data directory, and configuration must be validated + // before the first read of it rather than at each read site. + if prog_have { + emit_line(" __el_program_init();") + } // cgi init if needed let ns2: Int = native_list_len(sigs) @@ -4414,10 +4518,6 @@ fn codegen_streaming(tokens: [Any], sigs: [Map], source: String) -> emit_line(" return 0;") emit_line("}") emit_blank() - - emit_cap_violations() - emit_arity_violations() - emit_time_violations() el_arena_pop(main_arena_mark) "" diff --git a/lang/el-compiler/src/compiler.el b/lang/el-compiler/src/compiler.el index b9647bb..b06405c 100644 --- a/lang/el-compiler/src/compiler.el +++ b/lang/el-compiler/src/compiler.el @@ -414,11 +414,33 @@ fn parse_import_line(trimmed: String, dir: String) -> String { // Accumulates chunks into lists and joins once at the end to avoid the O(n²) // memory growth caused by repeated `prefix = prefix + chunk` concatenation. fn resolve_imports(src_path: String) -> String { + // Only the OUTERMOST call publishes provenance. Nested calls number their + // lines from 1 within themselves, so their spans are meaningless once the + // text is spliced into the parent. + let depth: String = state_get("__elc_prov_depth") + if str_eq(depth, "") { state_set("__elc_prov_depth", "1") } + let is_top: Bool = str_eq(depth, "") let seen_key: String = "__elc_imp__:" + src_path let already: String = state_get(seen_key) if !str_eq(already, "") { return "" } state_set(seen_key, "1") + // A missing file must be a hard error, never an empty string. + // + // fs_read returns "" both for "file is empty" and "file does not exist", and + // this function used the value without distinguishing them. So a broken + // import path — a typo, a moved file, a relative path resolved from the + // wrong working directory — compiled CLEANLY: exit 0, empty stderr, and a + // program silently missing everything it imported. Observed 2026-08-15: + // eleven consecutive "successful" compiles that had included no runtime at + // all, and a wrong conclusion drawn from them before anyone noticed. + // + // Missing dependency, confident success. fs_exists separates the two cases, + // so a genuinely empty file still resolves to "" and is fine. + if !fs_exists(src_path) { + println("elc: cannot resolve import: " + src_path) + exit_program(1) + } let source: String = fs_read(src_path) let dir: String = dirname_of(src_path) let lines: [String] = str_split(source, "\n") @@ -427,6 +449,7 @@ fn resolve_imports(src_path: String) -> String { // Collect chunks into lists — O(1) amortized per append. // Join once at the end — O(n) single pass. let prefix_chunks: [String] = native_list_empty() + let prefix_paths: [String] = native_list_empty() let body_chunks: [String] = native_list_empty() let i: Int = 0 while i < n { @@ -438,21 +461,54 @@ fn resolve_imports(src_path: String) -> String { // Only check .elh for imported files — never for the entry file itself. let imp_elh_path: String = str_slice(imp_path, 0, str_len(imp_path) - 3) + ".elh" let imp_elh: String = fs_read(imp_elh_path) + // Provenance: record which line range of the combined source came + // from which file, so a diagnostic can name the FILE and not just a + // line in a string that no longer exists on disk. if !str_eq(imp_elh, "") { // Header exists: mark the .el as seen (so it won't be re-inlined // if something else also imports it) and use the header text. let seen_imp_key: String = "__elc_imp__:" + imp_path state_set(seen_imp_key, "1") let prefix_chunks = native_list_append(prefix_chunks, imp_elh) + let prefix_paths = native_list_append(prefix_paths, imp_path) } else { let imp_body: String = resolve_imports(imp_path) let prefix_chunks = native_list_append(prefix_chunks, imp_body) + let prefix_paths = native_list_append(prefix_paths, imp_path) } } else { let body_chunks = native_list_append(body_chunks, line + "\n") } let i = i + 1 } + // Walk the assembled chunks once and publish spans . + // LIMIT: nested imports return a single string, so their internal + // boundaries are already lost by the time we see them -- a definition + // inside a transitively imported file is attributed to the direct import. + // Local, not accumulated in state: a nested call numbers its lines from 1 + // within itself, so letting it append to a shared buffer republishes + // meaningless spans under the parent's name. + let prov: String = "" + let line_at: Int = 1 + let ci: Int = 0 + let nchunks: Int = native_list_len(prefix_chunks) + while ci < nchunks { + let chunk: String = native_list_get(prefix_chunks, ci) + let nlines: Int = str_count_lines(chunk) + let src: String = native_list_get(prefix_paths, ci) + let prov = prov + src + " spans " + native_int_to_str(line_at) + " " + native_int_to_str(line_at + nlines - 1) + "\n" + let line_at = line_at + nlines + let ci = ci + 1 + } + let prov = prov + src_path + " spans " + native_int_to_str(line_at) + " 999999\n" + if is_top { + let prov_out: String = env("EL_RELATIONS_OUT") + if !str_eq(prov_out, "") { + let existing: String = "" + if fs_exists(prov_out) { let existing = fs_read(prov_out) } + fs_write(prov_out, existing + prov) + } + } return str_join(prefix_chunks, "") + str_join(body_chunks, "") } diff --git a/lang/el-compiler/src/lexer.el b/lang/el-compiler/src/lexer.el index 48cf5fe..358f486 100644 --- a/lang/el-compiler/src/lexer.el +++ b/lang/el-compiler/src/lexer.el @@ -138,13 +138,38 @@ fn lex_is_whitespace(ch: String) -> Bool { // tok_append — append a (kind, value) pair to a flat token list. // Returns the updated list. Gamma combines flat-list + char-code for max savings. +// A token is (kind, value, line). The line comes from state rather than a +// parameter so the ~200 existing tok_append call sites are untouched -- the +// lexer advances __lex_line as it walks, and every token minted takes the line +// it was minted on. +// +// WHY AT ALL: before this a token carried no position, so no diagnostic in El +// could name a place. Every error named a symbol and never a line, and after +// textual inlining there was no way to say which FILE a definition came from. fn tok_append(tokens: [Any], kind: String, value: String) -> [Any] { let tokens = native_list_append(tokens, kind) - native_list_append(tokens, value) + let tokens = native_list_append(tokens, value) + native_list_append(tokens, state_get("__lex_line")) } // -- Keyword lookup ------------------------------------------------------------ +// keyword_kind — the language's reserved spellings. +// +// A grammar is a BASIS: `fn` means function-start because someone said so, and +// nothing derives it. But unlike the other tables moved out this session, this +// one stays code, and the SHOULD gate is why. The keyword set is closed by the +// language definition -- it does not leak the way an allowlist does -- and the +// lexer runs before the program is understood, so a program can never declare +// its own keywords. Externalising it would cost file I/O on every compile and +// buy nothing. +// +// Removed 2026-08-17: sealed, activate, seed, protocol, impl. Reserved in the +// lexer, consumed by no parser or codegen path, and each one stole an +// identifier from users for nothing. `test` LOOKED inert by the same measure +// and is not -- codegen consumes it at 4135 for --test mode, 408 uses in the +// tree. The first measurement checked only parser.el and would have broken all +// of them. fn keyword_kind(word: String) -> String { if word == "let" { return "Let" } if word == "fn" { return "Fn" } @@ -161,14 +186,9 @@ fn keyword_kind(word: String) -> String { if word == "from" { return "From" } if word == "as" { return "As" } if word == "with" { return "With" } - if word == "sealed" { return "Sealed" } - if word == "activate" { return "Activate" } if word == "where" { return "Where" } if word == "test" { return "Test" } - if word == "seed" { return "Seed" } if word == "assert" { return "Assert" } - if word == "protocol" { return "Protocol" } - if word == "impl" { return "Impl" } if word == "retry" { return "Retry" } if word == "times" { return "Times" } if word == "fallback" { return "Fallback" } @@ -184,6 +204,7 @@ fn keyword_kind(word: String) -> String { if word == "false" { return "Bool" } if word == "cgi" { return "Cgi" } if word == "service" { return "Service" } + if word == "program" { return "Program" } if word == "manager" { return "Manager" } if word == "engine" { return "Engine" } if word == "accessor" { return "Accessor" } @@ -520,6 +541,12 @@ fn scan_interp_brace(src: String, start: Int, total: Int) -> Map { // interp_tokens_append_all - copy every (kind, value) pair from flat src list // into flat dst list, skipping the trailing Eof pair that lex() always appends. +// Splices re-lexed interpolation tokens into the stream. This walks the token +// list DIRECTLY rather than through tok_append, so it carries its own copy of +// the stride -- which is why giving tokens a line broke the compiler's second +// generation and not its first: the compiler's own source uses string +// interpolation, so gen1 (built by the old compiler) was fine and gen2 emitted +// a corrupted stream. fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] { let src_len: Int = native_list_len(src) let j = 0 @@ -530,9 +557,11 @@ fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] { let j = src_len } else { let val: String = native_list_get(src, j + 1) + let ln: String = native_list_get(src, j + 2) let result = native_list_append(result, kind) let result = native_list_append(result, val) - let j = j + 2 + let result = native_list_append(result, ln) + let j = j + 3 } } result @@ -763,8 +792,14 @@ fn lex(source: String) -> [Any] { let total: Int = str_len(source) let tokens: [Any] = native_list_empty() let i: Int = 0 + state_set("__lex_line", "1") + let line_no: Int = 1 while i < total { + if str_eq(str_slice(source, i, i + 1), "\n") { + let line_no = line_no + 1 + state_set("__lex_line", native_int_to_str(line_no)) + } let c: Int = str_char_code(source, i) // Skip whitespace (space=32, tab=9, newline=10, CR=13) diff --git a/lang/el-compiler/src/parser.el b/lang/el-compiler/src/parser.el index 7936e09..f72015f 100644 --- a/lang/el-compiler/src/parser.el +++ b/lang/el-compiler/src/parser.el @@ -17,8 +17,8 @@ // programs. All callers use these helpers -- only these three need updating. fn tok_at(tokens: [Any], pos: Int) -> Map { - let kind: String = native_list_get(tokens, pos * 2) - let value: String = native_list_get(tokens, pos * 2 + 1) + let kind: String = native_list_get(tokens, pos * 3) + let value: String = native_list_get(tokens, pos * 3 + 1) { "kind": kind, "value": value } } @@ -28,25 +28,32 @@ fn tok_kind(tokens: [Any], pos: Int) -> String { // single trailing Eof token returns runtime null (el_list_get OOB -> 0), // which matches no delimiter, letting inner parse loops append AST nodes // forever on malformed input -> unbounded allocation -> OOM. - let n: Int = native_list_len(tokens) / 2 + let n: Int = native_list_len(tokens) / 3 if pos < 0 { return "Eof" } if pos >= n { return "Eof" } - native_list_get(tokens, pos * 2) + native_list_get(tokens, pos * 3) +} + +fn tok_line(tokens: [Any], pos: Int) -> String { + let n: Int = native_list_len(tokens) / 3 + if pos < 0 { return "0" } + if pos >= n { return "0" } + native_list_get(tokens, pos * 3 + 2) } fn tok_value(tokens: [Any], pos: Int) -> String { - let n: Int = native_list_len(tokens) / 2 + let n: Int = native_list_len(tokens) / 3 if pos < 0 { return "" } if pos >= n { return "" } - native_list_get(tokens, pos * 2 + 1) + native_list_get(tokens, pos * 3 + 1) } // parse_progress_fatal — robustness backstop. Called by the token-consuming @@ -1230,7 +1237,7 @@ fn parse_block(tokens: [Any], pos: Int) -> Map { // Runaway backstop: a block can hold at most (token count) statements, since // every iteration consumes >= 1 token. If we exceed that, the cursor has run // off the end without terminating (malformed input) -> fail fast, don't hang. - let blk_total: Int = native_list_len(tokens) / 2 + let blk_total: Int = native_list_len(tokens) / 3 let blk_iters: Int = 0 while running { let blk_iters = blk_iters + 1 @@ -1550,7 +1557,10 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { let p = r2["pos"] // r2 result map fully consumed — release to free peak heap. el_release(r2) - return make_result({ "stmt": "FnDef", "name": name, "params": params, "body": body, "ret_type": ret_type }, p) + // The definition carries the line it was written on. Without it no + // diagnostic can name a place, and after textual inlining there is no + // way to say which FILE a definition came from. + return make_result({ "stmt": "FnDef", "name": name, "params": params, "body": body, "ret_type": ret_type, "line": tok_line(tokens, pos) }, p) } // type definition: `type Name = { field: Type, ... }` @@ -1842,6 +1852,7 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { "params": inner["params"], "body": inner["body"], "ret_type": inner["ret_type"], + "line": inner["line"], "decorator": dec_name, "decorators": dlist } @@ -1967,6 +1978,142 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { }, p) } + // program block: program "name" { singleton: "id", env NAME: Type = "default", ... } + // + // The program block is El's declaration surface for CROSS-CUTTING CONCERNS — + // properties of the whole process rather than of any one function, which + // otherwise degrade into "remember to call this at every site" conventions. + // + // singleton: "id" — process identity. The runtime takes an exclusive + // lock at startup; a SECOND start is refused, loudly, + // instead of two processes sharing one data dir. + // guards: — WHAT that singleton protects: an expression yielding + // the path of the guarded state directory, evaluated at + // startup. MANDATORY with `singleton:`, because a lock + // keyed on a program's NAME rather than on its STATE is + // not a guard — measured 2026-08-16, the name-keyed + // version refused unrelated instances (different data + // dirs) AND permitted concurrent ones (same data dir, + // different $TMPDIR). It is an expression and not a + // string so a program can point at the resolver that + // already OWNS the path (§18.4) instead of restating + // its default here, which would give the path two + // owners that can disagree. + // env NAME: T = "d" — one configuration entry. Its type and its default + // are declared ONCE, here, and resolved+validated + // before main() body runs. + // env NAME: T required + // — no default; the program refuses to start unless the + // variable is set. + // + // Both compile into calls injected at the head of main() — the same boundary + // seam `cgi` already uses (codegen.el emit_program_init). No call site in the + // program body has to remember anything, which is the whole point. + if k == "Program" { + let p = pos + 1 + let name = tok_value(tokens, p) + let p = p + 1 + let p = expect(tokens, p, "LBrace") + let singleton = "" + let has_singleton = false + let guards_node = { "expr": "Str", "value": "" } + let has_guards = false + let entries = native_list_empty() + // Entry-scratch declared at loop-body level (not inside the branch) so + // that inner `let` forms compile to assignment rather than a C-scoped + // redeclaration — the same idiom the service block above relies on. + let ename = "" + let etype = "" + let edefault = "" + let has_default = false + let erequired = false + let fname = "" + let fval = "" + let running = true + while running { + let k2 = tok_kind(tokens, p) + if k2 == "RBrace" { + let running = false + } else { + if k2 == "Eof" { + let running = false + } else { + let fname = tok_value(tokens, p) + let p = p + 1 + if str_eq(fname, "env") { + // env NAME: Type [= "default"] [required] + let ename = tok_value(tokens, p) + let p = p + 1 + let p = expect(tokens, p, "Colon") + let etype = tok_value(tokens, p) + let p = p + 1 + let edefault = "" + let has_default = false + let erequired = false + let k3 = tok_kind(tokens, p) + if str_eq(k3, "Eq") { + let p = p + 1 + let edefault = tok_value(tokens, p) + let has_default = true + let p = p + 1 + } + let k4 = tok_kind(tokens, p) + if str_eq(k4, "Ident") { + let w = tok_value(tokens, p) + if str_eq(w, "required") { + let erequired = true + let p = p + 1 + } + } + let entries = native_list_append(entries, { + "name": ename, + "etype": etype, + "default": edefault, + "has_default": has_default, + "required": erequired + }) + } else { + if str_eq(fname, "guards") { + // guards: — the STATE the singleton protects. + // Parsed as a full expression, not a string literal, so + // it can name the resolver that owns the path + // (`guards: engram_resolve_data_dir()`) rather than + // duplicating that resolver's default here. + let p = expect(tokens, p, "Colon") + let g_r = parse_expr(tokens, p) + let guards_node = g_r["node"] + let p = g_r["pos"] + let has_guards = true + } else { + // scalar field: `name: "value"` + let p = expect(tokens, p, "Colon") + let fval = tok_value(tokens, p) + let p = p + 1 + if str_eq(fname, "singleton") { + let singleton = fval + let has_singleton = true + } + } + } + let k5 = tok_kind(tokens, p) + if k5 == "Comma" { + let p = p + 1 + } + } + } + } + let p = expect(tokens, p, "RBrace") + return make_result({ + "stmt": "ProgramBlock", + "name": name, + "singleton": singleton, + "has_singleton": has_singleton, + "guards": guards_node, + "has_guards": has_guards, + "entries": entries + }, p) + } + // assert [ , ] // The message is optional — if the next token after the condition is not a // Comma, emit an empty string placeholder so the test still works. @@ -2022,7 +2169,7 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map { fn parse(tokens: [Any]) -> [Map] { // Flat list: 2 entries per token, so divide by 2 for token count. - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let stmts: [Map] = native_list_empty() let pos: Int = 0 let running = true @@ -2065,7 +2212,7 @@ fn parse_one(tokens: [Any], pos: Int) -> Map { // On entry, pos must point at the LBrace token. // Returns the position of the token AFTER the matching RBrace. fn skip_to_rbrace(tokens: [Any], pos: Int) -> Int { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let p: Int = pos + 1 let depth: Int = 1 let going: Bool = true @@ -2117,7 +2264,7 @@ fn is_stmt_start_kind(k: String) -> Bool { // token that could start a new top-level statement, staying depth-aware // so that braces inside expressions don't fool us. fn skip_expr_to_stmt_boundary(tokens: [Any], pos: Int) -> Int { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let p: Int = pos let depth: Int = 0 let going: Bool = true @@ -2283,7 +2430,7 @@ fn scan_params_el(tokens: [Any], pos: Int) -> Map { // // Peak memory: O(tokens) with no expression AST allocation. fn scan_fn_sigs_el(tokens: [Any]) -> [Map] { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let sigs: [Map] = native_list_empty() let pos: Int = 0 let going: Bool = true @@ -2419,12 +2566,13 @@ fn scan_params_c(tokens: [Any], pos: Int) -> Map { // toplevel_let: { "kind": "toplevel_let", "name": String, "ltype": String } // cgi_block: { "kind": "cgi_block", "name": String } // service_block: { "kind": "service_block", "name": String } +// program_block: { "kind": "program_block", "name": String } // // Import/TypeDef/EnumDef nodes are skipped (codegen treats them as no-ops). // // The scan allocates only small string values per entry, keeping peak RSS low. fn scan_fn_sigs(tokens: [Any]) -> [Map] { - let total: Int = native_list_len(tokens) / 2 + let total: Int = native_list_len(tokens) / 3 let sigs: [Map] = native_list_empty() let pos: Int = 0 let going: Bool = true @@ -2546,13 +2694,28 @@ fn scan_fn_sigs(tokens: [Any]) -> [Map] { "name": name }) let pos = p + } else { + // --- program block --- + if str_eq(k, "Program") { + let p: Int = pos + 1 + let name: String = tok_value(tokens, p) + let p = p + 1 + let k2: String = tok_kind(tokens, p) + if str_eq(k2, "LBrace") { + let p = skip_to_rbrace(tokens, p) + } + let sigs = native_list_append(sigs, { + "kind": "program_block", + "name": name + }) + let pos = p } else { // Import, Type, Enum, From, or any other token. // Skip ahead to the next statement boundary. let p: Int = pos + 1 let p = skip_expr_to_stmt_boundary(tokens, p) let pos = p - }}}}} + }}}}}} } } } diff --git a/lang/examples/transduce.el b/lang/examples/transduce.el new file mode 100644 index 0000000..8a18ce6 --- /dev/null +++ b/lang/examples/transduce.el @@ -0,0 +1,228 @@ +// transduce.el — transduction decomposes a signal into components and the +// relations between them. Runnable: this is the worked example for the +// transduce surface, and it exits non-zero if any claim in it stops being true. +// +// elc lang/examples/transduce.el > transduce.c +// cc -std=c11 -O2 -I lang/runtime -o transduce transduce.c \ +// lang/runtime/el_runtime.c lang/runtime/el_seed.c \ +// lang/runtime/engram_store.c lang/runtime/engram_vindex.c \ +// lang/runtime/engram_cognition.c lang/runtime/engram_geometry.c \ +// lang/runtime/engram_reason.c lang/runtime/engram_verify.c \ +// -lcurl -lpthread -lm +// ./transduce # exits 0 only if every check passes +// +// It writes to an IN-MEMORY engram (leave ENGRAM_STORE unset) and contacts no +// server. The same claims are asserted by the native harness in +// lang/tests/native/test_transduce.el. +// +// WHAT CHANGED, AND WHY IT MATTERS. #144 shipped +// `transduce(signal, modality) -> Geometry`: one vector per signal. That made +// transduction a CONVERSION — take a thing, encode it, store a position — and +// what a conversion returns is a fingerprint. A fingerprint can be matched and +// ranked, and that is all it can ever do. It cannot be decomposed, cannot have +// one part grounded while another is not, and cannot be contradicted in one +// part while holding in another, because it has no parts. +// +// A song is not a point. It decomposes into pitch, interval, rhythm, harmonic +// function — components, each with its own geometry, plus the relations among +// them. THE SONG IS THE STRUCTURE OF THE RELATIONS. transduce now returns a +// Manifold, and a realizer's job is to say what its modality's components ARE. + +fn check(ok: Int, label: String) -> Int { + if ok > 0 { + println(" ok " + label) + return 0 + } + println(" FAIL " + label) + exit(1) + return 1 +} + +fn near(a: Float, b: Float) -> Int { + let d: Float = a - b + if d > 0.001 { return 0 } + if d < -0.001 { return 0 } + return 1 +} + +fn eq_int(a: Int, b: Int) -> Int { + if a == b { return 1 } + return 0 +} + +// ── A DECOMPOSING realizer, written entirely in El ────────────────────────── +// "tone" signals are note letters, e.g. "CEG". This does NOT return one vector +// for the chord. It returns the PARTS — one component per note, one per +// interval between adjacent notes — and the relations that make those parts a +// chord rather than an unordered bag of pitches. +// +// The interval is deliberately a COMPONENT, not a field on a note. An interval +// is a thing with its own geometry belonging to neither endpoint; modelling it +// as an attribute of one of them is the same collapse, one level down. +fn tone_realizer(signal: String) -> Manifold { + let m: Manifold = manifold_new() + let n: Int = str_len(signal) + let i: Int = 0 + while i < n { + let code: Int = str_char_code(signal, i) + let g: Geometry = geometry_new(2) + let s0: Int = geometry_set(g, 0, int_to_float(code)) + let s1: Int = geometry_set(g, 1, int_to_float(i)) + let idx: Int = manifold_add(m, "note:" + int_to_str(i), "pitch", g) + let f: Int = geometry_free(g) + i = i + 1 + } + let j: Int = 1 + while j < n { + let a: Int = str_char_code(signal, j - 1) + let b: Int = str_char_code(signal, j) + let lo: String = "note:" + int_to_str(j - 1) + let hi: String = "note:" + int_to_str(j) + let key: String = "interval:" + int_to_str(j - 1) + "-" + int_to_str(j) + let g: Geometry = geometry_new(1) + let s: Int = geometry_set(g, 0, int_to_float(b - a)) + let idx: Int = manifold_add(m, key, "interval", g) + let f: Int = geometry_free(g) + let e1: Int = manifold_relate(m, key, "spans", lo, 0.9) + let e2: Int = manifold_relate(m, key, "spans", hi, 0.9) + let e3: Int = manifold_relate(m, lo, "sounds_before", hi, 0.8) + j = j + 1 + } + m +} + +// #144's contract, kept as a control: one vector for the whole signal. +fn fingerprint_realizer(signal: String) -> Geometry { + let g: Geometry = geometry_new(4) + let n: Int = str_len(signal) + let a: Int = geometry_set(g, 0, int_to_float(n)) + g +} + +fn main() -> Void { + println("a realizer declared in El is a first-class realizer") + let reg: Int = realizer_register("tone", "tone_realizer") + let _c: Int = check(reg, "an El fn registers as a realizer by name") + let _c: Int = check(realizer_has("tone"), "the modality now has an organ") + + println("transduction decomposes a signal into parts") + let m: Manifold = transduce("CEG", "tone") + let _c: Int = check(manifold_is(m), "transduce returns a real Manifold") + let sz: Int = manifold_size(m) + let _c: Int = check(eq_int(sz, 5), "three notes and two intervals are five parts") + let rc: Int = manifold_rel_count(m) + let _c: Int = check(eq_int(rc, 6), "and they stand in six stated relations") + + println("every part is addressable BY KEY, which is what survives persistence") + let i_c: Int = manifold_index_of(m, "note:0") + let _c: Int = check(1 - eq_int(i_c, -1), "the first note is addressable on its own") + let i_iv: Int = manifold_index_of(m, "interval:0-1") + let _c: Int = check(1 - eq_int(i_iv, -1), "so is the interval between the first two") + let miss: Int = manifold_index_of(m, "never_added") + let _c: Int = check(eq_int(miss, -1), "an unknown key is -1, not component 0") + + println("parts carry their own geometry, and may differ in width") + let gn: Geometry = manifold_geometry(m, i_c) + let _c: Int = check(eq_int(geometry_dim(gn), 2), "a note component is 2 wide") + let _c: Int = check(near(geometry_get(gn, 0), 67.0), "and it is C — the signal reached the realizer") + let gi: Geometry = manifold_geometry(m, i_iv) + let _c: Int = check(eq_int(geometry_dim(gi), 1), "an interval component is 1 wide") + // A single vector per signal cannot represent parts of unequal width at all. + let _c: Int = check(near(geometry_get(gi, 0), 2.0), "C to E is two semitones") + let f1: Int = geometry_free(gn) + let f2: Int = geometry_free(gi) + + println("the relations are content no single part carries") + // That "2" above is not a property of C and not a property of E. It exists + // only BETWEEN them, so a representation with no relations cannot hold it. + let spans: Int = 0 + let k: Int = 0 + while k < rc { + if str_eq(manifold_rel_name(m, k), "spans") { + if str_eq(manifold_rel_from(m, k), "interval:0-1") { spans = spans + 1 } + } + k = k + 1 + } + let _c: Int = check(eq_int(spans, 2), "the interval is wired to both notes it spans") + + println("relation weight IS the grounding (correspondence-and-censorship §1)") + let wk: Int = 0 + let found: Int = 0 + while wk < rc { + if str_eq(manifold_rel_name(m, wk), "sounds_before") { + if near(manifold_rel_weight(m, wk), 0.8) > 0 { found = 1 } + } + wk = wk + 1 + } + let _c: Int = check(found, "the ordering relation carries the weight its realizer stated") + + println("the decomposition persists as real, separately addressable nodes") + let ids: [String] = el_list_empty() + let n0: Int = engram_node_count() + let e0: Int = engram_edge_count() + let pi: Int = 0 + while pi < sz { + let key: String = manifold_key(m, pi) + let g: Geometry = manifold_geometry(m, pi) + let id: String = engram_node("component " + key, "Concept", 0.6) + let att: Int = node_attach_geometry(id, g) + ids = el_list_append(ids, id) + let ff: Int = geometry_free(g) + pi = pi + 1 + } + let ri: Int = 0 + while ri < rc { + let fi: Int = manifold_index_of(m, manifold_rel_from(m, ri)) + let ti: Int = manifold_index_of(m, manifold_rel_to(m, ri)) + engram_connect(el_list_get(ids, fi), el_list_get(ids, ti), + manifold_rel_weight(m, ri), manifold_rel_name(m, ri)) + ri = ri + 1 + } + let _c: Int = check(eq_int(engram_node_count() - n0, 5), "one signal became five nodes") + let _c: Int = check(eq_int(engram_edge_count() - e0, 6), "and six edges between them") + + println("each part's geometry is independently readable back off its node") + let id_c: String = el_list_get(ids, manifold_index_of(m, "note:0")) + let id_iv: String = el_list_get(ids, manifold_index_of(m, "interval:0-1")) + let _c: Int = check(eq_int(node_geometry_dim(id_c), 2), "note:0 node carries a 2-wide geometry") + let _c: Int = check(eq_int(node_geometry_dim(id_iv), 1), "interval:0-1 node carries a 1-wide one") + + println("one part can be grounded without touching its siblings") + let ear: String = engram_node("evidence: heard a C in the recording", "Memory", 0.7) + engram_connect(ear, id_c, 0.95, "corroborates") + let _c: Int = check(engram_edge_between(ear, id_c), "evidence attaches to note:0 specifically") + let id_g: String = el_list_get(ids, manifold_index_of(m, "note:2")) + let _c: Int = check(1 - engram_edge_between(ear, id_g), "and NOT to note:2 — the sibling is untouched") + // This is the whole gain, and it is impossible with a fingerprint: with one + // node per signal, "the C is corroborated" and "the G is not" have the same + // grounding target and cannot both be recorded. + let _c: Int = check(eq_int(node_geometry_dim(id_g), 2), "note:2 geometry is intact regardless") + + println("a fingerprint realizer transduces NOTHING") + // #144's contract exactly: signal in, one Geometry out. It resolves, so the + // organ is present — but it does not decompose, so it does not transduce. + // "No organ" and "an organ that only fingerprints" must not look alike. + let rf: Int = realizer_register("fingerprint", "fingerprint_realizer") + let _c: Int = check(rf, "the symbol resolves, so registration succeeds") + let mf: Manifold = transduce("x", "fingerprint") + let _c: Int = check(1 - manifold_is(mf), "a single vector is not a transduction") + + println("the one-part case is a size-one manifold, not a bare vector") + let g1: Geometry = geometry_new(3) + let s1: Int = geometry_set(g1, 0, 5.0) + let ms: Manifold = manifold_single("level", "scalar", g1) + let _c: Int = check(manifold_is(ms), "manifold_single yields a real Manifold") + let _c: Int = check(eq_int(manifold_size(ms), 1), "of size one — visibly degenerate, not hidden") + let fg: Int = geometry_free(g1) + let fs: Int = manifold_free(ms) + + println("no organ is still reported as no organ") + let me: Manifold = transduce("anything", "echolocation") + let _c: Int = check(1 - manifold_is(me), "no realizer means no manifold, not a fake one") + + let fm: Int = manifold_free(m) + + // Reaching here means nothing called exit(1) along the way. + println("") + println("all checks passed") +} diff --git a/lang/install.sh b/lang/install.sh index 9604ec3..ac895b6 100644 --- a/lang/install.sh +++ b/lang/install.sh @@ -49,21 +49,49 @@ download() { TMP_DIR="$(mktemp -d)" trap 'rm -rf "${TMP_DIR}"' EXIT -download "${RELEASE_BASE}/elc" "${TMP_DIR}/elc" -download "${RELEASE_BASE}/el_runtime.c" "${TMP_DIR}/el_runtime.c" -download "${RELEASE_BASE}/el_runtime.h" "${TMP_DIR}/el_runtime.h" +# The runtime is MULTI-FILE. el_runtime.c #includes six engram headers and makes +# hard cross-TU calls into all six sibling .c files, so installing el_runtime.c +# alone produces a lib/ that CANNOT LINK — `ld` fails with undefined +# engram_ground_json / engram_activate_inner / eg_find_relation / cog_assert_two_axis. +# This list mirrors lang/runtime/SOURCES (the in-repo source of truth); keep them +# in step. install.sh is standalone by design — it runs on machines with no repo +# checkout — so it cannot call scripts/el-runtime-sources.sh. +RUNTIME_SOURCES=( + el_runtime.c el_seed.c + engram_store.c engram_vindex.c engram_geometry.c + engram_reason.c engram_verify.c engram_cognition.c + engram_text.c + eg_cosine_batch.c eg_cosine_batch_strategy_cpu.c +) +RUNTIME_HEADERS=( + el_runtime.h el_seed.h + engram_store.h engram_vindex.h engram_geometry.h + engram_reason.h engram_verify.h engram_cognition.h + engram_text.h + eg_cosine_batch.h eg_cosine_batch_strategy.h +) + +download "${RELEASE_BASE}/elc" "${TMP_DIR}/elc" +for f in "${RUNTIME_SOURCES[@]}" "${RUNTIME_HEADERS[@]}"; do + download "${RELEASE_BASE}/${f}" "${TMP_DIR}/${f}" +done # Install -install -m 755 "${TMP_DIR}/elc" "${BIN_DIR}/elc" -install -m 644 "${TMP_DIR}/el_runtime.c" "${LIB_DIR}/el_runtime.c" -install -m 644 "${TMP_DIR}/el_runtime.h" "${LIB_DIR}/el_runtime.h" +install -m 755 "${TMP_DIR}/elc" "${BIN_DIR}/elc" +for f in "${RUNTIME_SOURCES[@]}" "${RUNTIME_HEADERS[@]}"; do + install -m 644 "${TMP_DIR}/${f}" "${LIB_DIR}/${f}" +done + +# Record the link set so downstream Makefiles can read it instead of hardcoding. +printf '%s\n' "${RUNTIME_SOURCES[@]}" > "${TMP_DIR}/SOURCES" +install -m 644 "${TMP_DIR}/SOURCES" "${LIB_DIR}/SOURCES" echo echo "==> El SDK installed successfully" echo echo " elc binary : ${BIN_DIR}/elc" -echo " runtime : ${LIB_DIR}/el_runtime.c" -echo " header : ${LIB_DIR}/el_runtime.h" +echo " runtime : ${LIB_DIR}/ (${#RUNTIME_SOURCES[@]} .c files, ${#RUNTIME_HEADERS[@]} headers)" +echo " link set : ${LIB_DIR}/SOURCES" echo echo "Add the following to your Makefile to build El programs:" echo @@ -71,10 +99,14 @@ echo " EL_LIB := ${LIB_DIR}" echo " ELC := elc" echo " CC := cc" echo " CFLAGS := -std=c11 -O2 -I\$(EL_LIB)" +echo " LDLIBS := -lcurl -lssl -lcrypto -lpthread -lm" +echo +echo " # The runtime is multi-file — link the whole set, not el_runtime.c alone." +echo " EL_RUNTIME := \$(addprefix \$(EL_LIB)/,\$(shell cat \$(EL_LIB)/SOURCES))" echo echo " dist/myapp.c: src/myapp.el" echo " \t\$(ELC) src/myapp.el > dist/myapp.c" echo echo " dist/myapp: dist/myapp.c" -echo " \t\$(CC) \$(CFLAGS) -o dist/myapp dist/myapp.c \$(EL_LIB)/el_runtime.c -lcurl -lpthread" +echo " \t\$(CC) \$(CFLAGS) -o dist/myapp dist/myapp.c \$(EL_RUNTIME) \$(LDLIBS)" echo diff --git a/lang/runtime/BUDGET b/lang/runtime/BUDGET new file mode 100644 index 0000000..6706a5d --- /dev/null +++ b/lang/runtime/BUDGET @@ -0,0 +1,41 @@ +# BUDGET — a RATCHET on lang/runtime/el_runtime.c. Enforced by +# scripts/check-runtime-growth.sh. These numbers may only ever go DOWN. +# +# WHY THIS FILE EXISTS +# -------------------- +# scripts/check-single-runtime.sh guards against el_runtime.c being COPIED. +# Nothing guarded against it GROWING. It grew from 10,607 lines to 20,527 — +# 94% — in 3.5 months, while under an explicit commit-message promise that it +# was a temporary shim about to be deleted. +# +# It grew because lang/AGENTS.md told every agent to grow it: it claimed +# el_runtime.c was "the authoritative single-file link target" and that a new +# C builtin "must live there to be linkable". That is false — placement is a +# link-time concern, `builtin_arity` is an arity guard not a dispatch table, +# and the shipped elc already links from ten translation units. The claim is +# corrected, and this file is the mechanism that keeps it corrected. +# +# THIS IS A RATCHET, NOT A LIMIT +# ------------------------------ +# The budget is set at the CURRENT size. There is no headroom, deliberately. +# The file cannot grow by even one line. Any new code goes in the .c that owns +# the concern — that is the whole point, and every other runtime file is +# deliberately UNCAPPED. +# +# When you move code OUT, lower the number in the same commit. The guard tells +# you to when you have earned it. +# +# FORMAT: — `#` comments and blank lines ignored. + +# Maximum lines in lang/runtime/el_runtime.c. +# 2026-08-16: 20,527 — the high-water mark. +# 2026-08-16: 20,427 — engram_text.c extracted (tokenize, token hygiene, +# word-boundary match, damage signature). Ratcheted down. +max_lines 20427 + +# Maximum top-level engram/eg_/cog_ function definitions in el_runtime.c. +# ~47.5% of the file is engram code, and engram already owns six dedicated +# sibling files (engram_{store,vindex,geometry,reason,verify,cognition}.c). +# Every one of these belongs in one of them. This is the Stage 3 scoreboard. +# 2026-08-16: 279 -> 275 (4 moved to engram_text.c). +max_engram_fns 275 diff --git a/lang/runtime/SOURCES b/lang/runtime/SOURCES new file mode 100644 index 0000000..d825445 --- /dev/null +++ b/lang/runtime/SOURCES @@ -0,0 +1,57 @@ +# SOURCES — the canonical El runtime link set. +# +# THIS FILE IS THE SINGLE SOURCE OF TRUTH for "what do I compile and link to +# get the El runtime". Every build path — CI, install.sh, the SDK release, the +# docs, elb, the engram test harnesses — reads it via scripts/el-runtime-sources.sh +# instead of hardcoding its own list. +# +# WHY THIS FILE EXISTS +# -------------------- +# The runtime has been multi-translation-unit since the engram siblings landed: +# el_runtime.c #includes engram_{store,vindex,geometry,reason,verify,cognition}.h +# and makes hard cross-TU calls into all six. Linking el_runtime.c ALONE has been +# broken since then — `ld` fails with undefined symbols (engram_ground_json, +# engram_activate_inner, eg_find_relation, cog_assert_two_axis, ...). +# +# It stayed broken because the link set was written out longhand in ~8 different +# places, each of which drifted independently. A list copied 8 times is a list +# that is wrong in 8 places. It is now written once, here. +# +# HOW TO USE IT +# ------------- +# scripts/el-runtime-sources.sh # bare names, one per line +# scripts/el-runtime-sources.sh lang/runtime # prefixed with a directory +# cc ... $(scripts/el-runtime-sources.sh lang/runtime) -lcurl -lssl -lcrypto -lpthread -lm +# +# ADDING A FILE +# ------------- +# Add the .c here and it is picked up by every build path at once. That is the +# point: a new concern gets its own translation unit and costs one line, instead +# of being appended to el_runtime.c because appending was the cheaper edit. +# +# Order is link order. Blank lines and `#` comments are ignored. + +# --- EL core language runtime ------------------------------------------------- +el_runtime.c +el_seed.c + +# --- Engram: store, index, geometry, reasoning, verification, cognition ------- +# These are the six concern-owned translation units el_runtime.c calls into. +engram_store.c +engram_vindex.c +engram_geometry.c +engram_reason.c +engram_verify.c +engram_cognition.c + +# --- Text: tokenization, token hygiene, damage signature --------------------- +# Extracted from el_runtime.c 2026-08-16. Plain C over / — +# touches no EL value type and no engram store type. New text helpers go HERE. +engram_text.c + +# --- Vector math: batch cosine + its CPU strategy ---------------------------- +# The ggml strategy (eg_cosine_batch_strategy_ggml.c) is an OPTIONAL swap-in and +# is deliberately NOT in the default set — it needs ggml headers. Link it in +# place of the cpu strategy when you have them. +eg_cosine_batch.c +eg_cosine_batch_strategy_cpu.c diff --git a/lang/runtime/build_vindex_bench.sh b/lang/runtime/build_vindex_bench.sh new file mode 100755 index 0000000..30076c1 --- /dev/null +++ b/lang/runtime/build_vindex_bench.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# build_vindex_bench.sh — build the vindex_bench oracle/proof harness, with +# the real ggml + hand-rolled-Metal batch-cosine strategies on Darwin and a +# zero-dependency CPU-only stub everywhere else. Mirrors the two-step recipe +# documented in vindex_bench.c's own header comment; this script exists so +# that recipe is one command, not a copy-pasted paragraph. +# +# Darwin build links FOUR strategy translation units: +# eg_cosine_batch.c — the Factory (always) +# eg_cosine_batch_strategy_cpu.c — universal fallback (always) +# eg_cosine_batch_strategy_ggml.c — ggml + dynamic Metal backend plugin +# eg_cosine_batch_strategy_metal_hand.m — PR #114's original hand-rolled +# Metal shader, preserved as one +# selectable strategy +# plus -DEG_HAVE_STRATEGY_GGML -DEG_HAVE_STRATEGY_METAL_HAND so the Factory +# (and vindex_bench.c's own direct strategy comparison) knows both exist. +# +# ggml is resolved via `brew --prefix ggml` when available (portable across +# Intel /usr/local and Apple Silicon /opt/homebrew installs), falling back to +# /opt/homebrew if brew isn't on PATH. Override with GGML_PREFIX=... env var. +# +# Usage: ./build_vindex_bench.sh [output_path] +set -euo pipefail +cd "$(dirname "$0")" +OUT="${1:-./vindex_bench}" +CC="${CC:-cc}" + +if [ "$(uname -s)" = "Darwin" ]; then + GGML_PREFIX="${GGML_PREFIX:-$(brew --prefix ggml 2>/dev/null || echo /opt/homebrew)}" + echo "== Darwin: building with the ggml + hand-rolled-Metal strategies (ggml prefix: $GGML_PREFIX) ==" + + "$CC" -O2 -std=c11 -x objective-c \ + -c eg_cosine_batch_strategy_metal_hand.m -o /tmp/eg_cosine_batch_strategy_metal_hand.o \ + -framework Metal -framework Foundation + + "$CC" -O2 -std=c11 -DEG_HAVE_STRATEGY_GGML -DEG_HAVE_STRATEGY_METAL_HAND -w \ + -I"$GGML_PREFIX/include" \ + vindex_bench.c engram_vindex.c \ + eg_cosine_batch.c eg_cosine_batch_strategy_cpu.c eg_cosine_batch_strategy_ggml.c \ + /tmp/eg_cosine_batch_strategy_metal_hand.o \ + -L"$GGML_PREFIX/lib" -lggml -lggml-base \ + -Wl,-rpath,"$GGML_PREFIX/lib" \ + -lm -framework Metal -framework Foundation -o "$OUT" +else + echo "== non-Darwin: building with the CPU-only fallback strategy (no ggml, no Metal) ==" + "$CC" -O2 -std=c11 -w vindex_bench.c engram_vindex.c \ + eg_cosine_batch.c eg_cosine_batch_strategy_cpu.c \ + -lm -o "$OUT" +fi + +echo "built: $OUT" diff --git a/lang/runtime/eg_cosine_batch.c b/lang/runtime/eg_cosine_batch.c new file mode 100644 index 0000000..8c92926 --- /dev/null +++ b/lang/runtime/eg_cosine_batch.c @@ -0,0 +1,121 @@ +/* eg_cosine_batch.c — the Factory. Implements the stable public interface + * declared in eg_cosine_batch.h by selecting ONE concrete + * EgCosineBatchStrategy (eg_cosine_batch_strategy.h) and dispatching every + * call to it. This is the ONLY file that branches on EG_HAVE_STRATEGY_* + * (build-time: which strategy .c/.m files were actually compiled in for + * this platform) — call sites never see those macros. + * + * Selection is lazy (first call) and cached — mirrors the lazy-init caching + * every individual strategy already does internally, so there is no added + * per-call cost after the first. + * + * Selection mechanism (env var + build-time + runtime capability probe, all + * three, exactly as directed): + * - BUILD-TIME decides which strategies exist to choose from at all: a + * Darwin build compiles+links the ggml strategy and the hand-rolled + * Metal strategy (EG_HAVE_STRATEGY_GGML / EG_HAVE_STRATEGY_METAL_HAND + * both defined); a non-Darwin build compiles neither, matching PR #114's + * original Linux behavior exactly (CPU-fallback only, no Objective-C + * compiler or Metal frameworks required). + * - RUNTIME CAPABILITY PROBE: each candidate strategy's own available() + * does the real, cheap-after-first-call check (device present, backend + * plugin loaded, pipeline compiled) — never assumed from build-time + * alone. A build that HAS the ggml strategy compiled in but is running + * on hardware/software where it can't actually initialize (backend + * plugin missing, no GPU) correctly falls through to the next candidate. + * - ENV VAR gives explicit, debuggable override for either axis: + * EL_COSINE_BATCH_STRATEGY = "ggml" | "metal" | "cpu" | unset/"auto" + * forces a specific strategy (falling back to cpu if the forced one + * isn't actually available), or leaves the default auto-preference + * order in place. + * EL_METAL_COSINE = 0/n/N/f/F (back-compat with PR #114's vindex_bench + * gate) disables ALL GPU-backed strategies outright, same as before. + * + * DEFAULT preference order when nothing is forced: ggml, then hand-rolled + * Metal, then CPU fallback — first candidate whose available() reports true + * wins. This is what makes "stop hand-rolling GPU kernels, use ggml" real + * rather than nominal: ggml is what actually runs by default on this + * machine today (see the PR body for the measured numbers backing that). + */ +#include "eg_cosine_batch.h" +#include "eg_cosine_batch_strategy.h" + +#include +#include + +static bool g_selected = false; +static const EgCosineBatchStrategy* g_active = NULL; + +static bool eg_env_truthy_off(const char* v) { + return v && (v[0]=='0' || v[0]=='n' || v[0]=='N' || v[0]=='f' || v[0]=='F'); +} + +static const EgCosineBatchStrategy* eg_select_strategy(void) { + if (g_selected) return g_active; + g_selected = true; + + const EgCosineBatchStrategy* cpu = eg_cosine_batch_strategy_cpu(); + const char* force = getenv("EL_COSINE_BATCH_STRATEGY"); + const char* legacy_off = getenv("EL_METAL_COSINE"); + + if (eg_env_truthy_off(legacy_off)) { g_active = cpu; return g_active; } + + if (force && strcmp(force, "cpu") == 0) { g_active = cpu; return g_active; } + + if (force && strcmp(force, "ggml") == 0) { +#ifdef EG_HAVE_STRATEGY_GGML + const EgCosineBatchStrategy* s = eg_cosine_batch_strategy_ggml(); + if (s->available()) { g_active = s; return g_active; } +#endif + g_active = cpu; return g_active; + } + + if (force && strcmp(force, "metal") == 0) { +#ifdef EG_HAVE_STRATEGY_METAL_HAND + const EgCosineBatchStrategy* s = eg_cosine_batch_strategy_metal_hand(); + if (s->available()) { g_active = s; return g_active; } +#endif + g_active = cpu; return g_active; + } + + /* auto (unset, or any other value): ggml -> metal-hand -> cpu, first + * available wins. */ +#ifdef EG_HAVE_STRATEGY_GGML + { + const EgCosineBatchStrategy* s = eg_cosine_batch_strategy_ggml(); + if (s->available()) { g_active = s; return g_active; } + } +#endif +#ifdef EG_HAVE_STRATEGY_METAL_HAND + { + const EgCosineBatchStrategy* s = eg_cosine_batch_strategy_metal_hand(); + if (s->available()) { g_active = s; return g_active; } + } +#endif + g_active = cpu; + return g_active; +} + +bool eg_cosine_batch_available(void) { + return eg_select_strategy()->available(); +} + +const char* eg_cosine_batch_strategy_name(void) { + return eg_select_strategy()->name; +} + +bool eg_cosine_batch(const float* query, int32_t qdim, + const float* const* node_ptrs, + const int32_t* node_dims, + int32_t n, + double* out_scores) { + return eg_select_strategy()->batch(query, qdim, node_ptrs, node_dims, n, out_scores); +} + +bool eg_cosine_batch_multi(const float* queries, int32_t qdim, int32_t nq, + const float* const* node_ptrs, + const int32_t* node_dims, + int32_t n, + double* out_scores) { + return eg_select_strategy()->batch_multi(queries, qdim, nq, node_ptrs, node_dims, n, out_scores); +} diff --git a/lang/runtime/eg_cosine_batch.h b/lang/runtime/eg_cosine_batch.h new file mode 100644 index 0000000..bd5cb55 --- /dev/null +++ b/lang/runtime/eg_cosine_batch.h @@ -0,0 +1,106 @@ +/* eg_cosine_batch.h — stable Adapter interface over batch-cosine-similarity + * BACKEND STRATEGIES. This header is the ONE thing call sites (el_runtime.c, + * vindex_bench.c, ...) talk to. Plain C11, safe to #include on every + * platform — the symbols declared here always exist and always link, + * regardless of what backend actually runs underneath. Zero #ifdef at call + * sites: which concrete strategy executes (ggml/Metal, hand-rolled Metal, or + * the always-false CPU fallback) is resolved once, lazily, inside + * eg_cosine_batch.c's factory — see eg_cosine_batch_strategy.h for that. + * + * This supersedes eg_metal_cosine.h (PR #114's single hand-rolled-Metal-only + * bridge). The contract is UNCHANGED — same shapes, same sentinel, same + * never-partial guarantee, same "caller must always be prepared to fall back + * to its own scalar per-node loop" rule — only the name changed, because the + * thing behind it is no longer "the Metal bridge," it is "whichever batch- + * cosine strategy the factory picked." eg_metal_cosine.h's original doc + * comments (byte-for-byte, this file is the direct descendant) are preserved + * below since they remain the precise spec any strategy must honor. + * + * On ANY failure at ANY step — no compute device, compile/init error, alloc + * failure, bad args — every function here returns false and writes nothing. + * Out-params are either fully populated or left completely untouched, never + * partial. Callers MUST always be prepared to fall back to their own scalar + * per-node CPU loop unconditionally. These functions must never crash, throw, + * or hang the calling process — several call sites run inside a long-lived + * daemon's request-handling hot path. + */ +#ifndef EG_COSINE_BATCH_H +#define EG_COSINE_BATCH_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Batched cosine similarity: one query vector against `n` node vectors. + * + * query — qdim floats, the query embedding. Raw/unnormalized. + * qdim — query dimensionality (e.g. 768 for nomic-embed-text). + * node_ptrs — array of n pointers, node_ptrs[i] pointing at a (possibly + * differently-owned, possibly NULL) float vector for node i. + * NOT required to be contiguous — every strategy performs the + * gather into a packed row-major matrix internally, exactly + * mirroring how EngramNode.emb is one malloc per node. + * node_dims — array of n ints, node_dims[i] = that node's real emb_dim + * (0 or mismatched vs qdim => that node scores -2.0, matching + * eg_cosine's null/dim-mismatch/zero-norm sentinel exactly). + * n — number of nodes. + * out_scores — caller-owned array of n doubles; out_scores[i] is filled + * with the cosine similarity of node i against query, or + * -2.0 for a null/dim-mismatched/zero-norm node — bit-for-bit + * the same contract as eg_cosine(node_ptrs[i], query, qdim). + * + * Returns true iff a real backend strategy ran and out_scores was fully + * populated. Returns false (out_scores left untouched) on ANY failure or + * unavailability — no compute device, compile/init failure, allocation + * failure, n<=0, qdim<=0, null query/node_ptrs/node_dims/out_scores. + */ +bool eg_cosine_batch(const float* query, int32_t qdim, + const float* const* node_ptrs, + const int32_t* node_dims, + int32_t n, + double* out_scores); + +/* True iff a real (non-CPU-fallback) strategy is available right now (cheap + * after the first call — cached). Purely informational (e.g. a startup log + * line or /api/stats field); callers should still treat a false return from + * eg_cosine_batch()/eg_cosine_batch_multi() itself as the authoritative + * fallback signal, not this function. */ +bool eg_cosine_batch_available(void); + +/* Which concrete strategy is currently selected — "ggml", "metal-hand", + * or "cpu-fallback". Purely informational/diagnostic, same spirit as + * eg_cosine_batch_available(). Never NULL. */ +const char* eg_cosine_batch_strategy_name(void); + +/* Multi-query batched cosine: nq query vectors against the SAME n node + * vectors, in one call. A real strategy uploads/prepares the node population + * ONCE and reuses it for every query, instead of nq separate + * eg_cosine_batch() calls each paying the full gather+upload cost — PR #114 + * measured this necessary: at N~=13.7k/dim=768, repeating the single-query + * call per query was slower than the CPU baseline; batching queries together + * is what makes a GPU-backed path a real win at this shape. Use this + * whenever multiple queries will run against an unchanged (or + * rarely-changing) node population; use eg_cosine_batch() for a genuinely + * one-off comparison. + * + * queries — nq*qdim floats, row-major (query i at queries+i*qdim). + * out_scores — caller-owned nq*n doubles, row-major + * (out_scores[i*n+j] = cosine(queries[i], node j)), same + * -2.0 sentinel semantics as eg_cosine_batch(). + * + * Returns true iff a real strategy ran and out_scores was fully populated + * (all nq*n entries); false (untouched) on any failure/unavailability. */ +bool eg_cosine_batch_multi(const float* queries, int32_t qdim, int32_t nq, + const float* const* node_ptrs, + const int32_t* node_dims, + int32_t n, + double* out_scores); + +#ifdef __cplusplus +} +#endif + +#endif /* EG_COSINE_BATCH_H */ diff --git a/lang/runtime/eg_cosine_batch.metal b/lang/runtime/eg_cosine_batch.metal new file mode 100644 index 0000000..b6868c6 --- /dev/null +++ b/lang/runtime/eg_cosine_batch.metal @@ -0,0 +1,156 @@ +/* eg_cosine_batch.metal — batched cosine similarity, one query vs N node vectors. + * + * GPU-shaped counterpart to eg_cosine() in el_runtime.c: same math, same + * dim-mismatch/zero-norm sentinel (-2.0), applied to N independent rows in + * parallel instead of one pair at a time in a CPU loop. + * + * Semantics MUST match eg_cosine() exactly: + * - inputs are raw, UNNORMALIZED vectors (nomic-embed-text magnitudes are + * not 1.0) — this kernel computes the full dot/(|a|*|b|) cosine, not a + * plain dot product. + * - a node whose declared dim differs from the query dim, or whose norm is + * zero, scores exactly -2.0 (below any valid cosine in [-1,1]), so a + * caller doing `if (score > threshold)` behaves identically whether the + * scalar or the batched path filled the array. + * + * Precision: Apple GPUs do not support double in Metal Shading Language — + * everything here is float32. eg_cosine accumulates in CPU double, but its + * *inputs* are float32 embeddings, so the achievable precision ceiling is + * bounded by the input data regardless of accumulator width. To keep the + * float32 reduction from drifting relative to the double-accumulated CPU + * result across dim=768 terms, each thread accumulates with 4 independent + * partial sums (unrolled) rather than one running scalar — the same + * error-reduction trick already used by the CPU brute-force loop in + * vindex_bench.c. The measured float-vs-double delta is reported in the PR + * description; this is not assumed to be "close enough" without measurement. + */ +#include +using namespace metal; + +/* Per-dispatch invariants. `dim` is the query's dimensionality — the + * dimensionality every comparable node vector must match. */ +struct EgCosineParams { + uint n; /* number of node rows */ + uint dim; /* vector width (both query and node rows are `dim` wide in + * the packed buffer; node_dims[] carries each node's REAL + * embedded dim for the mismatch check) */ +}; + +/* One thread per node row. node_matrix is n*dim floats, row-major, packed at + * `dim` stride regardless of a row's real dim (the CPU side zero-pads or + * skips packing rows that don't match — see eg_cosine_batch_metal in + * eg_metal_cosine.m for the exact packing contract). node_dims[i] is the + * node's true emb_dim, used only for the mismatch sentinel — never used to + * index, since every row is packed at uniform `dim` stride. */ +kernel void eg_cosine_batch_kernel( + device const float* query [[buffer(0)]], + device const float* node_matrix [[buffer(1)]], + device const int* node_dims [[buffer(2)]], + constant EgCosineParams& p [[buffer(3)]], + device float* out_scores [[buffer(4)]], + uint gid [[thread_position_in_grid]]) +{ + if (gid >= p.n) return; + + if (node_dims[gid] != int(p.dim)) { + out_scores[gid] = -2.0f; + return; + } + + device const float* row = node_matrix + (uint64_t)gid * (uint64_t)p.dim; + + /* 4-way partial accumulation — same shape as vindex_bench.c's brute_topk + * unroll, done here for float32 accuracy rather than raw throughput. */ + float dot0 = 0.0f, dot1 = 0.0f, dot2 = 0.0f, dot3 = 0.0f; + float na0 = 0.0f, na1 = 0.0f, na2 = 0.0f, na3 = 0.0f; + float nb0 = 0.0f, nb1 = 0.0f, nb2 = 0.0f, nb3 = 0.0f; + + uint d = 0; + uint dim4 = p.dim & ~3u; + for (; d < dim4; d += 4) { + float a0 = row[d], b0 = query[d]; + float a1 = row[d+1], b1 = query[d+1]; + float a2 = row[d+2], b2 = query[d+2]; + float a3 = row[d+3], b3 = query[d+3]; + dot0 += a0*b0; dot1 += a1*b1; dot2 += a2*b2; dot3 += a3*b3; + na0 += a0*a0; na1 += a1*a1; na2 += a2*a2; na3 += a3*a3; + nb0 += b0*b0; nb1 += b1*b1; nb2 += b2*b2; nb3 += b3*b3; + } + float dot = (dot0 + dot1) + (dot2 + dot3); + float na = (na0 + na1) + (na2 + na3); + float nb = (nb0 + nb1) + (nb2 + nb3); + for (; d < p.dim; d++) { + float a = row[d], b = query[d]; + dot += a*b; na += a*a; nb += b*b; + } + + if (na <= 0.0f || nb <= 0.0f) { + out_scores[gid] = -2.0f; + return; + } + out_scores[gid] = dot / sqrt(na * nb); +} + +/* ── multi-query variant ────────────────────────────────────────────────── + * Same per-pair math as eg_cosine_batch_kernel, but amortizes ONE upload of + * node_matrix (the expensive part at real store size — 13k*768 floats is + * ~42MB) across `nq` queries instead of re-uploading it once per query. + * Measured need: a naive one-query-at-a-time loop calling the single-query + * kernel nq times was SLOWER than the CPU oracle at N≈13.7k (re-gather + + * re-upload dominated the actual compute) — this is the fix, not a + * hypothetical optimization. + * + * 2D grid: x = node index [0,n), y = query index [0,nq). out_scores is + * nq*n, row-major by query (out_scores[qid*n + nid]). */ +struct EgCosineMultiParams { uint n; uint dim; uint nq; }; + +kernel void eg_cosine_batch_multi_kernel( + device const float* queries [[buffer(0)]], /* nq*dim */ + device const float* node_matrix [[buffer(1)]], /* n*dim */ + device const int* node_dims [[buffer(2)]], /* n */ + constant EgCosineMultiParams& p [[buffer(3)]], + device float* out_scores [[buffer(4)]], /* nq*n */ + uint2 gid [[thread_position_in_grid]]) +{ + uint nid = gid.x, qid = gid.y; + if (nid >= p.n || qid >= p.nq) return; + + uint64_t out_idx = (uint64_t)qid * (uint64_t)p.n + (uint64_t)nid; + + if (node_dims[nid] != int(p.dim)) { + out_scores[out_idx] = -2.0f; + return; + } + + device const float* row = node_matrix + (uint64_t)nid * (uint64_t)p.dim; + device const float* query = queries + (uint64_t)qid * (uint64_t)p.dim; + + float dot0 = 0.0f, dot1 = 0.0f, dot2 = 0.0f, dot3 = 0.0f; + float na0 = 0.0f, na1 = 0.0f, na2 = 0.0f, na3 = 0.0f; + float nb0 = 0.0f, nb1 = 0.0f, nb2 = 0.0f, nb3 = 0.0f; + + uint d = 0; + uint dim4 = p.dim & ~3u; + for (; d < dim4; d += 4) { + float a0 = row[d], b0 = query[d]; + float a1 = row[d+1], b1 = query[d+1]; + float a2 = row[d+2], b2 = query[d+2]; + float a3 = row[d+3], b3 = query[d+3]; + dot0 += a0*b0; dot1 += a1*b1; dot2 += a2*b2; dot3 += a3*b3; + na0 += a0*a0; na1 += a1*a1; na2 += a2*a2; na3 += a3*a3; + nb0 += b0*b0; nb1 += b1*b1; nb2 += b2*b2; nb3 += b3*b3; + } + float dot = (dot0 + dot1) + (dot2 + dot3); + float na = (na0 + na1) + (na2 + na3); + float nb = (nb0 + nb1) + (nb2 + nb3); + for (; d < p.dim; d++) { + float a = row[d], b = query[d]; + dot += a*b; na += a*a; nb += b*b; + } + + if (na <= 0.0f || nb <= 0.0f) { + out_scores[out_idx] = -2.0f; + return; + } + out_scores[out_idx] = dot / sqrt(na * nb); +} diff --git a/lang/runtime/eg_cosine_batch_strategy.h b/lang/runtime/eg_cosine_batch_strategy.h new file mode 100644 index 0000000..034ac10 --- /dev/null +++ b/lang/runtime/eg_cosine_batch_strategy.h @@ -0,0 +1,82 @@ +/* eg_cosine_batch_strategy.h — internal Strategy interface, NOT for call + * sites (they use eg_cosine_batch.h). Only eg_cosine_batch.c's factory and + * the concrete strategy implementation files include this. + * + * Each concrete strategy exposes exactly one getter returning a pointer to a + * static, immutable EgCosineBatchStrategy vtable. Which getters actually + * exist as linkable symbols is a BUILD-TIME concern (decided by + * build_vindex_bench.sh / the engram daemon's own build, via which .c/.m + * files get compiled per platform) gated by the EG_HAVE_STRATEGY_* macros + * below — the factory in eg_cosine_batch.c is the ONLY place that branches + * on those macros. Call sites never see them; that's the whole point of the + * Adapter in eg_cosine_batch.h. + * + * Three concrete strategies exist: + * eg_cosine_batch_strategy_ggml() — ggml + dynamically-loaded Metal + * backend plugin. Darwin only in + * this build; the default + * preferred strategy wherever + * available. EG_HAVE_STRATEGY_GGML. + * eg_cosine_batch_strategy_metal_hand() — the original hand-rolled Metal + * compute shader from PR #114 + * (eg_cosine_batch.metal), + * preserved verbatim as a + * selectable fallback strategy, + * not deleted. Darwin only. + * EG_HAVE_STRATEGY_METAL_HAND. + * eg_cosine_batch_strategy_cpu() — universal always-false + * fallback. Always compiled, on + * every platform; this is what a + * non-Darwin build links + * exclusively (matching PR #114's + * eg_metal_cosine_stub.c), and + * what any platform falls back + * to when no real strategy is + * available at runtime. + */ +#ifndef EG_COSINE_BATCH_STRATEGY_H +#define EG_COSINE_BATCH_STRATEGY_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct EgCosineBatchStrategy { + /* Stable, short, lowercase-hyphenated identifier — what + * eg_cosine_batch_strategy_name() surfaces. Never NULL. */ + const char* name; + + /* Cheap after the first call (lazy init, cached internally). Must never + * throw/crash/hang — mirrors eg_cosine_batch_available()'s contract. */ + bool (*available)(void); + + /* Same shape/contract as eg_cosine_batch() in eg_cosine_batch.h. */ + bool (*batch)(const float* query, int32_t qdim, + const float* const* node_ptrs, const int32_t* node_dims, + int32_t n, double* out_scores); + + /* Same shape/contract as eg_cosine_batch_multi() in eg_cosine_batch.h. */ + bool (*batch_multi)(const float* queries, int32_t qdim, int32_t nq, + const float* const* node_ptrs, const int32_t* node_dims, + int32_t n, double* out_scores); +} EgCosineBatchStrategy; + +#ifdef EG_HAVE_STRATEGY_GGML +const EgCosineBatchStrategy* eg_cosine_batch_strategy_ggml(void); +#endif + +#ifdef EG_HAVE_STRATEGY_METAL_HAND +const EgCosineBatchStrategy* eg_cosine_batch_strategy_metal_hand(void); +#endif + +/* Always declared/linked, on every platform/build. */ +const EgCosineBatchStrategy* eg_cosine_batch_strategy_cpu(void); + +#ifdef __cplusplus +} +#endif + +#endif /* EG_COSINE_BATCH_STRATEGY_H */ diff --git a/lang/runtime/eg_cosine_batch_strategy_cpu.c b/lang/runtime/eg_cosine_batch_strategy_cpu.c new file mode 100644 index 0000000..0e6640b --- /dev/null +++ b/lang/runtime/eg_cosine_batch_strategy_cpu.c @@ -0,0 +1,45 @@ +/* eg_cosine_batch_strategy_cpu.c — plain-C, zero-dependency universal + * fallback strategy. Always returns false / unavailable. Direct descendant + * of PR #114's eg_metal_cosine_stub.c, generalized from "the Metal stub" to + * "the strategy vtable's universal fallback entry" now that multiple real + * strategies can exist. + * + * Always compiled, on every platform. On Darwin builds it is the last-resort + * strategy the factory falls back to when neither ggml nor the hand-rolled + * Metal strategy is available at runtime (no device, compile failure, ...). + * On non-Darwin builds it is the ONLY strategy compiled in at all — no + * Objective-C, no Metal frameworks, no ggml/Metal backend plugin — so + * eg_cosine_batch()/eg_cosine_batch_multi() always return false there and + * every call site's existing CPU fallback runs unconditionally, exactly as + * before this PR. + */ +#include "eg_cosine_batch_strategy.h" + +static bool cpu_available(void) { + return false; +} + +static bool cpu_batch(const float* query, int32_t qdim, + const float* const* node_ptrs, const int32_t* node_dims, + int32_t n, double* out_scores) { + (void)query; (void)qdim; (void)node_ptrs; (void)node_dims; (void)n; (void)out_scores; + return false; +} + +static bool cpu_batch_multi(const float* queries, int32_t qdim, int32_t nq, + const float* const* node_ptrs, const int32_t* node_dims, + int32_t n, double* out_scores) { + (void)queries; (void)qdim; (void)nq; (void)node_ptrs; (void)node_dims; (void)n; (void)out_scores; + return false; +} + +static const EgCosineBatchStrategy g_cpu_strategy = { + .name = "cpu-fallback", + .available = cpu_available, + .batch = cpu_batch, + .batch_multi = cpu_batch_multi, +}; + +const EgCosineBatchStrategy* eg_cosine_batch_strategy_cpu(void) { + return &g_cpu_strategy; +} diff --git a/lang/runtime/eg_cosine_batch_strategy_ggml.c b/lang/runtime/eg_cosine_batch_strategy_ggml.c new file mode 100644 index 0000000..1bc68aa --- /dev/null +++ b/lang/runtime/eg_cosine_batch_strategy_ggml.c @@ -0,0 +1,515 @@ +/* eg_cosine_batch_strategy_ggml.c — the GGML Strategy, and the preferred + * default whenever it is available (see the factory's selection order in + * eg_cosine_batch.c). + * + * WHY: directive from Will Anderson — stop hand-rolling GPU kernels, use a + * real, proven, permissively-licensed library instead. ggml (the compute + * library underneath llama.cpp, MIT licensed) is already installed on this + * machine as a standalone Homebrew package (`brew info ggml`), independent + * of llama.cpp itself. This file is a genuinely bounded COMPUTE UTILITY — + * batch cosine-similarity math — analogous to a VBD Accessor calling out to + * infrastructure. It is explicitly NOT the engram's reasoning/persistence + * core; using ggml here does not cross the "own the core" line, because + * batch cosine math is infrastructure, not the graph traversal / activation + * spreading / "thinking" that IS the core and stays 100% own-code. + * + * ── The real API shape (verified against the installed headers + a + * standalone probe program, not assumed from memory of other tensor + * libraries) ────────────────────────────────────────────────────────── + * + * ggml ships its CPU and Metal implementations as DYNAMICALLY LOADED PLUGIN + * .so files (confirmed by nm: `ggml_backend_metal_init` is NOT an exported + * symbol of libggml.dylib/libggml-base.dylib — it exists ONLY inside + * libggml-metal.so under $(brew --prefix ggml)/libexec/). You cannot link + * `-lggml-metal`; you must go through ggml's backend REGISTRY: + * + * 1. ggml_backend_load_all_from_path(dir) — dlopen()s every backend plugin + * .so found in `dir` and registers its device(s). We point this at + * $(brew --prefix ggml)/libexec (resolved once, at build+init time; see + * eg_ggml_backend_dir() below) rather than relying on + * ggml_backend_load_all()'s own default search heuristics, which are + * tuned for an installed llama.cpp-style app bundle layout, not an + * arbitrary `cc`-built binary invoked from an arbitrary cwd — the exact + * same "must not silently fall back to CPU for reasons that have + * nothing to do with GPU availability" concern PR #114's hand-rolled + * bridge already documented for its own embedded-shader-source choice. + * 2. ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU) — find the + * registered Metal device. + * 3. ggml_backend_dev_init(dev, NULL) — get a live ggml_backend_t. + * 4. Build a tiny ggml_context (no_alloc=true; it holds only tensor + * metadata, not data), declare 2D F32 tensors, ggml_mul_mat(nodes, + * query) — ggml's documented convention: A is [k cols, n rows], B is + * [k cols, m rows] (transposed internally), result is [n cols, m rows] + * — i.e. mul_mat(node_matrix[dim,n], query_matrix[dim,nq]) yields + * out[n,nq] where out[j*n+i] = dot(node_i, query_j). A row-major + * (dim,n) node matrix and a row-major (dim,nq) query matrix is EXACTLY + * the packed layout the hand-rolled Metal kernel already used — one + * matmul replaces the whole per-row dot-product loop. + * 5. ggml_backend_alloc_ctx_tensors(ctx, backend) to actually allocate + * device buffers for those tensors, ggml_backend_tensor_set() to upload, + * ggml_backend_graph_compute() to run, ggml_backend_tensor_get() to + * read back. + * + * This exact sequence was verified end-to-end in a standalone probe (build + * it yourself: see the PR description) against a plain-C CPU dot product — + * bit-for-bit correct within float rounding. Real numbers against the + * el_runtime.c CPU oracle are reported in the PR body via vindex_bench. + * + * ggml_mul_mat only computes the raw dot products — it has no notion of + * "cosine" or of this codebase's -2.0 dim-mismatch/null/zero-norm sentinel. + * Per the adapter's directive: gather only VALID, uniform-dim rows into the + * packed matrix sent to the GPU (skipping null/mismatched rows entirely, + * rather than the hand-rolled kernel's zero-pad-and-sentinel-in-shader + * approach), then scatter -2.0 back for every row that was excluded — same + * gather/scatter contract eg_cosine_batch.h documents. Norms (||node||, + * ||query||) are computed on the CPU host in the same pass that already + * touches every element to gather/convert — essentially free — using the + * same 4-way-partial-sum accumulation the hand-rolled kernel and the CPU + * oracle both use, so the float32 error profile stays comparable across all + * three strategies. Only the O(n*dim*nq) dot-product matmul — the actual + * expensive part — is offloaded to the GPU. + * + * ── Precision: the ne11<=8 chunking, and why it is not optional ────────── + * + * The claim in the first version of this file — "ggml_mul_mat on F32 x F32 + * inputs computes in F32 on the Metal backend" — is WRONG, and the 0.9933 + * id-recall it shipped with (vs the hand-rolled kernel's 0.9997) was the + * symptom. ggml-metal has two F32xF32 matmul kernels and picks between them + * purely on ne11 (the number of B rows == our query count): + * + * ne11 <= 8 -> kernel_mul_mv_ext_f32_f32_* / kernel_mul_mv_f32_f32_* + * templated — genuine F32 accumulation. + * ne11 > 8 -> kernel_mul_mm_f32_f32, which is templated + * — i.e. BOTH operands are narrowed + * to F16 and accumulated in simdgroup_half8x8 tiles, even + * though the tensors are GGML_TYPE_F32 on both sides. + * + * (Read it yourself, no guessing — the kernel templates are literal strings + * in the shipped plugin: + * strings $(brew --prefix ggml)/libexec/libggml-metal.so \ + * | grep -E 'host_name\("kernel_mul_m[mv]_f32_f32' + * and the runtime pick is visible with GGML_METAL_DEBUG-style logging as + * "compiling pipeline: base = 'kernel_mul_mm_f32_f32'".) + * + * The previous code issued ONE ggml_mul_mat with ne11 = nq (300 in the + * benchmark), landing squarely on the F16 mul_mm path. Measured on this + * machine (M4 Pro), n=13415 x dim=768 x nq=300, against a CPU double- + * accumulated oracle: + * + * ne11=300 (one mul_mat, the old code) : mean |Δdot| = 1.038e-05 + * ne11=8 (chunked, this code) : mean |Δdot| = 3.863e-09 + * + * — a ~2700x reduction in dot-product error, which is exactly the gap that + * showed up as 0.9933-vs-0.9997 recall. + * + * ggml_mul_mat_set_prec(t, GGML_PREC_F32) does NOT fix this. It was tried: + * the error was bit-identical with and without it (1.038e-05 either way), + * because ggml-metal only consults the prec flag on paths that have an F32 + * variant to switch to, and there is no F32-accumulating mul_mm kernel in + * this build to select. The ONLY lever from outside ggml is ne11. + * + * So: instead of one mul_mat with ne11=nq, we emit ceil(nq/8) mul_mats, each + * over an ne11<=8 ggml_view_2d slice of the same query tensor, all into ONE + * graph and ONE ggml_backend_graph_compute. The node matrix is still uploaded + * exactly once and still read by the GPU as one shared operand — the whole + * point of batch_multi is preserved. + * + * The cost is real, and stated rather than buried. Timing the whole + * batch_multi() call (gather + norms + upload + GPU + scatter) on the real + * shape, median of 15 reps after a discarded warm-up, three separate runs: + * + * unchunked (old, F16 mm) : 13.19 / 13.35 / 14.42 ms -> ~0.044 ms/query + * chunked (this code) : 19.92 / 20.08 / 20.23 ms -> ~0.067 ms/query + * hand-rolled Metal : 17.74 / 17.88 / 17.99 ms -> ~0.060 ms/query + * + * So correctness here costs about +6.7ms per 300-query batch (~1.5x on this + * call), and leaves us ~12% behind the hand-rolled kernel instead of ~35% + * ahead of it. That is not free and should not be sold as free. The reason it + * cannot be recovered inside ggml: an fp32 matmul on Metal has to re-stream + * the whole node matrix once per <=8 queries (38 dispatches x ~41MB here), + * where the F16 mul_mm kernel tiles it in threadgroup memory and reads it far + * fewer times. ggml's Metal backend ships no fp32 TILED matmul, so on this + * backend "fast" and "fp32" are genuinely exclusive — the hand-rolled kernel + * escapes the choice only because it is an fp32 kernel written for this one + * shape. Trading precision back for speed is a one-line env change; trading + * the other way was not available before this commit at all. + * + * 8 is not a magic number we invented — it is ggml-metal's own mul_mm + * threshold, measured by sweeping ne11 and watching both the error and which + * pipeline ggml compiles (9 flips to mul_mm and the error jumps back to + * 1.0e-05 in the same step). EL_GGML_MULMAT_CHUNK overrides it: raise it to + * trade this precision back for throughput, or set it >= nq to reproduce the + * old single-mul_mat behaviour exactly. If a future ggml moves the threshold, + * the worst case is that we silently land back on mul_mm — the same accuracy + * we shipped before, never a correctness break. + * + * ── Cold start: what is and is not ours to fix ─────────────────────────── + * + * The ~7.8s first-call cost reported for the first version of this file is + * NOT this file re-initialising per call (init is, and always was, cached + * behind g_init_attempted below). It is Apple's Metal shader cache missing + * on ggml's embedded metallib — ggml-metal ships ~650 kernels in one + * __ggml_metallib section, and the first newLibraryWithData of it on a given + * machine costs seconds ("ggml_metal_library_init: loaded in 7.670 sec") + * while the driver populates ~/…/C/com.apple.metal/. That cache is keyed on + * the library, not on our binary, and is shared across processes: the very + * next run of a DIFFERENT binary linking the same ggml reports + * "loaded in 0.009 sec". So it is a once-per-machine, per-ggml-version cost, + * not a per-process one, and nothing this file does can avoid it — the + * hand-rolled strategy escapes it only because its shader is two small + * kernels instead of six hundred. + * + * The residual warm init IS ours to look at, and the answer there is "there + * was nothing much to win": ggml_backend_load_all_from_path() dlopens every + * plugin in the directory (three CPU micro-arch variants + BLAS + Metal) when + * we only ever use Metal, so we now load the single Metal plugin instead — + * but measured warm that is 44.7-52.4ms against 46.9-58.9ms, i.e. the same + * number inside noise, because libggml-metal.so's own init dominates. Warm + * ggml init lands at 44-53ms, against 36-117ms for the hand-rolled strategy's + * device+pipeline setup. Cold start was never the real defect here; precision + * was. + */ +#include "eg_cosine_batch_strategy.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +/* ── lazy, one-time backend init, cached ─────────────────────────────────── */ +static bool g_init_attempted = false; +static bool g_init_ok = false; +static ggml_backend_t g_backend = NULL; + +/* Where to look for the dynamically-loaded backend plugin .so files. + * EL_GGML_BACKEND_PATH overrides for non-standard installs; otherwise we try + * the Homebrew opt-prefix symlink (stable across ggml point-version bumps — + * $(brew --prefix ggml)/libexec — confirmed to exist and contain + * libggml-metal.so / libggml-cpu-*.so / libggml-blas.so on this machine), + * falling back to ggml's own default search (ggml_backend_load_all()) in + * case a different install layout (e.g. a from-source build with a + * standard-prefix install) makes that succeed instead. */ +static const char* eg_ggml_backend_dir(void) { + const char* s = getenv("EL_GGML_BACKEND_PATH"); + if (s && *s) return s; + return "/opt/homebrew/opt/ggml/libexec"; +} + +/* "/libggml-metal.so" in a static buffer. Only ever called once, from + * eg_ggml_ensure_init(), before any thread could race it. */ +static const char* eg_ggml_metal_plugin_path(const char* dir) { + static char buf[1024]; + snprintf(buf, sizeof buf, "%s/libggml-metal.so", dir); + return buf; +} + +/* Largest ne11 (query-batch rows per ggml_mul_mat) that keeps ggml-metal on + * its F32 mul_mv kernels instead of the F16-accumulating mul_mm kernel — see + * the precision discussion in this file's header. EL_GGML_MULMAT_CHUNK + * overrides; a value <= 0 means "use the default". */ +#define EG_GGML_MULMAT_CHUNK_DEFAULT 8 + +static int32_t eg_ggml_mulmat_chunk(void) { + static bool resolved = false; + static int32_t chunk = EG_GGML_MULMAT_CHUNK_DEFAULT; + if (!resolved) { + resolved = true; + const char* s = getenv("EL_GGML_MULMAT_CHUNK"); + if (s && *s) { + long v = strtol(s, NULL, 10); + if (v > 0 && v <= INT32_MAX) chunk = (int32_t)v; + } + } + return chunk; +} + +/* Which ggml device this strategy computes on. GPU (Metal) is the default + * because offloading is the architectural point — the engram's own graph + * traversal and activation spreading are CPU work, and a "GPU" strategy that + * quietly saturates the CPU steals from them. + * + * ACCEL (ggml's BLAS/Accelerate plugin) is reachable here mainly as a + * portability fallback and a diagnostic, and it is documented as MEASURED AND + * REJECTED rather than as a recommendation. In an isolated probe that timed + * only ggml_backend_graph_compute, BLAS looked excellent — 3.4-4.0ms for the + * 300-query batch at mean |Δdot| 1.5e-08, i.e. as fast as the old F16 path and + * far more accurate. End to end on the real store through vindex_bench it does + * not hold up: 0.191 ms/query at id-recall 0.9973, against 0.125-0.142 ms/query + * at 0.9987 for the Metal default. It is dominated on BOTH axes, because the + * isolated probe was not competing with the rest of the batch for the same CPU + * cores and the real call path is. Kept because a machine with no usable Metal + * device still wants a working ggml strategy — not because it is faster. */ +static enum ggml_backend_dev_type eg_ggml_device_type(void) { + const char* s = getenv("EL_GGML_DEVICE"); + if (s && *s) { + if (strcmp(s, "accel") == 0) return GGML_BACKEND_DEVICE_TYPE_ACCEL; + if (strcmp(s, "cpu") == 0) return GGML_BACKEND_DEVICE_TYPE_CPU; + } + return GGML_BACKEND_DEVICE_TYPE_GPU; +} + +static bool eg_ggml_ensure_init(void) { + if (g_init_attempted) return g_init_ok; + g_init_attempted = true; + + const char* dir = eg_ggml_backend_dir(); + const enum ggml_backend_dev_type want = eg_ggml_device_type(); + + /* Metal is the only backend this strategy uses by default, so load just + * that one plugin rather than dlopening the whole directory (three CPU + * micro-arch variants + BLAS + Metal here). + * + * Be honest about what this buys: almost nothing in wall time. Measured + * warm, three runs each — load-everything 58.9/46.9/55.1ms, Metal-only + * 52.4/51.2/44.7ms. The cost is dominated by dlopening and initialising + * libggml-metal.so itself, not by the four plugins we skip, so the two + * overlap inside noise. It is kept because registering four device types + * we will never dispatch to is untidy and makes ggml_backend_dev_by_type + * ambiguous, not because it is a speedup — do not cite it as one. + * + * ggml_backend_load() returns NULL for a missing or unloadable path, + * which simply falls through to the broader searches below; it is never + * fatal. Any non-default device needs the full directory scan to find + * its plugin, so skip the fast path there. */ + if (want == GGML_BACKEND_DEVICE_TYPE_GPU) + ggml_backend_load(eg_ggml_metal_plugin_path(dir)); + + ggml_backend_dev_t dev = ggml_backend_dev_by_type(want); + if (!dev) { + /* Non-standard layout, a ggml built with a differently-named Metal + * plugin, or a non-default device: dlopen every plugin in `dir`. */ + ggml_backend_load_all_from_path(dir); + dev = ggml_backend_dev_by_type(want); + } + if (!dev) { + /* Fall back to ggml's own default search heuristics only if the + * explicit path above found nothing — avoids double-registering the + * same plugins (ggml does not dedupe two different paths that + * happen to resolve to the same files, e.g. our stable opt-prefix + * symlink vs. its own Cellar-relative guess) in the common case + * where the explicit path already worked. */ + ggml_backend_load_all(); + dev = ggml_backend_dev_by_type(want); + } + if (!dev) return false; + + ggml_backend_t backend = ggml_backend_dev_init(dev, NULL); + if (!backend) return false; + + g_backend = backend; + g_init_ok = true; + return true; +} + +static bool ggml_strategy_available(void) { + return eg_ggml_ensure_init(); +} + +/* ── shared core: gather valid rows + norms, matmul, scatter ────────────── */ + +/* 4-way partial-sum squared-norm accumulation over `dim` floats — same shape + * as eg_cosine_batch.metal's per-thread accumulation and vindex_bench.c's + * CPU brute_topk unroll, kept consistent on purpose so the float32 error + * profile is comparable across all three strategies. */ +static float eg_norm_sq_f32(const float* v, int32_t dim) { + float s0 = 0, s1 = 0, s2 = 0, s3 = 0; + int32_t d = 0, dim4 = dim & ~3; + for (; d < dim4; d += 4) { + s0 += v[d] * v[d]; s1 += v[d+1] * v[d+1]; + s2 += v[d+2] * v[d+2]; s3 += v[d+3] * v[d+3]; + } + float s = (s0 + s1) + (s2 + s3); + for (; d < dim; d++) s += v[d] * v[d]; + return s; +} + +/* Runs one ggml_mul_mat(node_matrix[dim,n_valid], query_matrix[dim,nq]) and + * combines it with CPU-computed norms into cosine scores, scattering into + * out_scores at ORIGINAL (ungathered) indices. out_scores must already be + * fully sized for n*nq (or n for the single-query case, nq=1) — every entry + * gets written (valid rows get a real cosine, invalid rows get -2.0), so + * this never leaves a partial result. Returns false only on a genuine + * failure (alloc, compute) at which point out_scores is left as whatever a + * caller-supplied scratch buffer already contained — callers here always + * pass a fresh buffer they discard on false, matching the adapter contract + * of "on failure, out_scores is treated as untouched" from the caller's + * point of view. */ +static bool eg_ggml_run(const float* queries, int32_t qdim, int32_t nq, + const float* const* node_ptrs, const int32_t* node_dims, + int32_t n, double* out_scores) { + if (!queries || qdim <= 0 || nq <= 0 || !node_ptrs || !node_dims || n <= 0 || !out_scores) + return false; + if (!eg_ggml_ensure_init()) return false; + + /* Pass 1 (CPU): gather valid rows (non-NULL ptr, dim == qdim) into a + * packed (dim, n_valid) row-major matrix, remembering the original index + * of each packed row, and compute each valid row's squared norm in the + * same pass. Rows excluded here get -2.0 scattered for every query + * below without ever touching the GPU. */ + int32_t* valid_orig = (int32_t*)malloc((size_t)n * sizeof(int32_t)); + float* node_norm_sq = (float*)malloc((size_t)n * sizeof(float)); /* indexed by packed position */ + float* node_matrix = NULL; + if (!valid_orig || !node_norm_sq) { free(valid_orig); free(node_norm_sq); return false; } + + int32_t n_valid = 0; + for (int32_t i = 0; i < n; i++) { + if (node_ptrs[i] && node_dims[i] == qdim) n_valid++; + } + + if (n_valid > 0) { + node_matrix = (float*)malloc((size_t)n_valid * (size_t)qdim * sizeof(float)); + if (!node_matrix) { free(valid_orig); free(node_norm_sq); return false; } + int32_t w = 0; + for (int32_t i = 0; i < n; i++) { + if (!node_ptrs[i] || node_dims[i] != qdim) continue; + memcpy(node_matrix + (size_t)w * qdim, node_ptrs[i], (size_t)qdim * sizeof(float)); + node_norm_sq[w] = eg_norm_sq_f32(node_ptrs[i], qdim); + valid_orig[w] = i; + w++; + } + } + + /* Query norms — nq is typically small (1 or the size of one batch of + * comparison queries), so this loop is cheap regardless. */ + float* q_norm_sq = (float*)malloc((size_t)nq * sizeof(float)); + if (!q_norm_sq) { free(valid_orig); free(node_norm_sq); free(node_matrix); return false; } + for (int32_t j = 0; j < nq; j++) q_norm_sq[j] = eg_norm_sq_f32(queries + (size_t)j * qdim, qdim); + + /* Nothing valid to compare against: every output is -2.0. Still a fully + * and correctly populated result — no GPU dispatch was needed to know + * that. */ + if (n_valid == 0) { + for (size_t k = 0; k < (size_t)n * (size_t)nq; k++) out_scores[k] = -2.0; + free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); + return true; + } + + /* Pass 2 (GPU via ggml): dot[j*n_valid + i] = dot(node_i, query_j), + * computed as ceil(nq/chunk) separate ggml_mul_mat ops over ne11<=chunk + * ggml_view_2d slices of ONE query tensor, all expanded into ONE graph + * and run by ONE ggml_backend_graph_compute. Chunking is what keeps + * ggml-metal on its F32 mul_mv kernels rather than the F16-accumulating + * mul_mm kernel (see this file's header); sharing one graph and one + * t_nodes tensor is what keeps the node matrix uploaded exactly once, + * which is the entire reason batch_multi exists. */ + const int32_t chunk = eg_ggml_mulmat_chunk(); + const int32_t ngroups = (nq + chunk - 1) / chunk; + + /* Tensors held by the context: t_nodes, t_query, plus one view and one + * mul_mat result per group. The graph holds at most one node per view and + * one per mul_mat. Slack on both so a ggml that bookkeeps slightly + * differently cannot silently overflow the arena. */ + const size_t n_tensors = (size_t)2 * (size_t)ngroups + 8; + const size_t graph_size = (size_t)2 * (size_t)ngroups + 16; + struct ggml_init_params gp = { + .mem_size = ggml_tensor_overhead() * n_tensors + + ggml_graph_overhead_custom(graph_size, false), + .mem_buffer = NULL, + .no_alloc = true, + }; + struct ggml_context* ctx = ggml_init(gp); + if (!ctx) { free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; } + + struct ggml_tensor** t_dots = (struct ggml_tensor**)malloc((size_t)ngroups * sizeof(*t_dots)); + if (!t_dots) { ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; } + + struct ggml_tensor* t_nodes = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, qdim, n_valid); + struct ggml_tensor* t_query = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, qdim, nq); + struct ggml_cgraph* gf = t_nodes && t_query + ? ggml_new_graph_custom(ctx, graph_size, false) : NULL; + if (!gf) { free(t_dots); ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; } + + bool built = true; + for (int32_t g = 0; g < ngroups; g++) { + const int32_t start = g * chunk; + const int32_t count = (start + chunk <= nq) ? chunk : (nq - start); + struct ggml_tensor* t_qv = ggml_view_2d(ctx, t_query, qdim, count, + t_query->nb[1], + (size_t)start * t_query->nb[1]); + t_dots[g] = t_qv ? ggml_mul_mat(ctx, t_nodes, t_qv) : NULL; + if (!t_dots[g]) { built = false; break; } + ggml_build_forward_expand(gf, t_dots[g]); + } + if (!built) { free(t_dots); ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; } + + struct ggml_backend_buffer* buf = ggml_backend_alloc_ctx_tensors(ctx, g_backend); + if (!buf) { free(t_dots); ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; } + + ggml_backend_tensor_set(t_nodes, node_matrix, 0, (size_t)n_valid * qdim * sizeof(float)); + ggml_backend_tensor_set(t_query, queries, 0, (size_t)nq * qdim * sizeof(float)); + free(node_matrix); /* uploaded; the packed CPU copy is no longer needed */ + + enum ggml_status st = ggml_backend_graph_compute(g_backend, gf); + if (st != GGML_STATUS_SUCCESS) { + free(t_dots); ggml_backend_buffer_free(buf); ggml_free(ctx); + free(valid_orig); free(node_norm_sq); free(q_norm_sq); + return false; + } + + /* Each group's result is [n_valid, count] contiguous, so reading group g + * into dot + start*n_valid reconstructs exactly the same flat + * dot[j*n_valid + w] layout a single ne11=nq mul_mat would have produced — + * Pass 3 below is unchanged by the chunking. */ + float* dot = (float*)malloc((size_t)n_valid * (size_t)nq * sizeof(float)); + if (!dot) { free(t_dots); ggml_backend_buffer_free(buf); ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(q_norm_sq); return false; } + for (int32_t g = 0; g < ngroups; g++) { + const int32_t start = g * chunk; + const int32_t count = (start + chunk <= nq) ? chunk : (nq - start); + ggml_backend_tensor_get(t_dots[g], dot + (size_t)start * n_valid, 0, + (size_t)count * (size_t)n_valid * sizeof(float)); + } + free(t_dots); + + /* Pass 3 (CPU): combine dot/(||a||*||b||) per (query,node) pair, scatter + * into out_scores at ORIGINAL node indices; every excluded row gets + * -2.0 for every query. out_scores is fully populated either way. */ + for (int32_t j = 0; j < nq; j++) { + double* orow = out_scores + (size_t)j * n; + for (int32_t i = 0; i < n; i++) orow[i] = -2.0; /* default: excluded */ + for (int32_t w = 0; w < n_valid; w++) { + float na = node_norm_sq[w], nb = q_norm_sq[j]; + int32_t oi = valid_orig[w]; + if (na <= 0.0f || nb <= 0.0f) { orow[oi] = -2.0; continue; } + float d = dot[(size_t)j * n_valid + w]; + orow[oi] = (double)(d / sqrtf(na * nb)); + } + } + + free(dot); + ggml_backend_buffer_free(buf); + ggml_free(ctx); + free(valid_orig); free(node_norm_sq); free(q_norm_sq); + return true; +} + +static bool ggml_strategy_batch(const float* query, int32_t qdim, + const float* const* node_ptrs, const int32_t* node_dims, + int32_t n, double* out_scores) { + if (!query || qdim <= 0 || !node_ptrs || !node_dims || n <= 0 || !out_scores) return false; + /* out_scores here is n doubles (nq=1); eg_ggml_run writes n*nq = n of + * them, laid out identically to the single-query contract. */ + return eg_ggml_run(query, qdim, 1, node_ptrs, node_dims, n, out_scores); +} + +static bool ggml_strategy_batch_multi(const float* queries, int32_t qdim, int32_t nq, + const float* const* node_ptrs, const int32_t* node_dims, + int32_t n, double* out_scores) { + return eg_ggml_run(queries, qdim, nq, node_ptrs, node_dims, n, out_scores); +} + +static const EgCosineBatchStrategy g_ggml_strategy = { + .name = "ggml", + .available = ggml_strategy_available, + .batch = ggml_strategy_batch, + .batch_multi = ggml_strategy_batch_multi, +}; + +const EgCosineBatchStrategy* eg_cosine_batch_strategy_ggml(void) { + return &g_ggml_strategy; +} diff --git a/lang/runtime/eg_cosine_batch_strategy_metal_hand.m b/lang/runtime/eg_cosine_batch_strategy_metal_hand.m new file mode 100644 index 0000000..877bcfc --- /dev/null +++ b/lang/runtime/eg_cosine_batch_strategy_metal_hand.m @@ -0,0 +1,358 @@ +/* eg_cosine_batch_strategy_metal_hand.m — the HAND-ROLLED-METAL Strategy. + * + * This is PR #114's original Objective-C bridge (formerly eg_metal_cosine.m) + * exposing the hand-written Metal compute shader (eg_cosine_batch.metal) as + * one concrete EgCosineBatchStrategy. It is preserved here almost verbatim — + * real, carefully verified work, not discarded — now living behind the + * Adapter/Strategy/Factory restructuring (see eg_cosine_batch.h and + * eg_cosine_batch_strategy.h) alongside the new ggml-Metal strategy + * (eg_cosine_batch_strategy_ggml.c) and the universal CPU fallback + * (eg_cosine_batch_strategy_cpu.c). The factory in eg_cosine_batch.c prefers + * ggml by default when both are available; this strategy remains selectable + * via EL_COSINE_BATCH_STRATEGY=metal, and is what the factory falls back to + * if ggml's backend plugin fails to load/init for any reason. + * + * Apple-only (Metal has no other platform). This file is excluded from the + * build entirely on non-Darwin — see build_vindex_bench.sh, which only + * compiles/links this file and defines EG_HAVE_STRATEGY_METAL_HAND when + * `uname` is Darwin. On Linux the factory never sees this strategy at all — + * callers must always be prepared for the "no real strategy available" + * fallback via the CPU strategy, which is also exactly what happens here on + * Apple hardware with no usable GPU. + * + * Design (unchanged from PR #114): + * - Device/queue/pipeline are created lazily, once, and cached in static + * globals — every call after the first only allocates buffers + submits. + * - The Metal shader source is embedded as a C string literal (kMetalSrc + * below) rather than loaded from a file at runtime or shipped as a + * precompiled .metallib. Chosen over newLibraryWithFile: /a .metallib + * because the engram binary can be invoked from an arbitrary working + * directory (launchd job, nsbx sandbox, CI) and a file-path shader would + * be one relocation away from silently falling back to CPU for reasons + * that have nothing to do with Metal availability. Embedding costs one + * runtime shader compile (~tens of ms) on first use, amortized over the + * process lifetime, in exchange for a genuinely self-contained binary. + * Source of truth for review/tooling is eg_cosine_batch.metal — this + * string MUST be kept byte-identical to that file (a comment marks both + * ends of the copy). + * - Buffers use MTLResourceStorageModeShared: on Apple Silicon's unified + * memory, CPU and GPU read the same physical pages, so filling a buffer + * is a plain memcpy and there is no separate "upload" step. + * - ANY failure at ANY step (no device, pipeline compile error, buffer + * allocation failure, bad args) returns false and leaves out_scores + * untouched. This function is called from the request-handling hot path + * of a long-lived daemon — it must never throw, crash, or hang it. + */ +#import +#import +#include "eg_cosine_batch_strategy.h" +#include +#include + +/* ── BEGIN embedded shader source (keep in sync with eg_cosine_batch.metal) ── */ +static const char* kEgCosineBatchMetalSrc = +"#include \n" +"using namespace metal;\n" +"struct EgCosineParams { uint n; uint dim; };\n" +"kernel void eg_cosine_batch_kernel(\n" +" device const float* query [[buffer(0)]],\n" +" device const float* node_matrix [[buffer(1)]],\n" +" device const int* node_dims [[buffer(2)]],\n" +" constant EgCosineParams& p [[buffer(3)]],\n" +" device float* out_scores [[buffer(4)]],\n" +" uint gid [[thread_position_in_grid]])\n" +"{\n" +" if (gid >= p.n) return;\n" +" if (node_dims[gid] != int(p.dim)) { out_scores[gid] = -2.0f; return; }\n" +" device const float* row = node_matrix + (uint64_t)gid * (uint64_t)p.dim;\n" +" float dot0 = 0.0f, dot1 = 0.0f, dot2 = 0.0f, dot3 = 0.0f;\n" +" float na0 = 0.0f, na1 = 0.0f, na2 = 0.0f, na3 = 0.0f;\n" +" float nb0 = 0.0f, nb1 = 0.0f, nb2 = 0.0f, nb3 = 0.0f;\n" +" uint d = 0;\n" +" uint dim4 = p.dim & ~3u;\n" +" for (; d < dim4; d += 4) {\n" +" float a0 = row[d], b0 = query[d];\n" +" float a1 = row[d+1], b1 = query[d+1];\n" +" float a2 = row[d+2], b2 = query[d+2];\n" +" float a3 = row[d+3], b3 = query[d+3];\n" +" dot0 += a0*b0; dot1 += a1*b1; dot2 += a2*b2; dot3 += a3*b3;\n" +" na0 += a0*a0; na1 += a1*a1; na2 += a2*a2; na3 += a3*a3;\n" +" nb0 += b0*b0; nb1 += b1*b1; nb2 += b2*b2; nb3 += b3*b3;\n" +" }\n" +" float dot = (dot0 + dot1) + (dot2 + dot3);\n" +" float na = (na0 + na1) + (na2 + na3);\n" +" float nb = (nb0 + nb1) + (nb2 + nb3);\n" +" for (; d < p.dim; d++) {\n" +" float a = row[d], b = query[d];\n" +" dot += a*b; na += a*a; nb += b*b;\n" +" }\n" +" if (na <= 0.0f || nb <= 0.0f) { out_scores[gid] = -2.0f; return; }\n" +" out_scores[gid] = dot / sqrt(na * nb);\n" +"}\n" +"struct EgCosineMultiParams { uint n; uint dim; uint nq; };\n" +"kernel void eg_cosine_batch_multi_kernel(\n" +" device const float* queries [[buffer(0)]],\n" +" device const float* node_matrix [[buffer(1)]],\n" +" device const int* node_dims [[buffer(2)]],\n" +" constant EgCosineMultiParams& p [[buffer(3)]],\n" +" device float* out_scores [[buffer(4)]],\n" +" uint2 gid [[thread_position_in_grid]])\n" +"{\n" +" uint nid = gid.x, qid = gid.y;\n" +" if (nid >= p.n || qid >= p.nq) return;\n" +" uint64_t out_idx = (uint64_t)qid * (uint64_t)p.n + (uint64_t)nid;\n" +" if (node_dims[nid] != int(p.dim)) { out_scores[out_idx] = -2.0f; return; }\n" +" device const float* row = node_matrix + (uint64_t)nid * (uint64_t)p.dim;\n" +" device const float* query = queries + (uint64_t)qid * (uint64_t)p.dim;\n" +" float dot0 = 0.0f, dot1 = 0.0f, dot2 = 0.0f, dot3 = 0.0f;\n" +" float na0 = 0.0f, na1 = 0.0f, na2 = 0.0f, na3 = 0.0f;\n" +" float nb0 = 0.0f, nb1 = 0.0f, nb2 = 0.0f, nb3 = 0.0f;\n" +" uint d = 0;\n" +" uint dim4 = p.dim & ~3u;\n" +" for (; d < dim4; d += 4) {\n" +" float a0 = row[d], b0 = query[d];\n" +" float a1 = row[d+1], b1 = query[d+1];\n" +" float a2 = row[d+2], b2 = query[d+2];\n" +" float a3 = row[d+3], b3 = query[d+3];\n" +" dot0 += a0*b0; dot1 += a1*b1; dot2 += a2*b2; dot3 += a3*b3;\n" +" na0 += a0*a0; na1 += a1*a1; na2 += a2*a2; na3 += a3*a3;\n" +" nb0 += b0*b0; nb1 += b1*b1; nb2 += b2*b2; nb3 += b3*b3;\n" +" }\n" +" float dot = (dot0 + dot1) + (dot2 + dot3);\n" +" float na = (na0 + na1) + (na2 + na3);\n" +" float nb = (nb0 + nb1) + (nb2 + nb3);\n" +" for (; d < p.dim; d++) {\n" +" float a = row[d], b = query[d];\n" +" dot += a*b; na += a*a; nb += b*b;\n" +" }\n" +" if (na <= 0.0f || nb <= 0.0f) { out_scores[out_idx] = -2.0f; return; }\n" +" out_scores[out_idx] = dot / sqrt(na * nb);\n" +"}\n"; +/* ── END embedded shader source ── */ + +typedef struct EgCosineParamsC { uint32_t n; uint32_t dim; } EgCosineParamsC; +typedef struct EgCosineMultiParamsC { uint32_t n; uint32_t dim; uint32_t nq; } EgCosineMultiParamsC; + +static id g_device = nil; +static id g_queue = nil; +static id g_pipeline = nil; /* single-query kernel */ +static id g_pipeline_multi = nil; /* multi-query kernel */ +static bool g_init_attempted = false; +static bool g_init_ok = false; + +/* Lazy, one-time setup. Never throws — every Metal call here is the + * "returns nil/NSError on failure" flavor, not an exception-throwing one. */ +static bool eg_metal_ensure_init(void) { + if (g_init_attempted) return g_init_ok; + g_init_attempted = true; + + @autoreleasepool { + id dev = MTLCreateSystemDefaultDevice(); + if (!dev) return false; + + id q = [dev newCommandQueue]; + if (!q) return false; + + NSError* err = nil; + NSString* src = [NSString stringWithUTF8String:kEgCosineBatchMetalSrc]; + MTLCompileOptions* opts = [MTLCompileOptions new]; + id lib = [dev newLibraryWithSource:src options:opts error:&err]; + if (!lib) return false; + + id fn = [lib newFunctionWithName:@"eg_cosine_batch_kernel"]; + if (!fn) return false; + id pipe = [dev newComputePipelineStateWithFunction:fn error:&err]; + if (!pipe) return false; + + id fnMulti = [lib newFunctionWithName:@"eg_cosine_batch_multi_kernel"]; + if (!fnMulti) return false; + id pipeMulti = [dev newComputePipelineStateWithFunction:fnMulti error:&err]; + if (!pipeMulti) return false; + + g_device = dev; + g_queue = q; + g_pipeline = pipe; + g_pipeline_multi = pipeMulti; + g_init_ok = true; + return true; + } +} + +static bool mh_available(void) { + return eg_metal_ensure_init(); +} + +static bool mh_batch(const float* query, int32_t qdim, + const float* const* node_ptrs, + const int32_t* node_dims, + int32_t n, + double* out_scores) { + if (!query || qdim <= 0 || !node_ptrs || !node_dims || n <= 0 || !out_scores) return false; + if (!eg_metal_ensure_init()) return false; + + @autoreleasepool { + const size_t dim = (size_t)qdim; + const size_t nu = (size_t)n; + + /* Gather into a packed row-major matrix — EngramNode.emb is one + * malloc per node, not a contiguous array, so this copy is + * unavoidable regardless of backend. Rows whose real dim doesn't + * match qdim are zero-filled (harmless: the kernel sentinels them + * via node_dims before ever reading the row). */ + float* matrix = (float*)calloc(nu * dim, sizeof(float)); + int32_t* dims_i32 = (int32_t*)malloc(nu * sizeof(int32_t)); + if (!matrix || !dims_i32) { free(matrix); free(dims_i32); return false; } + + for (size_t i = 0; i < nu; i++) { + dims_i32[i] = node_dims[i]; + if (node_ptrs[i] && node_dims[i] == qdim) { + memcpy(matrix + i * dim, node_ptrs[i], dim * sizeof(float)); + } + /* else: leave zero-filled; node_dims[i] != qdim (or missing) + * makes the kernel sentinel it to -2.0 without reading the row. */ + } + + id bufQuery = [g_device newBufferWithBytes:query + length:dim * sizeof(float) + options:MTLResourceStorageModeShared]; + id bufMatrix = [g_device newBufferWithBytes:matrix + length:nu * dim * sizeof(float) + options:MTLResourceStorageModeShared]; + id bufDims = [g_device newBufferWithBytes:dims_i32 + length:nu * sizeof(int32_t) + options:MTLResourceStorageModeShared]; + EgCosineParamsC params = { (uint32_t)nu, (uint32_t)dim }; + id bufParams = [g_device newBufferWithBytes:¶ms + length:sizeof(params) + options:MTLResourceStorageModeShared]; + id bufOut = [g_device newBufferWithLength:nu * sizeof(float) + options:MTLResourceStorageModeShared]; + + free(matrix); free(dims_i32); + + if (!bufQuery || !bufMatrix || !bufDims || !bufParams || !bufOut) return false; + + id cmd = [g_queue commandBuffer]; + if (!cmd) return false; + id enc = [cmd computeCommandEncoder]; + if (!enc) return false; + + [enc setComputePipelineState:g_pipeline]; + [enc setBuffer:bufQuery offset:0 atIndex:0]; + [enc setBuffer:bufMatrix offset:0 atIndex:1]; + [enc setBuffer:bufDims offset:0 atIndex:2]; + [enc setBuffer:bufParams offset:0 atIndex:3]; + [enc setBuffer:bufOut offset:0 atIndex:4]; + + NSUInteger tgSize = g_pipeline.maxTotalThreadsPerThreadgroup; + if (tgSize > 256) tgSize = 256; + if (tgSize < 1) tgSize = 1; + MTLSize gridSize = MTLSizeMake(nu, 1, 1); + MTLSize threadgroupSize = MTLSizeMake(tgSize, 1, 1); + [enc dispatchThreads:gridSize threadsPerThreadgroup:threadgroupSize]; + [enc endEncoding]; + + [cmd commit]; + [cmd waitUntilCompleted]; + + if (cmd.status != MTLCommandBufferStatusCompleted) return false; + + const float* results = (const float*)bufOut.contents; + if (!results) return false; + for (size_t i = 0; i < nu; i++) out_scores[i] = (double)results[i]; + return true; + } +} + +static bool mh_batch_multi(const float* queries, int32_t qdim, int32_t nq, + const float* const* node_ptrs, + const int32_t* node_dims, + int32_t n, + double* out_scores) { + if (!queries || qdim <= 0 || nq <= 0 || !node_ptrs || !node_dims || n <= 0 || !out_scores) return false; + if (!eg_metal_ensure_init()) return false; + + @autoreleasepool { + const size_t dim = (size_t)qdim; + const size_t nu = (size_t)n; + const size_t nqu = (size_t)nq; + + float* matrix = (float*)calloc(nu * dim, sizeof(float)); + int32_t* dims_i32 = (int32_t*)malloc(nu * sizeof(int32_t)); + if (!matrix || !dims_i32) { free(matrix); free(dims_i32); return false; } + + for (size_t i = 0; i < nu; i++) { + dims_i32[i] = node_dims[i]; + if (node_ptrs[i] && node_dims[i] == qdim) { + memcpy(matrix + i * dim, node_ptrs[i], dim * sizeof(float)); + } + } + + /* This is the ONE upload of node_matrix for the whole nq-query batch — + * the fix for the measured re-upload-per-query slowdown. */ + id bufMatrix = [g_device newBufferWithBytes:matrix + length:nu * dim * sizeof(float) + options:MTLResourceStorageModeShared]; + id bufDims = [g_device newBufferWithBytes:dims_i32 + length:nu * sizeof(int32_t) + options:MTLResourceStorageModeShared]; + id bufQueries = [g_device newBufferWithBytes:queries + length:nqu * dim * sizeof(float) + options:MTLResourceStorageModeShared]; + EgCosineMultiParamsC params = { (uint32_t)nu, (uint32_t)dim, (uint32_t)nqu }; + id bufParams = [g_device newBufferWithBytes:¶ms + length:sizeof(params) + options:MTLResourceStorageModeShared]; + id bufOut = [g_device newBufferWithLength:nqu * nu * sizeof(float) + options:MTLResourceStorageModeShared]; + + free(matrix); free(dims_i32); + + if (!bufMatrix || !bufDims || !bufQueries || !bufParams || !bufOut) return false; + + id cmd = [g_queue commandBuffer]; + if (!cmd) return false; + id enc = [cmd computeCommandEncoder]; + if (!enc) return false; + + [enc setComputePipelineState:g_pipeline_multi]; + [enc setBuffer:bufQueries offset:0 atIndex:0]; + [enc setBuffer:bufMatrix offset:0 atIndex:1]; + [enc setBuffer:bufDims offset:0 atIndex:2]; + [enc setBuffer:bufParams offset:0 atIndex:3]; + [enc setBuffer:bufOut offset:0 atIndex:4]; + + /* 2D dispatch: x over nodes, y over queries. Threadgroup width picked + * from the pipeline's own limit, height fixed at 1 — nq is typically + * small (tens to low hundreds) relative to n (thousands+), so tiling + * the wide axis (n) is what matters for occupancy. */ + NSUInteger tgWidth = g_pipeline_multi.maxTotalThreadsPerThreadgroup; + if (tgWidth > 256) tgWidth = 256; + if (tgWidth < 1) tgWidth = 1; + MTLSize gridSize = MTLSizeMake(nu, nqu, 1); + MTLSize threadgroupSize = MTLSizeMake(tgWidth, 1, 1); + [enc dispatchThreads:gridSize threadsPerThreadgroup:threadgroupSize]; + [enc endEncoding]; + + [cmd commit]; + [cmd waitUntilCompleted]; + + if (cmd.status != MTLCommandBufferStatusCompleted) return false; + + const float* results = (const float*)bufOut.contents; + if (!results) return false; + for (size_t i = 0; i < nqu * nu; i++) out_scores[i] = (double)results[i]; + return true; + } +} + +static const EgCosineBatchStrategy g_metal_hand_strategy = { + .name = "metal-hand", + .available = mh_available, + .batch = mh_batch, + .batch_multi = mh_batch_multi, +}; + +const EgCosineBatchStrategy* eg_cosine_batch_strategy_metal_hand(void) { + return &g_metal_hand_strategy; +} diff --git a/lang/runtime/el_audio_darwin.m b/lang/runtime/el_audio_darwin.m new file mode 100644 index 0000000..da43a64 --- /dev/null +++ b/lang/runtime/el_audio_darwin.m @@ -0,0 +1,525 @@ +/* el_audio_darwin.m — the SPEAKER realizer. El's native audio output on Darwin. + * + * WHY THIS FILE EXISTS. + * + * Neuron could already turn meaning into samples — the render path in + * elp/src/speech.el superposes formant resonances over a glottal source and + * produces PCM. What it could not do was make a sound. Every path from those + * samples to the air ran outside the language: a 939-line Swift program + * (peripheral/src/periph.swift) that shelled out to /usr/bin/afplay. So the + * voice was not a capability of El or of Neuron. It was a separate binary + * standing next to them, and "speak" meant "ask that binary to speak." + * + * A speaker is not a language feature the way a string is, but it is exactly + * the kind of thing a runtime owns: a device. El already owns the filesystem, + * the network, the clock, and a graph. It should own the one output device that + * makes it audible. After this file, `speak` is an El operation. + * + * WHY IT IS A REALIZER AND NOT PURE EL. + * + * This is the boundary the whole design turns on. Everything ABOVE the sample + * buffer is arithmetic and belongs in El: formant geometry, superposition, + * envelopes, WAV framing, the voice signature. Everything in this file is the + * part that cannot be arithmetic — handing a buffer to CoreAudio and waiting + * for the hardware to drain it. There is no way to express "the DAC has now + * played these samples" in El, and there should not be. So the split is: El + * computes the sound, the realizer emits it, and the realizer is as thin as it + * can possibly be — it makes no decisions about content, it has no opinion + * about audio, and it cannot synthesize anything. + * + * The precedent is eg_cosine_batch_strategy_metal_hand.m: a platform-bound + * capability compiled as its OWN translation unit, declared in el_runtime.h, + * and linked in where the platform supports it. Deliberately NOT a patch to + * el_runtime.c — adding a device to El must not mean editing the core runtime, + * for the same reason adding a modality must not (see el_runtime.c's realizer + * registry: a realizer is resolved by name, so new organs never touch the + * middle of the language). el_audio_null.c is the same two entry points for + * every platform that is not Darwin, so El code that speaks still links + * everywhere and simply reports that it has no speaker. + * + * WHY AudioQueue AND NOT afplay. + * + * afplay is a process. Using it means the sound Neuron makes is a file it wrote + * and asked something else to open — which forces every utterance through the + * disk, cannot start until the whole utterance exists, and puts a fork/exec + * between the intent to speak and the sound. AudioQueue takes the samples + * directly out of memory. Nothing is written, nothing is spawned, and a caller + * that wants to stream can push buffers as it renders them. + * + * AudioToolbox ships with macOS, so this stays own-core: no cloud, no library + * to install, no model. The output is the local speaker and nothing leaves the + * machine — there is no network path in this file at all, by construction. + */ + +#import +#import +#include +#include +#include +#include +#include "el_runtime.h" + +/* Three buffers is the standard AudioQueue depth: one being played by the + * hardware, one queued behind it, one being refilled. Fewer risks a gap on a + * busy machine; more only adds latency before the first sound. */ +#define EL_AQ_NBUF 3 +#define EL_AQ_FRAMES 8192 + +typedef struct { + const int16_t* pcm; + int64_t frames; + int64_t pos; + volatile int inflight; /* buffers CoreAudio still owns */ + volatile int drained; /* set once the last buffer has been played */ +} ElAqState; + +/* Called on an AudioQueue-internal thread each time a buffer finishes playing. + * Refills and re-enqueues while samples remain; when the source is exhausted it + * lets the buffer die and counts it out. `drained` flips only when the queue is + * holding nothing, which is what makes the play call synchronous without + * clipping the tail — the same reason periph.swift used .dataPlayedBack rather + * than treating "consumed" as "heard". */ +static void el_aq_callback(void* userData, AudioQueueRef q, AudioQueueBufferRef buf) { + ElAqState* st = (ElAqState*)userData; + int64_t remain = st->frames - st->pos; + if (remain <= 0) { + if (--st->inflight <= 0) st->drained = 1; + return; + } + int64_t n = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; + memcpy(buf->mAudioData, st->pcm + st->pos, (size_t)n * sizeof(int16_t)); + buf->mAudioDataByteSize = (UInt32)(n * (int64_t)sizeof(int16_t)); + st->pos += n; + if (AudioQueueEnqueueBuffer(q, buf, 0, NULL) != noErr) { + if (--st->inflight <= 0) st->drained = 1; + } +} + +/* Play a 16-bit mono PCM buffer out the default output device, blocking until + * the hardware has actually finished. Returns 1 on success, 0 on any failure — + * never throws, never hangs indefinitely. */ +static int el_audio_play_raw(const int16_t* pcm, int64_t frames, int32_t sample_rate) { + if (!pcm || frames <= 0 || sample_rate <= 0) return 0; + + AudioStreamBasicDescription fmt; + memset(&fmt, 0, sizeof(fmt)); + fmt.mSampleRate = (Float64)sample_rate; + fmt.mFormatID = kAudioFormatLinearPCM; + fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked; + fmt.mFramesPerPacket = 1; + fmt.mChannelsPerFrame = 1; + fmt.mBitsPerChannel = 16; + fmt.mBytesPerFrame = 2; + fmt.mBytesPerPacket = 2; + + ElAqState st; + memset(&st, 0, sizeof(st)); + st.pcm = pcm; + st.frames = frames; + + AudioQueueRef q = NULL; + /* NULL run loop => callbacks arrive on an AudioQueue-internal thread, so + * this function can simply wait rather than having to pump a run loop it + * does not own. El programs are not required to have one. */ + if (AudioQueueNewOutput(&fmt, el_aq_callback, &st, NULL, NULL, 0, &q) != noErr || !q) { + return 0; + } + + AudioQueueBufferRef bufs[EL_AQ_NBUF]; + int prepared = 0; + for (int i = 0; i < EL_AQ_NBUF; i++) { + if (AudioQueueAllocateBuffer(q, EL_AQ_FRAMES * sizeof(int16_t), &bufs[i]) != noErr) break; + prepared++; + } + if (prepared == 0) { AudioQueueDispose(q, true); return 0; } + + /* Prime: fill what we can before starting, so playback begins immediately + * rather than after the first underrun. */ + for (int i = 0; i < prepared; i++) { + int64_t remain = st.frames - st.pos; + if (remain <= 0) break; + int64_t n = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; + memcpy(bufs[i]->mAudioData, st.pcm + st.pos, (size_t)n * sizeof(int16_t)); + bufs[i]->mAudioDataByteSize = (UInt32)(n * (int64_t)sizeof(int16_t)); + st.pos += n; + if (AudioQueueEnqueueBuffer(q, bufs[i], 0, NULL) != noErr) break; + st.inflight++; + } + if (st.inflight == 0) { AudioQueueDispose(q, true); return 0; } + + if (AudioQueueStart(q, NULL) != noErr) { AudioQueueDispose(q, true); return 0; } + + /* Bound the wait by the material's own duration plus a margin. A speaker + * that wedges a program is worse than a speaker that gives up. */ + double seconds = (double)frames / (double)sample_rate; + int64_t max_us = (int64_t)((seconds + 5.0) * 1000000.0); + int64_t waited = 0; + const int64_t tick = 5000; /* 5 ms */ + while (!st.drained && waited < max_us) { + usleep((useconds_t)tick); + waited += tick; + } + + AudioQueueStop(q, true); + AudioQueueDispose(q, true); + return st.drained ? 1 : 0; +} + +/* ── El entry points ──────────────────────────────────────────────────────── + * Declared in el_runtime.h; see there for the El-facing contract. */ + +/* 1 when this build has a real speaker behind it. El code should ask before + * speaking so the no-speaker case is a reported condition, not a silence that + * looks like success. */ +el_val_t speaker_available(void) { + return (el_val_t)1; +} + +el_val_t speaker_name(void) { + return EL_STR("coreaudio-audioqueue"); +} + +/* Play an El [Int] of 16-bit samples. Values are clamped, not wrapped: a + * render that overshoots should distort at the rails the way real clipping + * does, rather than invert phase and produce a sound nothing in the signal + * chain intended. */ +el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate) { + int64_t n = (int64_t)el_list_len(samples); + int32_t sr = (int32_t)sample_rate; + if (n <= 0 || sr <= 0) return (el_val_t)0; + + int16_t* pcm = (int16_t*)malloc((size_t)n * sizeof(int16_t)); + if (!pcm) return (el_val_t)0; + + for (int64_t i = 0; i < n; i++) { + int64_t v = (int64_t)el_list_get(samples, (el_val_t)i); + if (v > 32767) v = 32767; + if (v < -32768) v = -32768; + pcm[i] = (int16_t)v; + } + + int ok = el_audio_play_raw(pcm, n, sr); + free(pcm); + return (el_val_t)(ok ? 1 : 0); +} + +/* ── Asynchronous playback ─────────────────────────────────────────────────── + * + * converse needs this and a blocking play cannot give it. Barge-in means + * stopping ON THE SPOT when the user starts talking — not at the end of the + * current buffer, and certainly not at the end of the utterance. So the async + * path keeps one queue alive, reports how far the hardware actually got, and + * can be halted mid-buffer. + * + * `played_frames` is what makes an interrupted utterance resumable at the + * sample rather than at the segment: it is the position the DAC reached, not + * the position we enqueued to, and those differ by up to the full queue depth. + * + * One utterance at a time. A second async play stops the first — a mouth that + * can say two things at once is not a feature. */ + +static AudioQueueRef g_aq = NULL; +static ElAqState* g_aq_state = NULL; +static int16_t* g_aq_pcm = NULL; +static int32_t g_aq_sr = 0; + +static void el_audio_teardown(void) { + if (g_aq) { + AudioQueueStop(g_aq, true); + AudioQueueDispose(g_aq, true); + g_aq = NULL; + } + free(g_aq_pcm); g_aq_pcm = NULL; + free(g_aq_state); g_aq_state = NULL; + g_aq_sr = 0; +} + +el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate) { + el_audio_teardown(); + + int64_t n = (int64_t)el_list_len(samples); + int32_t sr = (int32_t)sample_rate; + if (n <= 0 || sr <= 0) return (el_val_t)0; + + g_aq_pcm = (int16_t*)malloc((size_t)n * sizeof(int16_t)); + if (!g_aq_pcm) return (el_val_t)0; + for (int64_t i = 0; i < n; i++) { + int64_t v = (int64_t)el_list_get(samples, (el_val_t)i); + if (v > 32767) v = 32767; + if (v < -32768) v = -32768; + g_aq_pcm[i] = (int16_t)v; + } + + g_aq_state = (ElAqState*)calloc(1, sizeof(ElAqState)); + if (!g_aq_state) { el_audio_teardown(); return (el_val_t)0; } + g_aq_state->pcm = g_aq_pcm; + g_aq_state->frames = n; + g_aq_sr = sr; + + AudioStreamBasicDescription fmt; + memset(&fmt, 0, sizeof(fmt)); + fmt.mSampleRate = (Float64)sr; + fmt.mFormatID = kAudioFormatLinearPCM; + fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked; + fmt.mFramesPerPacket = 1; + fmt.mChannelsPerFrame = 1; + fmt.mBitsPerChannel = 16; + fmt.mBytesPerFrame = 2; + fmt.mBytesPerPacket = 2; + + if (AudioQueueNewOutput(&fmt, el_aq_callback, g_aq_state, NULL, NULL, 0, &g_aq) != noErr || !g_aq) { + el_audio_teardown(); + return (el_val_t)0; + } + + for (int i = 0; i < EL_AQ_NBUF; i++) { + int64_t remain = g_aq_state->frames - g_aq_state->pos; + if (remain <= 0) break; + AudioQueueBufferRef b = NULL; + if (AudioQueueAllocateBuffer(g_aq, EL_AQ_FRAMES * sizeof(int16_t), &b) != noErr) break; + int64_t k = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; + memcpy(b->mAudioData, g_aq_state->pcm + g_aq_state->pos, (size_t)k * sizeof(int16_t)); + b->mAudioDataByteSize = (UInt32)(k * (int64_t)sizeof(int16_t)); + g_aq_state->pos += k; + if (AudioQueueEnqueueBuffer(g_aq, b, 0, NULL) != noErr) break; + g_aq_state->inflight++; + } + if (g_aq_state->inflight == 0) { el_audio_teardown(); return (el_val_t)0; } + + if (AudioQueueStart(g_aq, NULL) != noErr) { el_audio_teardown(); return (el_val_t)0; } + return (el_val_t)1; +} + +el_val_t speaker_playing(void) { + if (!g_aq || !g_aq_state) return (el_val_t)0; + return (el_val_t)(g_aq_state->drained ? 0 : 1); +} + +/* Frames the DAC has actually rendered. AudioQueueGetCurrentTime's mSampleTime + * is relative to queue start, which is exactly the "where was I really" figure + * a resumable utterance needs. Falls back to the enqueued position if the + * timeline is unavailable (it is, briefly, right after start). */ +el_val_t speaker_played_frames(void) { + if (!g_aq || !g_aq_state) return (el_val_t)0; + AudioTimeStamp ts; + memset(&ts, 0, sizeof(ts)); + Boolean discontinuity = false; + if (AudioQueueGetCurrentTime(g_aq, NULL, &ts, &discontinuity) == noErr && + (ts.mFlags & kAudioTimeStampSampleTimeValid)) { + int64_t played = (int64_t)ts.mSampleTime; + if (played < 0) played = 0; + if (played > g_aq_state->frames) played = g_aq_state->frames; + return (el_val_t)played; + } + return (el_val_t)g_aq_state->pos; +} + +/* Pause where we are, keeping the queue and its position intact. + * + * This is the difference between barge-in and "finish the buffer". The moment + * the microphone hears speech, output must stop AT THAT SAMPLE — a listener + * experiences even 200ms of continued talking as being talked over. Pause + * rather than stop because the interruption might turn out to be a backchannel + * ("mm-hm"), and the right response to a backchannel is to carry on as though + * nothing happened, which requires the queue to still be exactly where it was. + * A stop-and-restart would re-attack the buffer and be audible as a stutter. */ +el_val_t speaker_pause(void) { + if (!g_aq) return (el_val_t)0; + return (el_val_t)(AudioQueuePause(g_aq) == noErr ? 1 : 0); +} + +el_val_t speaker_resume(void) { + if (!g_aq) return (el_val_t)0; + return (el_val_t)(AudioQueueStart(g_aq, NULL) == noErr ? 1 : 0); +} + +el_val_t speaker_stop(void) { + if (!g_aq) return (el_val_t)0; + /* immediate: do NOT let the queue finish what it is holding */ + AudioQueueStop(g_aq, true); + el_audio_teardown(); + return (el_val_t)1; +} + +/* Decode a 16-bit RIFF/WAVE into a freshly malloc'd mono int16 buffer. + * Returns frames, or 0 on any failure; *out is set only on success. Shared by + * the blocking and async WAV paths. */ +static int64_t el_wav_load(const char* path, int16_t** out, int32_t* out_sr) { + if (!path || !out) return 0; + FILE* f = fopen(path, "rb"); + if (!f) return 0; + if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return 0; } + long size = ftell(f); + if (size <= 44) { fclose(f); return 0; } + rewind(f); + unsigned char* d = (unsigned char*)malloc((size_t)size); + if (!d) { fclose(f); return 0; } + size_t got = fread(d, 1, (size_t)size, f); + fclose(f); + if (got != (size_t)size) { free(d); return 0; } + if (memcmp(d, "RIFF", 4) != 0 || memcmp(d + 8, "WAVE", 4) != 0) { free(d); return 0; } + + int32_t sr = 0, channels = 0, bits = 0; + long dataOff = -1, dataLen = 0, o = 12; + /* Chunk-walk rather than assuming fmt-then-data at fixed offsets: recorders + * routinely interleave JUNK/FLLR padding, and a fixed-offset parser reads + * padding as audio. */ + while (o + 8 <= size) { + long sz = (long)d[o+4] | ((long)d[o+5] << 8) | ((long)d[o+6] << 16) | ((long)d[o+7] << 24); + if (sz < 0) break; + if (memcmp(d + o, "fmt ", 4) == 0 && o + 24 <= size) { + channels = (int32_t)(d[o+10] | (d[o+11] << 8)); + sr = (int32_t)((long)d[o+12] | ((long)d[o+13] << 8) | ((long)d[o+14] << 16) | ((long)d[o+15] << 24)); + bits = (int32_t)(d[o+22] | (d[o+23] << 8)); + } else if (memcmp(d + o, "data", 4) == 0) { + dataOff = o + 8; + dataLen = sz; + if (dataOff + dataLen > size) dataLen = size - dataOff; + } + o += 8 + sz + (sz & 1); + } + if (dataOff < 0 || sr <= 0 || bits != 16 || channels < 1 || dataLen <= 0) { free(d); return 0; } + + long frames = dataLen / (2 * channels); + int16_t* pcm = (int16_t*)malloc((size_t)frames * sizeof(int16_t)); + if (!pcm) { free(d); return 0; } + for (long i = 0; i < frames; i++) { + long b = dataOff + i * 2 * channels; + pcm[i] = (int16_t)((unsigned)d[b] | ((unsigned)d[b+1] << 8)); + } + free(d); + *out = pcm; + if (out_sr) *out_sr = sr; + return (int64_t)frames; +} + +/* Async WAV playback. converse speaks PRE-RENDERED segments and must keep + * listening while it does, so it needs the file on the queue without blocking + * and needs to be able to stop it mid-buffer. Going through the file rather + * than an El [Int] also avoids marshalling a million-element list per segment + * for audio the caller never intends to look at. */ +el_val_t speaker_play_wav_async(el_val_t path) { + const char* p = EL_CSTR(path); + if (!p) return (el_val_t)0; + + el_audio_teardown(); + + int32_t sr = 0; + int16_t* pcm = NULL; + int64_t frames = el_wav_load(p, &pcm, &sr); + if (frames <= 0 || !pcm) { free(pcm); return (el_val_t)0; } + + g_aq_pcm = pcm; + g_aq_sr = sr; + g_aq_state = (ElAqState*)calloc(1, sizeof(ElAqState)); + if (!g_aq_state) { el_audio_teardown(); return (el_val_t)0; } + g_aq_state->pcm = g_aq_pcm; + g_aq_state->frames = frames; + + AudioStreamBasicDescription fmt; + memset(&fmt, 0, sizeof(fmt)); + fmt.mSampleRate = (Float64)sr; + fmt.mFormatID = kAudioFormatLinearPCM; + fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked; + fmt.mFramesPerPacket = 1; + fmt.mChannelsPerFrame = 1; + fmt.mBitsPerChannel = 16; + fmt.mBytesPerFrame = 2; + fmt.mBytesPerPacket = 2; + + if (AudioQueueNewOutput(&fmt, el_aq_callback, g_aq_state, NULL, NULL, 0, &g_aq) != noErr || !g_aq) { + el_audio_teardown(); + return (el_val_t)0; + } + for (int i = 0; i < EL_AQ_NBUF; i++) { + int64_t remain = g_aq_state->frames - g_aq_state->pos; + if (remain <= 0) break; + AudioQueueBufferRef b = NULL; + if (AudioQueueAllocateBuffer(g_aq, EL_AQ_FRAMES * sizeof(int16_t), &b) != noErr) break; + int64_t k = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES; + memcpy(b->mAudioData, g_aq_state->pcm + g_aq_state->pos, (size_t)k * sizeof(int16_t)); + b->mAudioDataByteSize = (UInt32)(k * (int64_t)sizeof(int16_t)); + g_aq_state->pos += k; + if (AudioQueueEnqueueBuffer(g_aq, b, 0, NULL) != noErr) break; + g_aq_state->inflight++; + } + if (g_aq_state->inflight == 0) { el_audio_teardown(); return (el_val_t)0; } + if (AudioQueueStart(g_aq, NULL) != noErr) { el_audio_teardown(); return (el_val_t)0; } + return (el_val_t)1; +} + +/* Total frames and sample rate of a WAV, without playing it — wav-info, and the + * duration converse needs to compute progress through a segment. */ +el_val_t wav_frames(el_val_t path) { + const char* p = EL_CSTR(path); + int16_t* pcm = NULL; int32_t sr = 0; + int64_t n = el_wav_load(p, &pcm, &sr); + free(pcm); + return (el_val_t)n; +} + +el_val_t wav_rate(el_val_t path) { + const char* p = EL_CSTR(path); + int16_t* pcm = NULL; int32_t sr = 0; + int64_t n = el_wav_load(p, &pcm, &sr); + free(pcm); + return (el_val_t)(n > 0 ? sr : 0); +} + +/* Play a 16-bit mono RIFF/WAVE file. Present because the render already knows + * how to write a WAV and a caller may reasonably want to hear one back without + * re-rendering it; the parse is deliberately minimal and chunk-walking, so the + * JUNK/FLLR padding that recorders emit does not defeat it. */ +el_val_t speaker_play_wav(el_val_t path) { + const char* p = EL_CSTR(path); + if (!p) return (el_val_t)0; + FILE* f = fopen(p, "rb"); + if (!f) return (el_val_t)0; + + if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return (el_val_t)0; } + long size = ftell(f); + if (size <= 44) { fclose(f); return (el_val_t)0; } + rewind(f); + + unsigned char* d = (unsigned char*)malloc((size_t)size); + if (!d) { fclose(f); return (el_val_t)0; } + size_t got = fread(d, 1, (size_t)size, f); + fclose(f); + if (got != (size_t)size) { free(d); return (el_val_t)0; } + + if (memcmp(d, "RIFF", 4) != 0 || memcmp(d + 8, "WAVE", 4) != 0) { free(d); return (el_val_t)0; } + + int32_t sr = 0, channels = 0, bits = 0; + long dataOff = -1, dataLen = 0; + long o = 12; + while (o + 8 <= size) { + long sz = (long)d[o+4] | ((long)d[o+5] << 8) | ((long)d[o+6] << 16) | ((long)d[o+7] << 24); + if (sz < 0) break; + if (memcmp(d + o, "fmt ", 4) == 0 && o + 24 <= size) { + channels = (int32_t)(d[o+10] | (d[o+11] << 8)); + sr = (int32_t)((long)d[o+12] | ((long)d[o+13] << 8) | ((long)d[o+14] << 16) | ((long)d[o+15] << 24)); + bits = (int32_t)(d[o+22] | (d[o+23] << 8)); + } else if (memcmp(d + o, "data", 4) == 0) { + dataOff = o + 8; + dataLen = sz; + if (dataOff + dataLen > size) dataLen = size - dataOff; + } + o += 8 + sz + (sz & 1); + } + if (dataOff < 0 || sr <= 0 || bits != 16 || channels < 1 || dataLen <= 0) { free(d); return (el_val_t)0; } + + long frames = dataLen / (2 * channels); + int16_t* pcm = (int16_t*)malloc((size_t)frames * sizeof(int16_t)); + if (!pcm) { free(d); return (el_val_t)0; } + /* Take channel 0; the organ is mono by design and downmixing would be an + * opinion about content this layer is not entitled to have. */ + for (long i = 0; i < frames; i++) { + long b = dataOff + i * 2 * channels; + pcm[i] = (int16_t)((unsigned)d[b] | ((unsigned)d[b+1] << 8)); + } + free(d); + + int ok = el_audio_play_raw(pcm, frames, sr); + free(pcm); + return (el_val_t)(ok ? 1 : 0); +} diff --git a/lang/runtime/el_capture_darwin.m b/lang/runtime/el_capture_darwin.m new file mode 100644 index 0000000..3651408 --- /dev/null +++ b/lang/runtime/el_capture_darwin.m @@ -0,0 +1,841 @@ +/* el_capture_darwin.m — the MICROPHONE and CAMERA realizers. El's afferent + * organ on Darwin: the two entry points through which the world gets in. + * + * WHY THIS FILE EXISTS, AND WHY IT IS A REALIZER RATHER THAN PURE EL. + * + * el_audio_darwin.m argued the efferent half of this: El can compute a sound + * but it cannot make one, because "the DAC has now played these samples" is not + * a fact any amount of arithmetic can produce. This file is the same argument + * run backwards. El can compute *about* a sound — it can window it, take its + * autocorrelation, run Levinson-Durbin over that, find the formant peaks in the + * resulting all-pole envelope, and hand back a voiceprint — but it cannot ASK. + * There is no expression in El, and there must not be, whose value is "the next + * 1024 frames the microphone hears" or "what the camera is pointed at right + * now." Those are not computed; they are *requested*, from an operating system + * that owns the device, mediates consent for it, and delivers the answer on a + * thread of its choosing whenever it feels like it. Asking is the one primitive + * operation here. Everything else in this file is bookkeeping around the ask. + * + * So the line is drawn exactly where el_audio_darwin.m drew it, at the sample + * buffer, and it is drawn on purpose: + * + * BELOW the line (here): open the device, honour the OS permission gate, + * install a tap or a frame delegate, convert whatever the hardware happens to + * emit into the one shape El asked for, and hand it up. No opinions about + * content. No analysis. No decisions. + * + * ABOVE the line (El): energy, zero-crossing rate, spectral centroid, F0 by + * autocorrelation, LPC, formants F1-F5, the compact descriptors, the + * scene-geometry grid, the yield-or-hold turn-taking decision. All of it is + * arithmetic over a buffer, all of it belongs in El, and none of it appears + * below. The reference this file ports — peripheral/src/periph.swift — held + * both halves, and that was the problem worth fixing: the descriptors were + * trapped in a 939-line binary standing next to the language instead of being + * written in it. Porting the *whole* of periph.swift down here would have + * reproduced that mistake in C. Only the ask came down. + * + * The precedent for the file's SHAPE is eg_cosine_batch_strategy_metal_hand.m: + * a platform-bound capability compiled as its own translation unit, declared in + * el_runtime.h, linked in where the platform supports it, and deliberately NOT + * a patch to the middle of el_runtime.c. Acquiring a device must not mean + * editing the language, for the same reason acquiring a modality must not (see + * the realizer registry: organs are resolved by name). el_peripheral_null.c is + * the same entry points everywhere else, so El code that listens still links on + * every platform and merely reports having no ear. + * + * FAIL CLOSED, ALWAYS. + * + * A capture path that returns plausible-looking zeros when it was denied is + * worse than one that returns nothing, because the caller cannot tell the + * difference between a silent room and a refused microphone. Every entry point + * here checks AVCaptureDevice's authorization status BEFORE touching hardware + * and returns the empty value — an empty list, a 0 map — on anything short of + * .authorized. mic_available() and camera_available() report that state WITHOUT + * prompting, so El can ask "may I?" without the act of asking being a prompt. + * + * NEVER HANG. + * + * Every wait in this file is bounded: 30s on a permission prompt (the user has + * to walk to the dialog), seconds+5 on a capture of `seconds`, 10s on a camera + * frame. An organ that wedges the program holding it is not an organ, it is a + * fault. Every path also tears the device down on the way out, including the + * failure paths, so a timed-out capture does not leave the mic light on. + * + * OWN-CORE AND LOCAL BY CONSTRUCTION. + * + * AVFoundation, CoreVideo, CoreGraphics and ImageIO ship with macOS. There is + * no third-party library here, no model, and — the part that matters — no + * network path of any kind. Samples and pixels move from local hardware into an + * El value and stop. periph.swift had a URLSession in it; this file has no + * socket, no URL, and nothing that could grow one without being obvious in + * review. Consent is enforced above this layer in El and below it by the OS; + * this layer's whole contribution to that is refusing to proceed. + * + * DISCLOSURE. + * + * Every actual device touch writes one line to stderr and flushes it, before + * the device opens. stderr and not stdout: a program that announces "I am about + * to open the microphone" on stdout has corrupted its own output, and the + * caller must be able to separate the answer from how it was obtained. One line + * per touch, no more — a disclosure rail that spams is a rail people learn to + * ignore. + */ + +#if defined(__APPLE__) + +#import +#import +#import +#import +#import +#import + +#include +#include +#include +#include +#include +#include + +#include "el_runtime.h" + +/* ── Disclosure ────────────────────────────────────────────────────────────── + * One flushed line per real device touch, on stderr. Flushed rather than + * buffered so the line reaches the terminal BEFORE the mic light comes on + * rather than whenever the buffer happens to drain. */ +static void el_cap_disclose(const char* what) { + fprintf(stderr, " [peripheral] %s\n", what); + fflush(stderr); +} + +/* ── Permission ────────────────────────────────────────────────────────────── + * authorizationStatus is a pure read of the TCC database: it never prompts and + * never blocks, which is what lets mic_available()/camera_available() answer + * honestly without the question itself becoming an event. requestAccess DOES + * prompt, so it lives behind its own entry point and nothing calls it + * implicitly. */ + +static int el_cap_authorized(AVMediaType media) { + @try { + return [AVCaptureDevice authorizationStatusForMediaType:media] + == AVAuthorizationStatusAuthorized ? 1 : 0; + } @catch (NSException* e) { + (void)e; + return 0; + } +} + +static int el_cap_device_present(AVMediaType media) { + @try { + return [AVCaptureDevice defaultDeviceWithMediaType:media] != nil ? 1 : 0; + } @catch (NSException* e) { + (void)e; + return 0; + } +} + +/* Prompt once and wait, bounded. 30 seconds is the same budget periph.swift + * used: long enough for a human to notice a dialog and decide, short enough + * that an unattended run fails rather than parks forever. A timeout is reported + * as "not granted", which is the safe reading — we genuinely do not know that + * it was. */ +static int el_cap_request(AVMediaType media) { + __block int granted = 0; + dispatch_semaphore_t sem = dispatch_semaphore_create(0); + @try { + [AVCaptureDevice requestAccessForMediaType:media + completionHandler:^(BOOL ok) { + granted = ok ? 1 : 0; + dispatch_semaphore_signal(sem); + }]; + } @catch (NSException* e) { + (void)e; + return 0; + } + if (dispatch_semaphore_wait(sem, + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(30 * NSEC_PER_SEC))) != 0) { + return 0; /* timed out — treat as refused */ + } + return granted; +} + +/* ════════════════════════════════════════════════════════════════════════════ + * MICROPHONE — one-shot capture + * ══════════════════════════════════════════════════════════════════════════ */ + +/* The sink the tap block writes into. An object rather than a static so two + * captures can never share state, and so ARC keeps it alive for exactly as long + * as the block that captured it. The lock is real, not decorative: the tap runs + * on an AVAudioEngine-internal thread and the waiter runs on the caller's. */ +@interface ElCapMicSink : NSObject +@property (nonatomic, strong) NSMutableData* pcm; +@property (nonatomic, strong) NSLock* lock; +@end + +@implementation ElCapMicSink +- (instancetype)init { + self = [super init]; + if (self) { + _pcm = [NSMutableData data]; + _lock = [[NSLock alloc] init]; + } + return self; +} +@end + +/* Capture `seconds` of mono 16-bit PCM at `sample_rate`. + * + * The hardware format is NOT assumed. A built-in mic will typically hand back + * float32 at 44.1 or 48 kHz, an aggregate device may be 8 channels at 96 kHz, + * and a caller asking for 16 kHz mono (which is what the formant path wants) + * gets 16 kHz mono either way. AVAudioConverter does the rate conversion and + * the downmix; doing it by hand would mean writing a resampler in the one file + * that is supposed to contain no arithmetic. + * + * The converter is built ONCE, outside the tap, because a sample-rate converter + * carries filter state across buffers — rebuilding it per callback would put a + * discontinuity at every buffer boundary, which is audible and which would then + * show up in El's spectral descriptors as energy that was never in the room. */ +el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate) { + el_val_t empty = el_list_empty(); + + if (!el_cap_authorized(AVMediaTypeAudio)) return empty; + + int64_t secs = (int64_t)seconds; + int64_t sr = (int64_t)sample_rate; + if (secs <= 0 || sr <= 0) return empty; + /* Bound the ask. A caller that asks for a year of audio has made a mistake, + * and honouring it would mean an unkillable capture and an OOM. */ + if (secs > 300) secs = 300; + if (sr > 384000) sr = 384000; + + __block AVAudioEngine* engine = nil; + AVAudioInputNode* input = nil; + AVAudioFormat* hwFmt = nil; + int tapped = 0; + + @try { + engine = [[AVAudioEngine alloc] init]; + input = [engine inputNode]; + hwFmt = [input inputFormatForBus:0]; + } @catch (NSException* e) { + (void)e; + return empty; + } + if (!input || !hwFmt || hwFmt.sampleRate <= 0 || hwFmt.channelCount == 0) { + return empty; + } + + /* Preferred target: mono int16 at the requested rate. If the converter + * refuses that pairing (some exotic input layouts will not downmix), fall + * back to keeping the hardware's channel count and taking channel 0 on the + * way out — the organ is mono by design and inventing a downmix here would + * be an opinion about content this layer is not entitled to have. */ + AVAudioFormat* outFmt = + [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 + sampleRate:(double)sr + channels:1 + interleaved:YES]; + AVAudioConverter* conv = outFmt ? [[AVAudioConverter alloc] initFromFormat:hwFmt + toFormat:outFmt] : nil; + AVAudioChannelCount outCh = 1; + if (!conv) { + outFmt = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 + sampleRate:(double)sr + channels:hwFmt.channelCount + interleaved:YES]; + conv = outFmt ? [[AVAudioConverter alloc] initFromFormat:hwFmt toFormat:outFmt] : nil; + outCh = hwFmt.channelCount; + } + if (!conv || !outFmt) return empty; + + ElCapMicSink* sink = [[ElCapMicSink alloc] init]; + const int64_t want = secs * sr; /* frames we are waiting for */ + + { + char msg[192]; + snprintf(msg, sizeof(msg), + "MIC: opening the microphone for %llds -> %lld Hz mono PCM " + "(local, never egresses).", (long long)secs, (long long)sr); + el_cap_disclose(msg); + } + + const double ratio = (double)sr / hwFmt.sampleRate; + + @try { + [input installTapOnBus:0 + bufferSize:4096 + format:hwFmt + block:^(AVAudioPCMBuffer* _Nonnull buf, AVAudioTime* _Nonnull when) { + (void)when; + if (!buf || buf.frameLength == 0) return; + + AVAudioFrameCount cap = + (AVAudioFrameCount)((double)buf.frameLength * ratio) + 1024; + AVAudioPCMBuffer* out = + [[AVAudioPCMBuffer alloc] initWithPCMFormat:outFmt frameCapacity:cap]; + if (!out) return; + + __block BOOL fed = NO; + AVAudioConverterInputBlock feed = + ^AVAudioBuffer* _Nullable (AVAudioPacketCount need, + AVAudioConverterInputStatus* _Nonnull status) { + (void)need; + if (fed) { *status = AVAudioConverterInputStatus_NoDataNow; return nil; } + fed = YES; + *status = AVAudioConverterInputStatus_HaveData; + return buf; + }; + + NSError* err = nil; + AVAudioConverterOutputStatus st = + [conv convertToBuffer:out error:&err withInputFromBlock:feed]; + if (st == AVAudioConverterOutputStatus_Error || out.frameLength == 0) return; + + const int16_t* src = out.int16ChannelData ? out.int16ChannelData[0] : NULL; + if (!src) return; + + NSUInteger n = (NSUInteger)out.frameLength; + [sink.lock lock]; + if (outCh == 1) { + [sink.pcm appendBytes:src length:n * sizeof(int16_t)]; + } else { + /* Interleaved: stride to channel 0. */ + for (NSUInteger i = 0; i < n; i++) { + int16_t v = src[i * outCh]; + [sink.pcm appendBytes:&v length:sizeof(int16_t)]; + } + } + [sink.lock unlock]; + }]; + tapped = 1; + + [engine prepare]; + NSError* startErr = nil; + if (![engine startAndReturnError:&startErr]) { + [input removeTapOnBus:0]; + return empty; + } + } @catch (NSException* e) { + (void)e; + @try { if (tapped) [input removeTapOnBus:0]; } @catch (NSException* e2) { (void)e2; } + @try { [engine stop]; } @catch (NSException* e2) { (void)e2; } + return empty; + } + + /* Wait for `want` frames, bounded by the material's own duration plus a + * margin. A device that stops producing must not become a hang. */ + const int64_t deadline_us = (secs + 5) * 1000000; + int64_t waited_us = 0; + const int64_t tick_us = 5000; + for (;;) { + [sink.lock lock]; + int64_t have = (int64_t)([sink.pcm length] / sizeof(int16_t)); + [sink.lock unlock]; + if (have >= want || waited_us >= deadline_us) break; + usleep((useconds_t)tick_us); + waited_us += tick_us; + } + + @try { [input removeTapOnBus:0]; } @catch (NSException* e) { (void)e; } + @try { [engine stop]; } @catch (NSException* e) { (void)e; } + engine = nil; + + /* Hand up exactly what was asked for, or everything we got if the device + * came up short. Never padded: silence we invented is indistinguishable + * from silence we heard, and El has no way to tell them apart afterwards. */ + [sink.lock lock]; + int64_t have = (int64_t)([sink.pcm length] / sizeof(int16_t)); + int64_t n = have < want ? have : want; + const int16_t* pcm = (const int16_t*)[sink.pcm bytes]; + el_val_t list = empty; + for (int64_t i = 0; i < n; i++) { + list = el_list_append(list, (el_val_t)(int64_t)pcm[i]); + } + [sink.lock unlock]; + + return list; +} + +el_val_t mic_available(void) { + if (!el_cap_authorized(AVMediaTypeAudio)) return (el_val_t)0; + return (el_val_t)(el_cap_device_present(AVMediaTypeAudio) ? 1 : 0); +} + +el_val_t mic_request_access(void) { + return (el_val_t)(el_cap_request(AVMediaTypeAudio) ? 1 : 0); +} + +/* ════════════════════════════════════════════════════════════════════════════ + * MICROPHONE — live monitor (the full-duplex ear) + * + * converse needs to keep listening WHILE it speaks, which means the mic is open + * at the same time as the speaker. In a real room that is a feedback path: + * without cancellation Neuron hears its own voice, decides someone is talking, + * and barges in on itself. setVoiceProcessingEnabled: hands the input node to + * the OS voice-processing unit, which subtracts the known output signal from + * the input — the single thing that makes barge-in work outside a headset. + * + * It is not always available (some aggregate and virtual devices refuse it), so + * failure to enable it is reported as a DISTINCT return value (2) rather than + * folded into success. The caller needs to know, because the correct response + * is to raise the VAD floor, and a caller that thinks AEC is on will set that + * floor far too low. + * + * The tap keeps only a running short-window RMS in a static behind a mutex. + * Deliberately not a queue of samples: this path is polled at ~50 Hz by a loop + * that only ever asks "is someone talking", and buffering audio nobody reads + * would be an unbounded allocation in the middle of a conversation. + * ══════════════════════════════════════════════════════════════════════════ */ + +static AVAudioEngine* g_mon_engine = nil; +static int g_mon_running = 0; +static int g_mon_code = 0; /* what the successful start reported */ +static double g_mon_rms = 0.0; +static pthread_mutex_t g_mon_lock = PTHREAD_MUTEX_INITIALIZER; + +el_val_t mic_monitor_start(void) { + /* Idempotent, and it re-reports the ORIGINAL code rather than a bare 1: a + * caller that starts twice must not be told AEC is on when the first start + * already discovered it was not. */ + if (g_mon_running) return (el_val_t)g_mon_code; + if (!el_cap_authorized(AVMediaTypeAudio)) return (el_val_t)0; + + AVAudioEngine* engine = nil; + AVAudioInputNode* input = nil; + AVAudioFormat* fmt = nil; + int aec = 0; + int tapped = 0; + + @try { + engine = [[AVAudioEngine alloc] init]; + input = [engine inputNode]; + } @catch (NSException* e) { + (void)e; + return (el_val_t)0; + } + if (!input) return (el_val_t)0; + + /* Enable AEC BEFORE reading the format: the voice-processing unit imposes + * its own input format, and a tap installed with the pre-VP format would be + * rejected at start. */ + @try { + NSError* vpErr = nil; + if ([input respondsToSelector:@selector(setVoiceProcessingEnabled:error:)]) { + aec = [input setVoiceProcessingEnabled:YES error:&vpErr] ? 1 : 0; + } + } @catch (NSException* e) { + (void)e; + aec = 0; + } + + @try { + fmt = [input inputFormatForBus:0]; + } @catch (NSException* e) { + (void)e; + return (el_val_t)0; + } + if (!fmt || fmt.sampleRate <= 0 || fmt.channelCount == 0) return (el_val_t)0; + + el_cap_disclose(aec + ? "MIC: opening the microphone for live monitoring, echo-cancelled (local)." + : "MIC: opening the microphone for live monitoring, NO echo cancellation (local)."); + + @try { + [input installTapOnBus:0 + bufferSize:1024 + format:fmt + block:^(AVAudioPCMBuffer* _Nonnull buf, AVAudioTime* _Nonnull when) { + (void)when; + if (!buf) return; + AVAudioFrameCount n = buf.frameLength; + if (n == 0) return; + + double sum = 0.0; + /* Whatever the VP unit hands back — float32 is the norm, int16 and + * int32 are possible on odd hardware — normalise to -1..1 so the + * Float El sees means the same thing on every device. */ + if (buf.floatChannelData) { + const float* ch = buf.floatChannelData[0]; + for (AVAudioFrameCount i = 0; i < n; i++) sum += (double)ch[i] * (double)ch[i]; + } else if (buf.int16ChannelData) { + const int16_t* ch = buf.int16ChannelData[0]; + for (AVAudioFrameCount i = 0; i < n; i++) { + double v = (double)ch[i] / 32768.0; + sum += v * v; + } + } else if (buf.int32ChannelData) { + const int32_t* ch = buf.int32ChannelData[0]; + for (AVAudioFrameCount i = 0; i < n; i++) { + double v = (double)ch[i] / 2147483648.0; + sum += v * v; + } + } else { + return; + } + + double rms = sqrt(sum / (double)n); + if (rms < 0.0) rms = 0.0; + if (rms > 1.0) rms = 1.0; + + pthread_mutex_lock(&g_mon_lock); + g_mon_rms = rms; + pthread_mutex_unlock(&g_mon_lock); + }]; + tapped = 1; + + [engine prepare]; + NSError* startErr = nil; + if (![engine startAndReturnError:&startErr]) { + [input removeTapOnBus:0]; + return (el_val_t)0; + } + } @catch (NSException* e) { + (void)e; + @try { if (tapped) [input removeTapOnBus:0]; } @catch (NSException* e2) { (void)e2; } + @try { [engine stop]; } @catch (NSException* e2) { (void)e2; } + return (el_val_t)0; + } + + pthread_mutex_lock(&g_mon_lock); + g_mon_rms = 0.0; + pthread_mutex_unlock(&g_mon_lock); + + g_mon_engine = engine; + g_mon_running = 1; + g_mon_code = aec ? 1 : 2; + return (el_val_t)g_mon_code; +} + +/* Float in 0..1. Reads the last window the tap computed; never blocks on the + * audio thread beyond the mutex, because this is polled inside a turn-taking + * loop where a stall IS a missed barge-in. */ +el_val_t mic_monitor_rms(void) { + double rms = 0.0; + pthread_mutex_lock(&g_mon_lock); + rms = g_mon_rms; + pthread_mutex_unlock(&g_mon_lock); + return el_from_float(rms); +} + +el_val_t mic_monitor_stop(void) { + AVAudioEngine* engine = g_mon_engine; + g_mon_engine = nil; + g_mon_running = 0; + g_mon_code = 0; + + if (engine) { + @try { [[engine inputNode] removeTapOnBus:0]; } @catch (NSException* e) { (void)e; } + @try { [engine stop]; } @catch (NSException* e) { (void)e; } + } + pthread_mutex_lock(&g_mon_lock); + g_mon_rms = 0.0; + pthread_mutex_unlock(&g_mon_lock); + return (el_val_t)1; +} + +/* ════════════════════════════════════════════════════════════════════════════ + * CAMERA + * ══════════════════════════════════════════════════════════════════════════ */ + +static void el_cap_free_bitmap(void* info, const void* data, size_t size) { + (void)info; (void)size; + free((void*)data); +} + +/* CVPixelBuffer -> CGImage, own-core, no CoreImage. + * + * The output is pinned to 32BGRA at the AVCaptureVideoDataOutput (see below) + * precisely so this conversion can be a memcpy and a CGImageCreate. The + * alternative — accepting the camera's native 2vuy/420v and colour-converting + * here — would mean either pulling in CoreImage or writing a YUV->RGB matrix in + * the file that is supposed to contain no arithmetic. Asking the capture output + * for BGRA moves that work into AVFoundation, where it is already written and + * already hardware-accelerated. + * + * The rows are copied out rather than aliased because the CVPixelBuffer is + * recycled by the capture session the moment the delegate returns; a CGImage + * pointing at it would be pointing at the NEXT frame by the time anyone looked. */ +static CGImageRef el_cap_cgimage_from_pixelbuffer(CVPixelBufferRef pb) { + if (!pb) return NULL; + if (CVPixelBufferGetPixelFormatType(pb) != kCVPixelFormatType_32BGRA) return NULL; + if (CVPixelBufferLockBaseAddress(pb, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) return NULL; + + size_t w = CVPixelBufferGetWidth(pb); + size_t h = CVPixelBufferGetHeight(pb); + size_t src_bpr = CVPixelBufferGetBytesPerRow(pb); + const uint8_t* base = (const uint8_t*)CVPixelBufferGetBaseAddress(pb); + + CGImageRef img = NULL; + if (base && w > 0 && h > 0 && src_bpr >= w * 4) { + size_t dst_bpr = w * 4; + uint8_t* copy = (uint8_t*)malloc(dst_bpr * h); + if (copy) { + for (size_t y = 0; y < h; y++) { + memcpy(copy + y * dst_bpr, base + y * src_bpr, dst_bpr); + } + CGDataProviderRef dp = + CGDataProviderCreateWithData(NULL, copy, dst_bpr * h, el_cap_free_bitmap); + if (dp) { + CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB(); + if (cs) { + img = CGImageCreate(w, h, 8, 32, dst_bpr, cs, + (CGBitmapInfo)(kCGBitmapByteOrder32Little | + kCGImageAlphaNoneSkipFirst), + dp, NULL, false, kCGRenderingIntentDefault); + CGColorSpaceRelease(cs); + } + CGDataProviderRelease(dp); /* provider owns `copy` from here */ + } else { + free(copy); + } + } + } + + CVPixelBufferUnlockBaseAddress(pb, kCVPixelBufferLock_ReadOnly); + return img; +} + +/* The frame delegate. AVCaptureVideoDataOutput is used rather than + * AVCapturePhotoOutput for the same reason periph.swift used it: the photo path + * wants KVO and a session owned by an app object, and this runs in a plain CLI + * process with no run loop it can assume. A data output just calls back. + * + * The first frames are dropped on purpose. A camera that has just been powered + * on is still converging exposure and white balance, and the first frame is + * reliably darker and greener than the room. El's scene-geometry descriptors + * are brightness and mean-colour statistics, so handing up an unsettled frame + * would not produce a slightly worse answer, it would produce a confidently + * wrong one. */ +@interface ElCapFrameGrabber : NSObject { + CGImageRef _img; + int _seen; + dispatch_semaphore_t _sem; +} +- (dispatch_semaphore_t)sem; +- (CGImageRef)takeImage; /* transfers ownership to the caller */ +@end + +@implementation ElCapFrameGrabber + +- (instancetype)init { + self = [super init]; + if (self) { + _img = NULL; + _seen = 0; + _sem = dispatch_semaphore_create(0); + } + return self; +} + +- (dispatch_semaphore_t)sem { return _sem; } + +- (CGImageRef)takeImage { + CGImageRef out = _img; + _img = NULL; + return out; +} + +- (void)dealloc { + if (_img) { CGImageRelease(_img); _img = NULL; } +} + +/* Runs on the serial delegate queue, so no lock is needed among callbacks; the + * waiter only reads _img after the semaphore has been signalled AND the session + * has been stopped, which orders it after the last callback. */ +- (void)captureOutput:(AVCaptureOutput*)output +didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection*)connection { + (void)output; (void)connection; + _seen++; + if (_img != NULL || _seen < 5) return; /* let exposure settle */ + + CVImageBufferRef pb = CMSampleBufferGetImageBuffer(sampleBuffer); + if (!pb) return; + CGImageRef img = el_cap_cgimage_from_pixelbuffer(pb); + if (!img) return; + _img = img; + dispatch_semaphore_signal(_sem); +} + +@end + +/* Bring the camera up, take exactly one settled frame, put it back down. + * Returns a +1 CGImageRef the caller releases, or NULL. Bounded at 10s: a + * camera held by another process, or one whose TCC grant was revoked between + * the check and the open, must fail rather than park. */ +static CGImageRef el_cap_grab_frame(void) { + AVCaptureSession* session = nil; + AVCaptureVideoDataOutput* output = nil; + ElCapFrameGrabber* grabber = nil; + CGImageRef img = NULL; + + @try { + AVCaptureDevice* dev = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; + if (!dev) return NULL; + + NSError* err = nil; + AVCaptureDeviceInput* in = [AVCaptureDeviceInput deviceInputWithDevice:dev error:&err]; + if (!in) return NULL; + + session = [[AVCaptureSession alloc] init]; + session.sessionPreset = AVCaptureSessionPresetPhoto; + if (![session canAddInput:in]) return NULL; + [session addInput:in]; + + output = [[AVCaptureVideoDataOutput alloc] init]; + output.alwaysDiscardsLateVideoFrames = YES; + /* Pin the pixel format so the CGImage conversion above stays a memcpy. + * Every macOS capture device advertises 32BGRA. */ + output.videoSettings = @{ (id)kCVPixelBufferPixelFormatTypeKey : + @(kCVPixelFormatType_32BGRA) }; + + grabber = [[ElCapFrameGrabber alloc] init]; + dispatch_queue_t q = dispatch_queue_create("el.capture.camera", DISPATCH_QUEUE_SERIAL); + [output setSampleBufferDelegate:grabber queue:q]; + + if (![session canAddOutput:output]) return NULL; + [session addOutput:output]; + + el_cap_disclose("CAMERA: opening the camera for one frame (local, never egresses)."); + [session startRunning]; + } @catch (NSException* e) { + (void)e; + @try { [session stopRunning]; } @catch (NSException* e2) { (void)e2; } + return NULL; + } + + long timed_out = dispatch_semaphore_wait([grabber sem], + dispatch_time(DISPATCH_TIME_NOW, (int64_t)(10 * NSEC_PER_SEC))); + + /* Stop first, then detach the delegate, then read. In that order the last + * callback has already returned by the time anyone touches the image. */ + @try { [session stopRunning]; } @catch (NSException* e) { (void)e; } + @try { [output setSampleBufferDelegate:nil queue:NULL]; } @catch (NSException* e) { (void)e; } + + if (timed_out == 0) img = [grabber takeImage]; + return img; +} + +el_val_t camera_available(void) { + if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0; + return (el_val_t)(el_cap_device_present(AVMediaTypeVideo) ? 1 : 0); +} + +el_val_t camera_request_access(void) { + return (el_val_t)(el_cap_request(AVMediaTypeVideo) ? 1 : 0); +} + +/* Longest edge of the grid handed to El. 64 is not a resolution, it is a budget: + * a 1920x1080 frame is 6.2 MILLION packed RGB ints, and building that as an El + * list would cost more time and memory than everything El then does with it. + * The descriptors El computes over this — mean colour, brightness, a 3x3 + * luminance grid — are region statistics, and region statistics do not get + * meaningfully better above a 64-wide grid. The TRUE frame dimensions are + * reported separately so nothing downstream has to guess what was thrown away. */ +#define EL_CAP_GRID_MAX 64 + +/* One frame as Map{width, height, grid_w, grid_h, pixels:[Int]}. + * + * "pixels" is packed R,G,B with NO alpha — three ints per grid cell, row-major + * from the TOP-LEFT. (CGBitmapContext lays its buffer out top row first and + * CGContextDrawImage does the flip, so row 0 here is the top of the frame, the + * same convention periph.swift's grid indexing assumed.) Alpha is dropped + * because a camera frame has none worth carrying and it would inflate the list + * by a third to say "opaque" six thousand times. */ +el_val_t camera_capture_rgb(void) { + if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0; + + CGImageRef img = el_cap_grab_frame(); + if (!img) return (el_val_t)0; + + size_t w = CGImageGetWidth(img); + size_t h = CGImageGetHeight(img); + if (w == 0 || h == 0) { CGImageRelease(img); return (el_val_t)0; } + + /* Preserve aspect ratio, longest edge capped. */ + size_t gw = w, gh = h; + size_t longest = w > h ? w : h; + if (longest > EL_CAP_GRID_MAX) { + double s = (double)EL_CAP_GRID_MAX / (double)longest; + gw = (size_t)((double)w * s + 0.5); + gh = (size_t)((double)h * s + 0.5); + if (gw == 0) gw = 1; + if (gh == 0) gh = 1; + } + + size_t bpr = gw * 4; + uint8_t* buf = (uint8_t*)calloc(1, bpr * gh); + if (!buf) { CGImageRelease(img); return (el_val_t)0; } + + CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB(); + CGContextRef ctx = cs ? CGBitmapContextCreate(buf, gw, gh, 8, bpr, cs, + (CGBitmapInfo)kCGImageAlphaPremultipliedLast) + : NULL; + if (cs) CGColorSpaceRelease(cs); + if (!ctx) { free(buf); CGImageRelease(img); return (el_val_t)0; } + + /* Nearest-neighbour. This is a decimation for statistics, not a thumbnail + * for a human to look at; smoothing would only cost time and blur the very + * region boundaries the grid exists to measure. */ + CGContextSetInterpolationQuality(ctx, kCGInterpolationNone); + CGContextDrawImage(ctx, CGRectMake(0, 0, (CGFloat)gw, (CGFloat)gh), img); + CGContextRelease(ctx); + CGImageRelease(img); + + el_val_t pixels = el_list_empty(); + for (size_t y = 0; y < gh; y++) { + const uint8_t* row = buf + y * bpr; + for (size_t x = 0; x < gw; x++) { + const uint8_t* p = row + x * 4; /* RGBA8, premultiplied-last */ + pixels = el_list_append(pixels, (el_val_t)(int64_t)p[0]); + pixels = el_list_append(pixels, (el_val_t)(int64_t)p[1]); + pixels = el_list_append(pixels, (el_val_t)(int64_t)p[2]); + } + } + free(buf); + + el_val_t m = el_map_new((el_val_t)0); + if (!m) return (el_val_t)0; + m = el_map_set(m, EL_STR("width"), (el_val_t)(int64_t)w); + m = el_map_set(m, EL_STR("height"), (el_val_t)(int64_t)h); + m = el_map_set(m, EL_STR("grid_w"), (el_val_t)(int64_t)gw); + m = el_map_set(m, EL_STR("grid_h"), (el_val_t)(int64_t)gh); + m = el_map_set(m, EL_STR("pixels"), pixels); + return m; +} + +/* One frame to disk as JPEG, at FULL resolution — the opposite budget from + * camera_capture_rgb, and for the opposite reason. A file is not being walked + * element-by-element by an interpreter; it costs one ImageIO call and it is the + * artefact a human or a later pass will actually look at. The encoder is + * ImageIO's because a JPEG encoder is a codec, and re-implementing one in El + * would be a large amount of arithmetic that buys nothing: the point of keeping + * work in El is the reasoning, not the entropy coding. */ +el_val_t camera_capture_jpeg(el_val_t path) { + const char* p = EL_CSTR(path); + if (!p || !*p) return (el_val_t)0; + if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0; + + CGImageRef img = el_cap_grab_frame(); + if (!img) return (el_val_t)0; + + int ok = 0; + @autoreleasepool { + NSString* ns = [NSString stringWithUTF8String:p]; + NSURL* url = ns ? [NSURL fileURLWithPath:ns] : nil; + if (url) { + CGImageDestinationRef dst = + CGImageDestinationCreateWithURL((__bridge CFURLRef)url, CFSTR("public.jpeg"), 1, NULL); + if (dst) { + CGImageDestinationAddImage(dst, img, NULL); + ok = CGImageDestinationFinalize(dst) ? 1 : 0; + CFRelease(dst); + } + } + } + CGImageRelease(img); + return (el_val_t)ok; +} + +#endif /* __APPLE__ */ diff --git a/lang/runtime/el_peripheral_null.c b/lang/runtime/el_peripheral_null.c new file mode 100644 index 0000000..6e333c8 --- /dev/null +++ b/lang/runtime/el_peripheral_null.c @@ -0,0 +1,89 @@ +/* el_peripheral_null.c — the no-device build of El's I/O organ. + * + * Every entry point declared in el_runtime.h's "Peripheral" block, implemented + * as an honest refusal. This is what a platform without an El audio/capture + * realizer links instead of el_audio_darwin.m + el_capture_darwin.m, so an El + * program that speaks or listens still COMPILES AND LINKS everywhere. + * + * The distinction that matters: these do not pretend. speaker_available() and + * mic_available() return 0, and every operation returns its failure sentinel. + * A program asking "can I speak here?" gets a truthful no, rather than a + * silence it would have to infer something from. Silent success is the failure + * mode this whole change exists to eliminate — El spent this entire codebase's + * history writing WAV files full of zeros and reporting ok=true, and nobody + * caught it because nothing ever said "there is no sound here". + * + * Compiled INSTEAD OF the Darwin realizers, never alongside them — the symbols + * are the same by design, which is the point: the El side never branches on + * platform, it branches on speaker_available(). + */ + +#include "el_runtime.h" + +#if !defined(__APPLE__) + +/* ── Speaker ─────────────────────────────────────────────────────────────── */ + +el_val_t speaker_available(void) { return (el_val_t)0; } +el_val_t speaker_name(void) { return EL_STR("none"); } + +el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate) { + (void)samples; (void)sample_rate; + return (el_val_t)0; +} + +el_val_t speaker_play_wav(el_val_t path) { + (void)path; + return (el_val_t)0; +} + +el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate) { + (void)samples; (void)sample_rate; + return (el_val_t)0; +} + +el_val_t speaker_play_wav_async(el_val_t path) { + (void)path; + return (el_val_t)0; +} + +el_val_t speaker_pause(void) { return (el_val_t)0; } +el_val_t speaker_resume(void) { return (el_val_t)0; } +el_val_t speaker_playing(void) { return (el_val_t)0; } +el_val_t speaker_stop(void) { return (el_val_t)0; } +el_val_t speaker_played_frames(void) { return (el_val_t)0; } + +/* WAV geometry is pure parsing and would work fine here, but reporting a + * duration for audio this build cannot play would invite a caller to sequence + * around a silence. Refuse consistently with the rest of the file. */ +el_val_t wav_frames(el_val_t path) { (void)path; return (el_val_t)0; } +el_val_t wav_rate(el_val_t path) { (void)path; return (el_val_t)0; } + +/* ── Microphone ──────────────────────────────────────────────────────────── */ + +el_val_t mic_available(void) { return (el_val_t)0; } +el_val_t mic_request_access(void) { return (el_val_t)0; } + +/* Empty list, not 0: the contract says capture returns samples, and a caller + * iterating the result must find nothing rather than dereference a non-list. */ +el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate) { + (void)seconds; (void)sample_rate; + return el_list_empty(); +} + +el_val_t mic_monitor_start(void) { return (el_val_t)0; } +el_val_t mic_monitor_rms(void) { return el_from_float(0.0); } +el_val_t mic_monitor_stop(void) { return (el_val_t)0; } + +/* ── Camera ──────────────────────────────────────────────────────────────── */ + +el_val_t camera_available(void) { return (el_val_t)0; } +el_val_t camera_request_access(void) { return (el_val_t)0; } +el_val_t camera_capture_rgb(void) { return (el_val_t)0; } + +el_val_t camera_capture_jpeg(el_val_t path) { + (void)path; + return (el_val_t)0; +} + +#endif /* !__APPLE__ */ diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index a896549..73c04e0 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -40,9 +40,11 @@ #include #include #include +#include /* SIGPIPE disposition: a hung-up client must not kill us */ #include /* dlsym for http_set_handler fallback */ #include #include +#include /* flock — process-identity singleton (program block) */ #include #include #include @@ -105,23 +107,14 @@ static void el_arena_track(char* p) { _tl_arena.ptrs[_tl_arena.count++] = p; } -/* Called by http_worker before dispatching the El handler. */ -void el_request_start(void) { - _tl_arena.count = 0; - _tl_arena_active = 1; - _tl_fs_read_len = 0; /* never let a previous request's file length */ - _tl_fs_read_buf = NULL; /* leak into this response's byte accounting */ -} - -/* Called by http_worker after the El handler returns and the response is sent. - * Frees every intermediate string allocated during the request. */ -void el_request_end(void) { - _tl_arena_active = 0; - for (size_t i = 0; i < _tl_arena.count; i++) { - free(_tl_arena.ptrs[i]); - } - _tl_arena.count = 0; -} +/* el_request_start / el_request_end moved to el_seed.c (see its comment at the + * definition: "formerly defined in el_runtime.c. Now self-contained in + * el_seed.c, delegating to the seed arena."). The copies here were left behind + * during that move and made el_seed.o + el_runtime.o fail to link together with + * duplicate symbols — which is exactly the link the real product build does. + * Declared (not defined) here: el_runtime.c's http_worker still calls them. */ +void el_request_start(void); +void el_request_end(void); /* ── Scoped arena for CLI use ─────────────────────────────────────────────── * * CLI programs never call el_request_start/end, so all strdup allocations are @@ -149,6 +142,45 @@ el_val_t el_arena_push(void) { return (el_val_t)(int64_t)_tl_arena.count; } +/* ── String-length cache ───────────────────────────────────────────────────── + * + * THE COMPILER'S QUADRATIC LIVED HERE. str_char_code and str_slice each called + * strlen() on every invocation. The lexer walks source one character at a time, + * so每 access rescanned the whole remaining input: O(n) per character over n + * characters = O(n^2). Measured on a geometric sweep of synthetic sources, + * wall-clock rose 3.0x, 3.0x, 4.0x, 4.14x per doubling — converging on 4x, a + * textbook quadratic — and a stack sample put 779 of 779 samples inside lex(), + * every one bottoming out in _platform_strlen. + * + * The fix is to remember the length instead of recomputing it. The subtlety is + * INVALIDATION: El strings are arena-allocated, so a freed pointer can be + * reused for a different string at the same address. A naive pointer-keyed + * cache would then hand back a stale length and read past the end of the new + * string — trading a performance bug for a memory-safety one. + * + * So entries carry a generation. Anything that frees or mutates runtime strings + * bumps the generation, and a cache hit requires both the pointer AND the + * generation to match. Stale entries can never be believed; they simply miss + * and recompute. + * ──────────────────────────────────────────────────────────────────────────── */ +#define EL_SLC_SLOTS 8 +typedef struct { const char* ptr; size_t len; uint64_t gen; } ElStrLenEnt; +static ElStrLenEnt _el_slc[EL_SLC_SLOTS]; +static uint64_t _el_str_gen = 1; + +/* Called by every path that frees or mutates a runtime string. */ +void el_str_cache_flush(void) { _el_str_gen++; } + +static size_t el_strlen_cached(const char* s) { + if (!s) return 0; + size_t slot = ((uintptr_t)s >> 4) & (EL_SLC_SLOTS - 1); + ElStrLenEnt* e = &_el_slc[slot]; + if (e->ptr == s && e->gen == _el_str_gen) return e->len; + size_t n = strlen(s); + e->ptr = s; e->len = n; e->gen = _el_str_gen; + return n; +} + el_val_t el_arena_pop(el_val_t mark) { size_t save = (size_t)(int64_t)mark; if (save > _tl_arena.count) save = 0; @@ -161,24 +193,59 @@ el_val_t el_arena_pop(el_val_t mark) { _tl_arena.count = save; if (_tl_arena_scope_depth > 0) _tl_arena_scope_depth--; if (save == 0) _tl_arena_active = 0; + el_str_cache_flush(); /* freed pointers may be reused — see cache note */ return 0; } +/* ── Allocation accounting ─────────────────────────────────────────────────── + * + * Every string allocation in the runtime funnels through the four functions + * below, so counting here counts everything the language does. + * + * WHY THIS EXISTS: a growth-curve gate needs a signal that is DETERMINISTIC. + * Wall-clock needs statistics, warmup, and a quiet machine; it is noisy on + * shared CI and unusable as a hard build gate. Allocation COUNT has none of + * those problems — the same input allocates the same number of times on every + * machine, every run. Fit allocations against input size and a complexity + * regression becomes a build failure with zero flake. + * + * This is not hypothetical. elc's known defect is quadratic ALLOCATION VOLUME. + * The old shipped binary paid it in RSS (27 GB, OOM); the rebuilt one pays the + * same quadratic in malloc/free churn (42s on a 1.4 MB input). The allocation + * count was the invariant across both — RSS and wall-clock were just the two + * ways it surfaced. An `expect allocs O(n)` assertion on the compile path + * would have failed the build the day it was introduced. + * + * Peak RSS is exported too but is explicitly NOT the gating signal: it is + * perturbed by allocator behaviour, page cache, and the OS. Gate on counts, + * report RSS as context. + * + * Counters are plain unsigned longs, incremented on the allocating thread with + * no synchronisation: this is measurement, and a lock here would change the + * thing being measured. Under threads the count is approximate; for the + * single-threaded compile path it is exact. + * ──────────────────────────────────────────────────────────────────────────── */ +static unsigned long _el_alloc_count = 0; +static unsigned long _el_alloc_bytes = 0; + /* Persistent allocation — bypasses the arena (state_set, engram internals). */ static char* el_strdup_persist(const char* s) { - if (!s) return strdup(""); + if (!s) { _el_alloc_count++; _el_alloc_bytes += 1; return strdup(""); } + _el_alloc_count++; _el_alloc_bytes += strlen(s) + 1; return strdup(s); } static char* el_strbuf_persist(size_t n) { char* p = malloc(n + 1); if (!p) { fputs("el_runtime: out of memory\n", stderr); exit(1); } p[0] = '\0'; + _el_alloc_count++; _el_alloc_bytes += n + 1; return p; } static char* el_strdup(const char* s) { - if (!s) { char* p = strdup(""); el_arena_track(p); return p; } + if (!s) { char* p = strdup(""); _el_alloc_count++; _el_alloc_bytes += 1; el_arena_track(p); return p; } char* p = strdup(s); + _el_alloc_count++; _el_alloc_bytes += strlen(s) + 1; el_arena_track(p); return p; } @@ -187,6 +254,7 @@ static char* el_strbuf(size_t n) { char* p = malloc(n + 1); if (!p) { fputs("el_runtime: out of memory\n", stderr); exit(1); } p[0] = '\0'; + _el_alloc_count++; _el_alloc_bytes += n + 1; el_arena_track(p); return p; } @@ -283,7 +351,7 @@ el_val_t str_to_int(el_val_t sv) { el_val_t str_slice(el_val_t sv, el_val_t start, el_val_t end) { const char* s = EL_CSTR(sv); if (!s) return el_wrap_str(el_strdup("")); - int64_t len = (int64_t)strlen(s); + int64_t len = (int64_t)el_strlen_cached(s); if (start < 0) start = 0; if (end > len) end = len; if (start >= end) return el_wrap_str(el_strdup("")); @@ -410,12 +478,14 @@ typedef struct { static ElList* list_alloc(int64_t cap) { if (cap < 4) cap = 4; ElList* lst = malloc(sizeof(ElList)); + _el_alloc_count++; _el_alloc_bytes += sizeof(ElList); if (!lst) { fputs("el_runtime: out of memory\n", stderr); exit(1); } lst->hdr.magic = EL_MAGIC_LIST; lst->hdr.refcount = 1; lst->length = 0; lst->capacity = cap; lst->elems = malloc((size_t)cap * sizeof(el_val_t)); + _el_alloc_count++; _el_alloc_bytes += (size_t)cap * sizeof(el_val_t); if (!lst->elems) { fputs("el_runtime: out of memory\n", stderr); exit(1); } return lst; } @@ -465,6 +535,7 @@ el_val_t el_list_append(el_val_t listv, el_val_t elem) { if (old->length >= old->capacity) { int64_t new_cap = old->capacity > 0 ? old->capacity * 2 : 4; el_val_t* grown = realloc(old->elems, (size_t)new_cap * sizeof(el_val_t)); + _el_alloc_count++; _el_alloc_bytes += (size_t)new_cap * sizeof(el_val_t); if (!grown) { fputs("el_runtime: out of memory\n", stderr); exit(1); } old->elems = grown; old->capacity = new_cap; @@ -477,12 +548,14 @@ el_val_t el_list_append(el_val_t listv, el_val_t elem) { int64_t new_cap = old->length + 1; if (new_cap < 4) new_cap = 4; ElList* fresh = malloc(sizeof(ElList)); + _el_alloc_count++; _el_alloc_bytes += sizeof(ElList); if (!fresh) { fputs("el_runtime: out of memory\n", stderr); exit(1); } fresh->hdr.magic = EL_MAGIC_LIST; fresh->hdr.refcount = 1; fresh->length = old->length + 1; fresh->capacity = new_cap; fresh->elems = malloc((size_t)new_cap * sizeof(el_val_t)); + _el_alloc_count++; _el_alloc_bytes += (size_t)new_cap * sizeof(el_val_t); if (!fresh->elems) { fputs("el_runtime: out of memory\n", stderr); exit(1); } if (old->length > 0) { memcpy(fresh->elems, old->elems, (size_t)old->length * sizeof(el_val_t)); @@ -504,12 +577,14 @@ el_val_t el_list_clone(el_val_t listv) { if (cap < old->length) cap = old->length; if (cap < 4) cap = 4; ElList* fresh = malloc(sizeof(ElList)); + _el_alloc_count++; _el_alloc_bytes += sizeof(ElList); if (!fresh) { fputs("el_runtime: out of memory\n", stderr); exit(1); } fresh->hdr.magic = EL_MAGIC_LIST; fresh->hdr.refcount = 1; fresh->length = old->length; fresh->capacity = cap; fresh->elems = malloc((size_t)cap * sizeof(el_val_t)); + _el_alloc_count++; _el_alloc_bytes += (size_t)cap * sizeof(el_val_t); if (!fresh->elems) { fputs("el_runtime: out of memory\n", stderr); exit(1); } if (old->length > 0) { memcpy(fresh->elems, old->elems, (size_t)old->length * sizeof(el_val_t)); @@ -530,6 +605,7 @@ typedef struct { static ElMap* map_alloc(int64_t cap) { if (cap < 4) cap = 4; ElMap* m = malloc(sizeof(ElMap)); + _el_alloc_count++; _el_alloc_bytes += sizeof(ElMap); if (!m) { fputs("el_runtime: out of memory\n", stderr); exit(1); } m->hdr.magic = EL_MAGIC_MAP; m->hdr.refcount = 1; @@ -605,6 +681,7 @@ el_val_t el_map_set(el_val_t mapv, el_val_t keyv, el_val_t value) { int64_t new_cap = m->count + 1; if (new_cap < 4) new_cap = 4; ElMap* fresh = malloc(sizeof(ElMap)); + _el_alloc_count++; _el_alloc_bytes += sizeof(ElMap); if (!fresh) { fputs("el_runtime: out of memory\n", stderr); exit(1); } fresh->hdr.magic = EL_MAGIC_MAP; fresh->hdr.refcount = 1; @@ -635,6 +712,20 @@ el_val_t el_map_set(el_val_t mapv, el_val_t keyv, el_val_t value) { * happen to look like aligned heap pointers are exceedingly unlikely to land * on a page whose first 4 bytes match either magic. */ +/* el_tagged — THE gate for "is this slot a heap object carrying this tag". + * + * el_val_t carries both integers and tagged heap pointers, so deciding which + * requires checking the value BEFORE dereferencing it. That check was a + * convention every author had to know rather than a gate they had to pass + * through, and the result is measurable: geom_of and mfld_of call + * looks_like_heap_obj and are correct; el_bin_lookup checked only a 4096 floor + * -- no alignment, no small-int, no negative -- and reads EIGHT BYTES BACKWARD + * from the pointer. sha256_hex(50000) therefore compiled clean and segfaulted. + * + * Exported, so the engram siblings stop re-deriving it. Anything that + * dereferences a slot without passing through here is the defect. */ +int el_tagged(el_val_t v, uint32_t magic); + static int looks_like_heap_obj(el_val_t v) { if (v == 0) return 0; int64_t s = (int64_t)v; @@ -645,6 +736,12 @@ static int looks_like_heap_obj(el_val_t v) { return 1; } +int el_tagged(el_val_t v, uint32_t magic) { + if (!looks_like_heap_obj(v)) return 0; + return *(const uint32_t*)(uintptr_t)v == magic; +} + + void el_retain(el_val_t v) { if (!looks_like_heap_obj(v)) return; ElHeader* h = (ElHeader*)(uintptr_t)v; @@ -1096,6 +1193,128 @@ void http_set_handler(el_val_t name) { pthread_mutex_unlock(&_http_handler_mu); } +/* ── Ambient consolidation: dreaming ──────────────────────────────────────── + * + * Dreaming is not sleep, and it is not scheduled. A brain has no cron job. + * The default mode network is ANTICORRELATED WITH TASK ENGAGEMENT: attention + * drops, it activates — hundreds of times a day, for seconds at a time. + * Daydreaming and sleep-dreaming are one process at different depths, and the + * depth is set by how much capacity is unclaimed, not by a time of day. + * + * WHY THIS EXISTS (2026-08-16). Consolidation had no owner, so it was + * implemented at every site that needed a piece of it — measured: soul's + * in-process awareness loop, three POST beats on the engram, a 600s ticker, + * two resident Python services, and three cron entries at 23:55 / 06:00 / + * 08:30. That last trio is a sleep cycle written as crontab. Seven systems + * dreaming into one graph with no owner for dreaming is what crashed soul on + * this date; the contention was the symptom of the missing owner. + * + * Every ticker is the diagnostic. A StartInterval, an Hour/Minute, a + * POST-to-beat — each marks a place where an intrinsic rhythm was replaced by + * an external clock, which is a supervisor invented for something that should + * be a property of the substrate. + * + * The engagement signal already existed and needed no invention: + * _http_conn_active under _http_conn_mu is exactly "capacity currently + * claimed." The dreamer waits for it to reach zero and yields the moment it + * does not. That is the anticorrelation, literally rather than by analogy. + * + * CONTRACT: the handler performs ONE step and returns. The runtime cannot + * preempt El code, so interruptibility is at step granularity — a step must + * be small enough that a request arriving mid-step is not made to wait. It + * returns non-zero if it did work. Returning zero means "nothing to + * consolidate," and the dreamer then blocks until activity changes rather + * than spinning. There is no timer anywhere in this file for this purpose, + * and adding one would be the defect described above. + * + * `depth` is derived from CONTINUOUS unclaimed time: a brief gap affords a + * shallow recombination; a long quiet affords a deep one. Same process. Sleep + * is where unclaimed capacity is greatest, not where the process lives. */ +typedef el_val_t (*dream_fn)(el_val_t depth); +static char* _dream_handler = NULL; +static int _dream_started = 0; + +static int64_t dream_now_ms(void) { + struct timespec ts; +#if defined(CLOCK_MONOTONIC) + clock_gettime(CLOCK_MONOTONIC, &ts); +#else + clock_gettime(CLOCK_REALTIME, &ts); +#endif + return (int64_t)ts.tv_sec * 1000 + ts.tv_nsec / 1000000; +} + +static dream_fn dream_lookup(void) { + dream_fn out = NULL; + pthread_mutex_lock(&_http_handler_mu); + if (_dream_handler && *_dream_handler) + out = (dream_fn)dlsym(RTLD_DEFAULT, _dream_handler); + pthread_mutex_unlock(&_http_handler_mu); + return out; +} + +static void* dream_loop(void* unused) { + (void)unused; + int64_t idle_since = 0; + for (;;) { + /* Wait for unclaimed capacity. Any engagement resets the depth clock: + * depth reflects CONTINUOUS quiet, so an interruption starts it over. */ + pthread_mutex_lock(&_http_conn_mu); + while (_http_conn_active > 0) { + idle_since = 0; + pthread_cond_wait(&_http_conn_cv, &_http_conn_mu); + } + pthread_mutex_unlock(&_http_conn_mu); + + int64_t now = dream_now_ms(); + if (idle_since == 0) idle_since = now; + int64_t quiet = now - idle_since; + + /* Depth from unclaimed capacity. Not a schedule — a gradient. */ + int depth = quiet < 1000 ? 1 /* a gap between requests */ + : quiet < 30000 ? 2 /* a lull */ + : quiet < 300000 ? 3 /* sustained quiet */ + : 4; /* deep: the "sleep" case */ + + dream_fn fn = dream_lookup(); + if (!fn) return NULL; /* handler vanished: stop, do not spin */ + + el_val_t did_work = fn((el_val_t)depth); + + if (!(int64_t)did_work) { + /* Nothing to consolidate. Do NOT poll — block until engagement + * changes. If there is nothing to dream about, wait for something + * to happen rather than asking again on a timer. */ + pthread_mutex_lock(&_http_conn_mu); + while (_http_conn_active == 0) + pthread_cond_wait(&_http_conn_cv, &_http_conn_mu); + pthread_mutex_unlock(&_http_conn_mu); + idle_since = 0; + } + } + return NULL; +} + +/* dream_set_handler(name) — register the consolidation step and start + * dreaming. Resolves by dlsym against the running binary, the same mechanism + * http_set_handler uses: every El `fn name(...)` compiles to a global C symbol + * with that exact name. Inert until called, so a program that never registers + * one simply never dreams and pays nothing. */ +void dream_set_handler(el_val_t name) { + const char* n = EL_CSTR(name); + pthread_mutex_lock(&_http_handler_mu); + free(_dream_handler); + _dream_handler = el_strdup(n ? n : ""); + int start = (!_dream_started && n && *n && dlsym(RTLD_DEFAULT, n) != NULL); + if (start) _dream_started = 1; + pthread_mutex_unlock(&_http_handler_mu); + if (start) { + pthread_t tid; + if (pthread_create(&tid, NULL, dream_loop, NULL) == 0) pthread_detach(tid); + else { pthread_mutex_lock(&_http_handler_mu); _dream_started = 0; pthread_mutex_unlock(&_http_handler_mu); } + } +} + static http_handler_fn http_lookup_active(void) { http_handler_fn out = NULL; pthread_mutex_lock(&_http_handler_mu); @@ -1259,10 +1478,63 @@ static const char* http_reason_phrase(int status) { } } -/* Best-effort send with retry on partial writes. */ +/* A DISCONNECTING CLIENT MUST NOT KILL THE SERVER (2026-08-16). + * + * There was no SIGPIPE handling anywhere in this runtime: no signal disposition, + * no MSG_NOSIGNAL, no SO_NOSIGPIPE, and send() called with bare flags. The + * default disposition of SIGPIPE is to TERMINATE THE PROCESS, so any client that + * hung up mid-response — a curl that hit its timeout, a browser tab closed + * during a large read, a proxy giving up — took the whole engram down with it. + * + * Measured on the live instance: 18 boots in the log, and `launchctl list` + * reporting the previous exit for ai.neuron.engram as -13, i.e. killed by + * signal 13 = SIGPIPE. Reproduced by the cause: pulling /api/nodes/list (26 MB) + * with a client-side timeout. launchd's KeepAlive then restarts it, so the + * failure looks like a mysterious restart rather than a crash, and the graph + * silently reloads under whatever was mid-flight. + * + * This is an exemption in the §8 sense: the write never checked whether the + * peer was still there, and the consequence of not checking was fatal rather + * than merely wrong. + * + * Two layers, because neither alone is portable: + * - SO_NOSIGPIPE per socket (Darwin/BSD) and MSG_NOSIGNAL per send (Linux), + * so the signal is never raised for socket writes in the first place. + * - A process-wide SIG_IGN as the backstop for platforms/paths with neither, + * installed once and idempotent. With the signal ignored, send() returns + * -1/EPIPE and the existing error path closes the connection. */ +#ifndef MSG_NOSIGNAL +#define MSG_NOSIGNAL 0 +#endif + +static void el_ignore_sigpipe_once(void) { + static int done = 0; + if (done) return; + done = 1; +#ifndef _WIN32 + signal(SIGPIPE, SIG_IGN); +#endif +} + +/* Per-socket suppression where the platform offers it. Best-effort: a failure + * here is not fatal because el_ignore_sigpipe_once() already covers the case. */ +static void el_sock_nosigpipe(int fd) { +#if defined(SO_NOSIGPIPE) + int on = 1; + setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &on, sizeof(on)); +#else + (void)fd; +#endif +} + +/* Best-effort send with retry on partial writes. EPIPE/ECONNRESET are a client + * that left, not a server fault: return -1 so the caller closes the connection, + * and never let it reach the process as a signal. */ static int http_send_all(int fd, const char* p, size_t left) { + el_ignore_sigpipe_once(); while (left > 0) { - ssize_t w = send(fd, p, left, 0); + ssize_t w = send(fd, p, left, MSG_NOSIGNAL); + if (w < 0 && errno == EINTR) continue; if (w <= 0) return -1; p += w; left -= (size_t)w; } @@ -1525,8 +1797,64 @@ typedef struct { * no longer blocks ingest/reads (measured: non-health latency during a beat * 13.9s → sub-second). */ static pthread_mutex_t g_engram_req_lock = PTHREAD_MUTEX_INITIALIZER; -void engram_req_unlock(void){ pthread_mutex_unlock(&g_engram_req_lock); } -void engram_req_lock(void){ pthread_mutex_lock(&g_engram_req_lock); } + +/* ── AWARENESS-THREAD GUARD (2026-08-16 self-review) ───────────────────────── + * The request lock above serialized http_worker threads against EACH OTHER, but + * the soul daemon has a SECOND, unsynchronized engram caller: soul.el starts the + * HTTP server with http_serve_async (spawning worker threads) and then runs + * awareness_run() on the MAIN thread, whose perceive() -> engram_activate_json() + * -> engram_activate() -> eg_vindex_sync() path mutates the very same RAM graph + * and the process-global _eg_vindex HNSW index. Nothing in any .el source ever + * called engram_req_lock, so that whole loop ran lock-free beside the workers. + * + * Measured consequence (2026-08-16): five crashes in ~4 minutes, all one bug — + * SIGSEGV in search_layer<-vindex_insert<-eg_vindex_sync at address + * 0x65646f4e6d617267 (little-endian ASCII "gramNode": a string being + * dereferenced as an Elem vector pointer), plus a SIGABRT and a fault inside + * xzm_realloc's freelist, i.e. corrupted allocator metadata. Confirmed by + * bisection: replaying ALL 13,820 real dim-768 store vectors through the index + * single-threaded under ASan is 100% clean, while two threads on one index trip + * ThreadSanitizer instantly at engram_vindex.c:195 (visited_reset) — VIndex keeps + * a SHARED visited-epoch scratch buffer, so even two concurrent READS stomp each + * other's traversal state and walk bogus element indices. So this is purely a + * concurrency defect, not a logic error in the HNSW code. + * + * Fix: a thread-local ownership depth lets engram entry points self-guard. A call + * arriving on the awareness thread (depth 0) acquires the lock; one arriving from + * inside an http_worker that already holds it (depth > 0) is a no-op, so there is + * no self-deadlock on this NON-recursive mutex. Depth is a plain counter, never a + * recursive-mutex count, which preserves engram_self_reify_beat_json's contract of + * really releasing the lock mid-beat (see engram_req_unlock at the reify beat). + * + * SCOPE NARROWED (2026-08-16, vindex publication boundary): this guard originally + * covered TWO hazards — the RAM graph AND the process-global _eg_vindex. The vindex + * half is retired: the index now has its own publication boundary (_eg_vindex_rw), + * search takes a `const VIndex*`, and no read path can mutate the index at all. + * + * What REMAINS load-bearing here is the RAM graph alone, and it is a genuine, + * measured hazard independent of the index: g->nodes / g->edges are realloc'd in + * place (el_runtime.c:7618, 7629), so an awareness-thread reader holding + * `EngramNode* n = &g->nodes[i]` across a concurrent append from an http_worker + * holds a dangling pointer — and engram_activate_inner's embed-backfill WRITES + * n->emb through exactly such a pointer. That is a separate residue with its own + * fix (the resident graph wants the same publication treatment the index just got); + * until it lands, this guard stays. Do NOT delete it as "the fb32d15 vindex lock". */ +static __thread int _eg_req_depth = 0; +void engram_req_unlock(void){ if(_eg_req_depth > 0) _eg_req_depth--; pthread_mutex_unlock(&g_engram_req_lock); } +void engram_req_lock(void){ pthread_mutex_lock(&g_engram_req_lock); _eg_req_depth++; } +/* Acquire only if this thread does not already hold the request lock. + * Returns 1 if this call took ownership (caller must release), 0 if nested. */ +static int eg_guard_enter(void){ + if (_eg_req_depth > 0) return 0; + pthread_mutex_lock(&g_engram_req_lock); + _eg_req_depth++; + return 1; +} +static void eg_guard_exit(int owned){ + if (!owned) return; + if (_eg_req_depth > 0) _eg_req_depth--; + pthread_mutex_unlock(&g_engram_req_lock); +} static void* http_worker(void* arg) { HttpWorkerArg* a = (HttpWorkerArg*)arg; @@ -1567,7 +1895,7 @@ static void* http_worker(void* arg) { (plen == 1 && path[0] == '/')) health_exempt = 1; } - if (!health_exempt) pthread_mutex_lock(&g_engram_req_lock); + if (!health_exempt) engram_req_lock(); /* tracks _eg_req_depth for eg_guard_enter */ if (h) { el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), EL_STR(body)); const char* rs = EL_CSTR(r); @@ -1594,7 +1922,7 @@ static void* http_worker(void* arg) { } /* end of the engram critical section — the response is now a private malloc'd * copy; arena teardown + socket write touch no shared engram state. */ - if (!health_exempt) pthread_mutex_unlock(&g_engram_req_lock); + if (!health_exempt) engram_req_unlock(); el_request_end(); /* free all intermediate strings */ _tl_http_head_only = head_only; http_send_response(fd, response); @@ -1606,7 +1934,12 @@ static void* http_worker(void* arg) { /* release a slot */ pthread_mutex_lock(&_http_conn_mu); _http_conn_active--; - pthread_cond_signal(&_http_conn_cv); + /* BROADCAST, not signal (2026-08-16): the ambient consolidation thread + * waits on this same condvar for _http_conn_active == 0. cond_signal wakes + * exactly one waiter, so the accept loop could take every wake and starve + * the dreamer indefinitely. Both wait sites re-check their predicate in a + * while loop, so broadcasting is safe. */ + pthread_cond_broadcast(&_http_conn_cv); pthread_mutex_unlock(&_http_conn_mu); return NULL; } @@ -1656,6 +1989,7 @@ void http_serve(el_val_t port, el_val_t handler) { pthread_mutex_unlock(&_http_conn_mu); HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); if (!arg) { el_closesocket(cfd); continue; } + el_sock_nosigpipe(cfd); arg->fd = cfd; pthread_t tid; if (pthread_create(&tid, NULL, http_worker, arg) != 0) { @@ -1702,6 +2036,7 @@ static void* _http_serve_async_loop(void* raw) { pthread_mutex_unlock(&_http_conn_mu); HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); if (!arg) { close(cfd); continue; } + el_sock_nosigpipe(cfd); arg->fd = cfd; pthread_t tid; if (pthread_create(&tid, NULL, http_worker, arg) != 0) { @@ -1951,7 +2286,12 @@ static void* http_worker_v2(void* arg) { el_closesocket(fd); pthread_mutex_lock(&_http_conn_mu); _http_conn_active--; - pthread_cond_signal(&_http_conn_cv); + /* BROADCAST, not signal (2026-08-16): the ambient consolidation thread + * waits on this same condvar for _http_conn_active == 0. cond_signal wakes + * exactly one waiter, so the accept loop could take every wake and starve + * the dreamer indefinitely. Both wait sites re-check their predicate in a + * while loop, so broadcasting is safe. */ + pthread_cond_broadcast(&_http_conn_cv); pthread_mutex_unlock(&_http_conn_mu); return NULL; } @@ -2002,6 +2342,7 @@ void http_serve_v2(el_val_t port, el_val_t handler) { pthread_mutex_unlock(&_http_conn_mu); HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); if (!arg) { el_closesocket(cfd); continue; } + el_sock_nosigpipe(cfd); arg->fd = cfd; pthread_t tid; if (pthread_create(&tid, NULL, http_worker_v2, arg) != 0) { @@ -3347,28 +3688,74 @@ static void jb_puts(JsonBuf* b, const char* s) { b->buf[b->len] = '\0'; } +/* UTF-8 VALIDITY IS THE EMITTER'S CONTRACT (2026-08-16 self-review). + * + * This copied every byte >= 0x20 through verbatim, so a malformed sequence + * anywhere in the store became malformed output. Measured against the live + * graph: three nodes carry labels truncated to exactly 80 bytes ending in a + * lone 0xE2 — the first byte of an em-dash, cut mid-sequence by some producer + * that is NOT this runtime (no 80-byte truncation exists here; the content + * itself is 2572 and 2746 bytes). Those three nodes made the ENTIRE 26 MB + * /api/nodes/list response undecodable, so a strict parser could not read the + * graph at all. + * + * Fixing only the writer would not have helped: the store already contains the + * damage, and it accepts data from importers, other producers and older + * binaries. A serializer that promises JSON owes valid UTF-8 regardless of what + * it is handed — so validate here, at the boundary that makes the promise. + * Invalid bytes become U+FFFD rather than being dropped, so damage stays + * visible in the output instead of being silently papered over. + * + * Well-formed input is byte-identical to before: valid sequences are copied + * verbatim, and only structurally invalid ones (bad lead byte, missing or bad + * continuation, overlong encoding, UTF-16 surrogate, or > U+10FFFF) are + * replaced. */ static void jb_emit_escaped(JsonBuf* b, const char* s) { jb_putc(b, '"'); - for (; *s; s++) { - unsigned char c = (unsigned char)*s; + const unsigned char* p = (const unsigned char*)s; + while (*p) { + unsigned char c = *p; switch (c) { - case '"': jb_puts(b, "\\\""); break; - case '\\': jb_puts(b, "\\\\"); break; - case '\b': jb_puts(b, "\\b"); break; - case '\f': jb_puts(b, "\\f"); break; - case '\n': jb_puts(b, "\\n"); break; - case '\r': jb_puts(b, "\\r"); break; - case '\t': jb_puts(b, "\\t"); break; - default: - if (c < 0x20) { - char tmp[8]; - snprintf(tmp, sizeof(tmp), "\\u%04x", c); - jb_puts(b, tmp); - } else { - jb_putc(b, (char)c); - } - break; + case '"': jb_puts(b, "\\\""); p++; continue; + case '\\': jb_puts(b, "\\\\"); p++; continue; + case '\b': jb_puts(b, "\\b"); p++; continue; + case '\f': jb_puts(b, "\\f"); p++; continue; + case '\n': jb_puts(b, "\\n"); p++; continue; + case '\r': jb_puts(b, "\\r"); p++; continue; + case '\t': jb_puts(b, "\\t"); p++; continue; + default: break; } + if (c < 0x20) { + char tmp[8]; + snprintf(tmp, sizeof(tmp), "\\u%04x", c); + jb_puts(b, tmp); + p++; + continue; + } + if (c < 0x80) { jb_putc(b, (char)c); p++; continue; } + + /* Multi-byte: validate the whole sequence before emitting any of it. */ + int len; unsigned int cp; + if ((c & 0xE0) == 0xC0) { len = 2; cp = c & 0x1Fu; } + else if ((c & 0xF0) == 0xE0) { len = 3; cp = c & 0x0Fu; } + else if ((c & 0xF8) == 0xF0) { len = 4; cp = c & 0x07u; } + else { jb_puts(b, "\\ufffd"); p++; continue; } + + int ok = 1; + for (int i = 1; i < len; i++) { + if ((p[i] & 0xC0) != 0x80) { ok = 0; break; } /* also catches NUL */ + cp = (cp << 6) | (unsigned int)(p[i] & 0x3F); + } + if (ok) { + if (len == 2 && cp < 0x80) ok = 0; /* overlong */ + else if (len == 3 && cp < 0x800) ok = 0; /* overlong */ + else if (len == 4 && cp < 0x10000) ok = 0; /* overlong */ + else if (cp >= 0xD800 && cp <= 0xDFFF) ok = 0; /* UTF-16 surrogate */ + else if (cp > 0x10FFFF) ok = 0; /* out of range */ + } + if (!ok) { jb_puts(b, "\\ufffd"); p++; continue; } + for (int i = 0; i < len; i++) jb_putc(b, (char)p[i]); + p += len; } jb_putc(b, '"'); } @@ -5093,10 +5480,23 @@ el_val_t state_get(el_val_t key) { if (!k) return el_wrap_str(el_strdup("")); pthread_mutex_lock(&_state_mu); StateEntry* e = state_find(k); - char* result = el_strdup_persist(e ? e->value : ""); + /* ONE arena-tracked copy, taken under the lock. + * + * This used to make TWO copies: an el_strdup_persist temporary, then an + * arena-tracked copy of that temporary. The persistent one was never + * returned and never freed — el_strdup_persist bypasses the arena by + * design ("state_set, engram internals"), so arena-pop could not reclaim + * it. Every state_get therefore leaked its full value string, permanently. + * + * The soul's awareness loop has 68 state_get call sites and ticks every + * 200ms; measured leak was ~1.1 MB per tick, about 19 GB/hour. It went + * unnoticed for as long as the soul restarted often enough to mask it. + * + * el_strdup tracks into the thread-local arena, which touches no shared + * state, so doing it under _state_mu is safe and removes the need for the + * temporary entirely. */ + char* copy = el_strdup(e ? e->value : ""); pthread_mutex_unlock(&_state_mu); - /* wrap in arena-tracked copy for the caller's request lifetime */ - char* copy = el_strdup(result); return el_wrap_str(copy); } @@ -5174,7 +5574,12 @@ el_val_t str_to_float(el_val_t s) { /* ── Math (Float-aware) ──────────────────────────────────────────────────── */ el_val_t math_sqrt(el_val_t f) { return el_from_float(sqrt(el_to_float(f))); } -el_val_t math_log(el_val_t f) { return el_from_float(log(el_to_float(f))); } +/* base-10, matching runtime/math.el's documented contract ("math_log — base-10 + * logarithm") and el_seed.c's __log_f. This returned NATURAL log, so math_log + * and math_ln were the same function: log10(100) gave 4.605 instead of 2. + * Caught by tests/native/test_math.el on the new framework's first run — the + * assertion existed all along, the suite just had no way to report it. */ +el_val_t math_log(el_val_t f) { return el_from_float(log10(el_to_float(f))); } el_val_t math_ln(el_val_t f) { return el_from_float(log(el_to_float(f))); } el_val_t math_sin(el_val_t f) { return el_from_float(sin(el_to_float(f))); } el_val_t math_cos(el_val_t f) { return el_from_float(cos(el_to_float(f))); } @@ -5231,7 +5636,7 @@ el_val_t str_char_code(el_val_t s, el_val_t i) { const char* str = EL_CSTR(s); int64_t idx = (int64_t)i; if (!str) return 0; - int64_t n = (int64_t)strlen(str); + int64_t n = (int64_t)el_strlen_cached(str); if (idx < 0 || idx >= n) return 0; return (el_val_t)(unsigned char)str[idx]; } @@ -5367,6 +5772,45 @@ el_val_t str_count(el_val_t sv, el_val_t subv) { return (el_val_t)count; } +/* el_utf8_safe_len — the largest byte length <= max_bytes that does NOT split a + * UTF-8 codepoint. + * + * WHY (2026-08-16 self-review): engram_first_n_chars truncated with a plain + * `if (l > n) l = n; memcpy(...)`, i.e. by BYTES despite its name. Any content + * carrying a multi-byte character across the 60-byte boundary produced a label + * ending in a half codepoint. That label is copied verbatim into every JSON + * document containing the node, so a single such node makes the WHOLE response + * invalid UTF-8 — /api/nodes/list failed to decode at byte 89261 against the + * live store, which breaks any strict parser reading the graph. + * + * This lives beside str_count_chars rather than in the engram because the rest + * of el's string layer is already codepoint-aware (str_count_chars counts + * codepoints, str_reverse walks codepoint lengths). Byte-truncation was the + * outlier, and the concern is a string concern. Bounded by BYTES, not + * codepoints, so existing labels never grow — only stop splitting. + * + * A lead byte with no room for its full sequence is dropped entirely; a stray + * continuation byte (already-invalid input) is passed through unchanged rather + * than silently repaired, so this never manufactures data. */ +size_t el_utf8_safe_len(const char* s, size_t max_bytes) { + if (!s) return 0; + size_t len = strlen(s); + if (len <= max_bytes) return len; + size_t i = 0; + while (i < max_bytes) { + unsigned char c = (unsigned char)s[i]; + size_t cp_len; + if ((c & 0x80) == 0x00) cp_len = 1; + else if ((c & 0xE0) == 0xC0) cp_len = 2; + else if ((c & 0xF0) == 0xE0) cp_len = 3; + else if ((c & 0xF8) == 0xF0) cp_len = 4; + else cp_len = 1; /* stray continuation: passthrough */ + if (i + cp_len > max_bytes) break; /* would split — stop before it */ + i += cp_len; + } + return i; +} + /* Codepoint count: walk bytes, count those NOT matching 10xxxxxx. */ el_val_t str_count_chars(el_val_t sv) { const char* s = EL_CSTR(sv); @@ -5810,6 +6254,622 @@ void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal, } +/* ── Geometry: signal as a first-class el value ────────────────────────────── + * + * WHY THIS IS IN THE LANGUAGE, AND WHY IT IS DEFINED HERE (2026-08-16). + * + * Until yesterday no El ingest path could carry a vector. Nodes took text, + * and geometry was DERIVED from that text by engram_embed_backfill. Text was + * therefore the mandatory entry medium: any non-text modality — audio, image, + * sensor — had to be DESCRIBED in prose first, so the geometry we then + * reasoned over was the geometry OF THE DESCRIPTION, not of the signal. That + * is faking it. The architecture is: geometry in, always; we do not fake it, + * we project. + * + * The first fix (#141, engram_node_set_emb) proved the path end to end but + * placed it wrong in three ways, each of which this section corrects: + * + * 1. It sat at the CONSUMER. Transduction is a LANGUAGE concern — every El + * program touching any modality needs it, not just the one that happens + * to hold a graph. So this section is defined HERE, immediately above + * the engram block, and depends on nothing inside it. The engram is a + * client of this surface, not its owner. That ordering is the point: + * you can delete the entire engram and geometry still enters El. + * + * 2. It marshalled the vector as a hex STRING, because El had no + * first-class geometry value — which reintroduced text as the TRANSPORT + * medium one layer below the problem being fixed. Geometry is now a + * value. Hex survives only as a wire ADAPTER at the edge + * (geometry_from/to_f32le_hex), which is all an encoding should ever be. + * + * 3. It needed an arbitrary `dim <= 8192` bound, purely to check a + * caller-supplied dim against a string's length before allocating. A + * real geometry value CARRIES its own width, so here the width is + * derived and never asserted, and there is nothing left to validate. + * The bound is gone rather than merely raised — the only thing that can + * fail is the allocation itself, which is an honest failure. + * + * REPRESENTATION: magic-tagged heap object (see "Refcounted heap objects"), + * carried in an el_val_t. The payload is a separate allocation so the header + * never moves. The magic word is >= 0x80 in its MSB so the string/small-int + * sniffing in looks_like_heap_obj can never confuse a Geometry for either. + * + * OWNERSHIP: a Geometry is owned by the El caller and released with + * geometry_free. node_attach_geometry COPIES its payload into the node, so a + * node and the caller's value have independent lifetimes and freeing one + * never touches the other. Geometry deliberately does NOT participate in + * el_retain/el_release: the shipped elc emits neither on let-bindings + * (measured), so hooking it there would be dead code that could only ever + * free a live vector early. + */ + +#define EL_MAGIC_GEOM 0xE1608E01u + +typedef struct { + ElHeader hdr; + int32_t dim; + float* v; +} ElGeometry; + +/* Resolve an el_val_t to a live Geometry, or NULL. Every accessor goes + * through this, so a stale/foreign/zero value is a clean 0-return rather + * than a dereference. */ +static ElGeometry* geom_of(el_val_t g) { + if (!looks_like_heap_obj(g)) return NULL; + ElGeometry* p = (ElGeometry*)(uintptr_t)g; + if (p->hdr.magic != EL_MAGIC_GEOM) return NULL; + return p; +} + +el_val_t geometry_new(el_val_t dim) { + int32_t d = (int32_t)(int64_t)dim; + if (d <= 0) return (el_val_t)0; + ElGeometry* g = (ElGeometry*)malloc(sizeof(ElGeometry)); + if (!g) return (el_val_t)0; + g->v = (float*)calloc((size_t)d, sizeof(float)); + if (!g->v) { free(g); return (el_val_t)0; } + g->hdr.magic = EL_MAGIC_GEOM; + g->hdr.refcount = 1; + g->dim = d; + return (el_val_t)(uintptr_t)g; +} + +el_val_t geometry_dim(el_val_t g) { + ElGeometry* p = geom_of(g); + return p ? (el_val_t)p->dim : (el_val_t)0; +} + +el_val_t geometry_is(el_val_t g) { + return geom_of(g) ? (el_val_t)1 : (el_val_t)0; +} + +el_val_t geometry_get(el_val_t g, el_val_t i) { + ElGeometry* p = geom_of(g); + int64_t k = (int64_t)i; + if (!p || k < 0 || k >= (int64_t)p->dim) return el_from_float(0.0); + return el_from_float((double)p->v[k]); +} + +el_val_t geometry_set(el_val_t g, el_val_t i, el_val_t x) { + ElGeometry* p = geom_of(g); + int64_t k = (int64_t)i; + if (!p || k < 0 || k >= (int64_t)p->dim) return (el_val_t)0; + p->v[k] = (float)el_to_float(x); + return (el_val_t)1; +} + +el_val_t geometry_norm(el_val_t g) { + ElGeometry* p = geom_of(g); + if (!p) return el_from_float(0.0); + double s = 0.0; + for (int32_t i = 0; i < p->dim; i++) s += (double)p->v[i] * (double)p->v[i]; + return el_from_float(sqrt(s)); +} + +el_val_t geometry_free(el_val_t g) { + ElGeometry* p = geom_of(g); + if (!p) return (el_val_t)0; + free(p->v); + p->hdr.magic = 0; /* poison so use-after-free is detected, as List/Map do */ + free(p); + return (el_val_t)1; +} + +/* geometry_from_f32le_hex — decode little-endian float32 hex INTO geometry. + * + * This is the ONE place hex appears, and it appears as what it actually is: + * an encoding at the boundary, not the medium El reasons in. The width is + * DERIVED from the input length (8 hex chars per float32) and never supplied + * by the caller — which is precisely why #141's arbitrary `dim <= 8192` + * bound has no counterpart here. There is nothing to validate. + * + * Returns 0 on empty input, a length that is not a multiple of 8, or any + * non-hex character. */ +el_val_t geometry_from_f32le_hex(el_val_t hex) { + const char* s = EL_CSTR(hex); + if (!s) return (el_val_t)0; + size_t n = strlen(s); + if (n == 0 || (n % 8u) != 0) return (el_val_t)0; + size_t d = n / 8u; + if (d > (size_t)INT32_MAX) return (el_val_t)0; + + el_val_t gv = geometry_new((el_val_t)(int64_t)d); + ElGeometry* g = geom_of(gv); + if (!g) return (el_val_t)0; + + for (size_t i = 0; i < d; i++) { + uint32_t w = 0; + for (int k = 0; k < 8; k++) { + char c = s[i * 8u + (size_t)k]; + uint32_t nib; + if (c >= '0' && c <= '9') nib = (uint32_t)(c - '0'); + else if (c >= 'a' && c <= 'f') nib = (uint32_t)(c - 'a' + 10); + else if (c >= 'A' && c <= 'F') nib = (uint32_t)(c - 'A' + 10); + else { geometry_free(gv); return (el_val_t)0; } + w = (w << 4) | nib; + } + /* Hex is emitted little-endian byte order; rebuild the word. */ + uint32_t le = ((w & 0x000000FFu) << 24) | ((w & 0x0000FF00u) << 8) | + ((w & 0x00FF0000u) >> 8) | ((w & 0xFF000000u) >> 24); + float f; + memcpy(&f, &le, sizeof(f)); + g->v[i] = f; + } + return gv; +} + +/* geometry_to_f32le_hex — the egress adapter, exact inverse of the above. + * Present so a program that must hand geometry to a non-El peer over a text + * wire can do so explicitly, at the edge, instead of the language pretending + * text was the medium all along. */ +el_val_t geometry_to_f32le_hex(el_val_t g) { + ElGeometry* p = geom_of(g); + if (!p) return EL_STR(""); + static const char* HEXD = "0123456789abcdef"; + size_t n = (size_t)p->dim * 8u; + char* out = el_strbuf(n); /* arena-tracked; allocates n+1, exits on OOM */ + for (int32_t i = 0; i < p->dim; i++) { + uint32_t w; + memcpy(&w, &p->v[i], sizeof(w)); + /* Emit little-endian byte order: low byte first. */ + for (int b = 0; b < 4; b++) { + uint32_t byte = (w >> (8 * b)) & 0xFFu; + out[(size_t)i * 8u + (size_t)b * 2u] = HEXD[(byte >> 4) & 0xF]; + out[(size_t)i * 8u + (size_t)b * 2u + 1] = HEXD[byte & 0xF]; + } + } + out[n] = '\0'; + return (el_val_t)(uintptr_t)out; +} + + +/* ── Manifold: a transduced signal is a SUBGRAPH, not a point ──────────────── + * + * WHAT THIS CORRECTS. #144 gave transduction a home in the language and got + * the DISPATCH right — realizers declared in El, resolved by name, no runtime + * patch per modality. It got the OUTPUT TYPE wrong. + * `transduce(signal, modality) -> Geometry` yields one vector per signal, and + * one vector is a FINGERPRINT. A fingerprint can be matched and it can be + * ranked; that is the whole of what it can ever do. It cannot be decomposed, + * cannot be partially grounded, and cannot be contradicted in one part while + * holding in another — because it has no parts. + * + * A song is not a point. It decomposes into pitch, interval, rhythm, harmonic + * function, phrase structure: components, each with its own geometry, plus the + * relations between them. THE SONG IS THE STRUCTURE OF THE RELATIONS. A + * transducer that returns a single vector has not transduced the song, it has + * summarised it — and the summary discards precisely the thing that made the + * song reasonable-about. + * + * So transduction produces a MANIFOLD: named components, each carrying its own + * geometry, and typed weighted relations among them. Signal in, subgraph out. + * Conversion was never the operation. + * + * COMPONENTS ARE ADDRESSED BY KEY, NEVER BY INDEX. The key is what survives + * persistence: a component becomes a node, and that node is separately + * groundable precisely because it is separately NAMED. Index-addressing would + * make a grounding reference positional, and a positional reference into a + * decomposition whose arity can change is not a reference at all. Duplicate + * keys are refused for the same reason: two components answering to one name + * is not an addressing scheme. + * + * RELATION WEIGHT IS THE GROUNDING — there is no second field and no score to + * compute. Per correspondence-and-censorship.md §1, grounding is an attribute + * of the edge and it IS the hebbian weight; a grounding subsystem is a + * supervisor invented for something that should be a property of the + * substrate. A relation emitted by a realizer therefore arrives with its + * grounding already on it and moves thereafter by use and by decay (§4: change + * is not a consequence of use, it is use). Nothing in here computes a + * grounding, and nothing observes one. + * + * A relation naming an endpoint that does not exist is REFUSED, not dropped. A + * decomposition that silently loses edges is indistinguishable from one that + * never had them — the same class of defect #141 exists to end. + * + * OWNERSHIP mirrors Geometry exactly. A Manifold is owned by the El caller and + * released with manifold_free. manifold_add COPIES the geometry handed to it, + * so a caller may free its own vector immediately and no component's geometry + * is ever aliased. Keys, roles and relation strings are _persist copies, NOT + * arena copies: a Manifold outlives the request arena that built it (a + * realizer can be invoked from inside a handler), so an arena-tracked key + * would dangle at el_request_end. manifold_free owns their release. + */ + +#define EL_MAGIC_MFLD 0xE1608E02u + +typedef struct { + char* key; /* addressable name, unique within the manifold */ + char* role; /* what KIND of component this is, realizer's vocabulary */ + ElGeometry* g; /* owned copy; never aliases the caller's value */ +} ElComponent; + +typedef struct { + char* from; /* component key */ + char* rel; /* relation name */ + char* to; /* component key */ + double weight; /* the grounding; §1 — one quantity, not two fields */ +} ElRelation; + +typedef struct { + ElHeader hdr; + ElComponent* comps; + size_t ncomp, capcomp; + ElRelation* rels; + size_t nrel, caprel; +} ElManifold; + +/* Resolve an el_val_t to a live Manifold, or NULL. Every accessor goes through + * this, so a stale/foreign/zero value is a clean 0-return, never a deref. */ +static ElManifold* mfld_of(el_val_t m) { + if (!looks_like_heap_obj(m)) return NULL; + ElManifold* p = (ElManifold*)(uintptr_t)m; + if (p->hdr.magic != EL_MAGIC_MFLD) return NULL; + return p; +} + +static int mfld_find(ElManifold* p, const char* key) { + for (size_t i = 0; i < p->ncomp; i++) + if (strcmp(p->comps[i].key, key) == 0) return (int)i; + return -1; +} + +el_val_t manifold_new(void) { + ElManifold* p = (ElManifold*)calloc(1, sizeof(ElManifold)); + if (!p) return (el_val_t)0; + p->hdr.magic = EL_MAGIC_MFLD; + p->hdr.refcount = 1; + return (el_val_t)(uintptr_t)p; +} + +el_val_t manifold_is(el_val_t m) { + return mfld_of(m) ? (el_val_t)1 : (el_val_t)0; +} + +/* manifold_add — add one COMPONENT: a named part with its own geometry. + * Returns the component's index, or -1 on any refusal. Refusals are real and + * distinct: an empty key (unaddressable), a duplicate key (ambiguous + * addressing), a value that is not a live Geometry (a part with no geometry is + * not a part). Each is a caller error worth surfacing at the point of the + * mistake rather than as a missing node three layers downstream. */ +el_val_t manifold_add(el_val_t m, el_val_t key, el_val_t role, el_val_t g) { + ElManifold* p = mfld_of(m); + if (!p) return (el_val_t)(int64_t)-1; + const char* k = EL_CSTR(key); + const char* r = EL_CSTR(role); + if (!k || !*k) return (el_val_t)(int64_t)-1; + if (!r) r = ""; + ElGeometry* src = geom_of(g); + if (!src || src->dim <= 0) return (el_val_t)(int64_t)-1; + if (mfld_find(p, k) >= 0) return (el_val_t)(int64_t)-1; /* duplicate key */ + + if (p->ncomp == p->capcomp) { + size_t nc = p->capcomp ? p->capcomp * 2 : 8; + ElComponent* nb = (ElComponent*)realloc(p->comps, nc * sizeof(ElComponent)); + if (!nb) return (el_val_t)(int64_t)-1; + p->comps = nb; p->capcomp = nc; + } + + /* COPY the payload — a component's geometry must not alias the caller's. */ + ElGeometry* cp = (ElGeometry*)malloc(sizeof(ElGeometry)); + if (!cp) return (el_val_t)(int64_t)-1; + cp->v = (float*)malloc(sizeof(float) * (size_t)src->dim); + if (!cp->v) { free(cp); return (el_val_t)(int64_t)-1; } + memcpy(cp->v, src->v, sizeof(float) * (size_t)src->dim); + cp->hdr.magic = EL_MAGIC_GEOM; + cp->hdr.refcount = 1; + cp->dim = src->dim; + + p->comps[p->ncomp].key = el_strdup_persist(k); + p->comps[p->ncomp].role = el_strdup_persist(r); + p->comps[p->ncomp].g = cp; + p->ncomp++; + return (el_val_t)(int64_t)(p->ncomp - 1); +} + +/* manifold_relate — state a relation BETWEEN two components. This is the part + * that carries the meaning: the components are the parts, the relations are + * what the thing IS. + * + * Both endpoints must already exist. An edge to a name that was never added is + * refused with 0, never silently discarded — see the header note. */ +el_val_t manifold_relate(el_val_t m, el_val_t from, el_val_t rel, + el_val_t to, el_val_t weight) { + ElManifold* p = mfld_of(m); + if (!p) return (el_val_t)0; + const char* f = EL_CSTR(from); + const char* r = EL_CSTR(rel); + const char* t = EL_CSTR(to); + if (!f || !*f || !r || !*r || !t || !*t) return (el_val_t)0; + if (mfld_find(p, f) < 0) return (el_val_t)0; + if (mfld_find(p, t) < 0) return (el_val_t)0; + + if (p->nrel == p->caprel) { + size_t nc = p->caprel ? p->caprel * 2 : 8; + ElRelation* nb = (ElRelation*)realloc(p->rels, nc * sizeof(ElRelation)); + if (!nb) return (el_val_t)0; + p->rels = nb; p->caprel = nc; + } + p->rels[p->nrel].from = el_strdup_persist(f); + p->rels[p->nrel].rel = el_strdup_persist(r); + p->rels[p->nrel].to = el_strdup_persist(t); + p->rels[p->nrel].weight = el_to_float(weight); + p->nrel++; + return (el_val_t)1; +} + +el_val_t manifold_size(el_val_t m) { + ElManifold* p = mfld_of(m); + return p ? (el_val_t)(int64_t)p->ncomp : (el_val_t)0; +} + +el_val_t manifold_rel_count(el_val_t m) { + ElManifold* p = mfld_of(m); + return p ? (el_val_t)(int64_t)p->nrel : (el_val_t)0; +} + +/* Index of a component BY KEY, or -1. This is the addressability primitive: + * everything downstream that wants to ground, weight or contradict one part + * finds it through here. */ +el_val_t manifold_index_of(el_val_t m, el_val_t key) { + ElManifold* p = mfld_of(m); + const char* k = EL_CSTR(key); + if (!p || !k || !*k) return (el_val_t)(int64_t)-1; + return (el_val_t)(int64_t)mfld_find(p, k); +} + +el_val_t manifold_key(el_val_t m, el_val_t i) { + ElManifold* p = mfld_of(m); + int64_t k = (int64_t)i; + if (!p || k < 0 || k >= (int64_t)p->ncomp) return el_wrap_str(el_strdup("")); + return el_wrap_str(el_strdup(p->comps[k].key)); +} + +el_val_t manifold_role(el_val_t m, el_val_t i) { + ElManifold* p = mfld_of(m); + int64_t k = (int64_t)i; + if (!p || k < 0 || k >= (int64_t)p->ncomp) return el_wrap_str(el_strdup("")); + return el_wrap_str(el_strdup(p->comps[k].role)); +} + +/* manifold_geometry — the geometry OF ONE COMPONENT, as a fresh Geometry the + * caller owns and frees. A borrowed interior pointer would let a caller's + * geometry_free corrupt the manifold; copying is the same discipline + * node_attach_geometry already applies in the other direction. */ +el_val_t manifold_geometry(el_val_t m, el_val_t i) { + ElManifold* p = mfld_of(m); + int64_t k = (int64_t)i; + if (!p || k < 0 || k >= (int64_t)p->ncomp) return (el_val_t)0; + ElGeometry* src = p->comps[k].g; + el_val_t out = geometry_new((el_val_t)(int64_t)src->dim); + ElGeometry* dst = geom_of(out); + if (!dst) return (el_val_t)0; + memcpy(dst->v, src->v, sizeof(float) * (size_t)src->dim); + return out; +} + +el_val_t manifold_rel_from(el_val_t m, el_val_t j) { + ElManifold* p = mfld_of(m); + int64_t k = (int64_t)j; + if (!p || k < 0 || k >= (int64_t)p->nrel) return el_wrap_str(el_strdup("")); + return el_wrap_str(el_strdup(p->rels[k].from)); +} + +el_val_t manifold_rel_name(el_val_t m, el_val_t j) { + ElManifold* p = mfld_of(m); + int64_t k = (int64_t)j; + if (!p || k < 0 || k >= (int64_t)p->nrel) return el_wrap_str(el_strdup("")); + return el_wrap_str(el_strdup(p->rels[k].rel)); +} + +el_val_t manifold_rel_to(el_val_t m, el_val_t j) { + ElManifold* p = mfld_of(m); + int64_t k = (int64_t)j; + if (!p || k < 0 || k >= (int64_t)p->nrel) return el_wrap_str(el_strdup("")); + return el_wrap_str(el_strdup(p->rels[k].to)); +} + +el_val_t manifold_rel_weight(el_val_t m, el_val_t j) { + ElManifold* p = mfld_of(m); + int64_t k = (int64_t)j; + if (!p || k < 0 || k >= (int64_t)p->nrel) return el_from_float(0.0); + return el_from_float(p->rels[k].weight); +} + +/* manifold_single — the DEGENERATE case, expressible but visibly degenerate. + * + * Sometimes a modality really does have one part (a scalar sensor). That is a + * manifold of size 1, not a different kind of thing, and writing it this way + * keeps the fingerprint as a SPECIAL CASE of decomposition rather than a + * parallel path back to #144's contract. Anything reading it still asks + * manifold_size and still gets a real answer. */ +el_val_t manifold_single(el_val_t key, el_val_t role, el_val_t g) { + el_val_t m = manifold_new(); + if (!mfld_of(m)) return (el_val_t)0; + if ((int64_t)manifold_add(m, key, role, g) < 0) { manifold_free(m); return (el_val_t)0; } + return m; +} + +el_val_t manifold_free(el_val_t m) { + ElManifold* p = mfld_of(m); + if (!p) return (el_val_t)0; + for (size_t i = 0; i < p->ncomp; i++) { + free(p->comps[i].key); + free(p->comps[i].role); + if (p->comps[i].g) { free(p->comps[i].g->v); p->comps[i].g->hdr.magic = 0; free(p->comps[i].g); } + } + for (size_t i = 0; i < p->nrel; i++) { + free(p->rels[i].from); free(p->rels[i].rel); free(p->rels[i].to); + } + free(p->comps); + free(p->rels); + p->hdr.magic = 0; /* poison, as Geometry/List/Map do */ + free(p); + return (el_val_t)1; +} + +/* ── Realizers: transduction declared in El, not patched into the runtime ──── + * + * A REALIZER DECOMPOSES one modality into components and their relations. It + * does not encode a signal to a point — that is the operation one layer below + * it, and it is called geometry, not transduction. A realizer for a modality + * declares what that modality's COMPONENTS ARE: for audio, not one MFCC + * vector, but pitch, interval, rhythm, harmonic function, and how they stand + * to one another. + * + * The whole reason transduction belongs in the language is that ADDING A + * MODALITY MUST NOT REQUIRE A RUNTIME PATCH — otherwise "the realizers are in + * the engram" just becomes "the realizers are in the runtime" and nothing has + * actually moved. So realizers are declared in El and registered by NAME: + * + * fn tone_realizer(signal: String) -> Manifold { + * let m: Manifold = manifold_new() + * let a: Int = manifold_add(m, "pitch", "spectral", pitch_geom) + * let b: Int = manifold_add(m, "interval", "relation", interval_geom) + * let e: Int = manifold_relate(m, "pitch", "spans", "interval", 0.9) + * m + * } + * + * realizer_register("tone", "tone_realizer") + * let m: Manifold = transduce(sample, "tone") + * + * A realizer's DECLARED COMPONENT VOCABULARY is the interesting part of its + * contract, and it is what a caller can then ground, weight and contradict + * one part at a time. + * + * The name→symbol step rides the identical, already load-bearing mechanism + * http_set_handler uses (see "HTTP server"): every El `fn name(...)` compiles + * to a global C symbol with that exact name, so dlsym(RTLD_DEFAULT, name) + * against the running binary resolves an El-defined function. No codegen + * change, no first-class function references, no runtime edit per modality. + * A realizer written in El is a first-class realizer. + * + * A realizer may equally be a C symbol linked into the program; the registry + * cannot tell the difference and has no reason to care. + */ + +typedef el_val_t (*el_realizer_fn)(el_val_t); + +typedef struct { + char* modality; + el_realizer_fn fn; +} ElRealizer; + +static ElRealizer _realizers[64]; +static size_t _realizer_count = 0; +static pthread_mutex_t _realizer_mu = PTHREAD_MUTEX_INITIALIZER; + +static el_realizer_fn realizer_lookup(const char* m) { + el_realizer_fn out = NULL; + pthread_mutex_lock(&_realizer_mu); + for (size_t i = 0; i < _realizer_count; i++) { + if (strcmp(_realizers[i].modality, m) == 0) { out = _realizers[i].fn; break; } + } + pthread_mutex_unlock(&_realizer_mu); + return out; +} + +el_val_t realizer_register(el_val_t modality, el_val_t fn_name) { + const char* m = EL_CSTR(modality); + const char* fn = EL_CSTR(fn_name); + if (!m || !*m || !fn || !*fn) return (el_val_t)0; + + /* An unresolvable name is a REGISTRATION FAILURE, reported as 0 — not a + * silent no-op that only surfaces later as "this modality produces + * nothing". Distinguishing "no organ" from "broken organ" at the moment + * of wiring is the lesson #141 was written to enforce. */ + void* sym = dlsym(RTLD_DEFAULT, fn); + if (!sym) return (el_val_t)0; + + pthread_mutex_lock(&_realizer_mu); + for (size_t i = 0; i < _realizer_count; i++) { + if (strcmp(_realizers[i].modality, m) == 0) { + _realizers[i].fn = (el_realizer_fn)sym; /* re-registration replaces */ + pthread_mutex_unlock(&_realizer_mu); + return (el_val_t)1; + } + } + if (_realizer_count < sizeof(_realizers) / sizeof(_realizers[0])) { + /* _persist, NOT el_strdup: the registry outlives any request, and an + * arena-tracked copy would be freed at el_request_end — leaving a + * dangling modality name if a program registers a realizer from + * inside a handler rather than at startup. */ + _realizers[_realizer_count].modality = el_strdup_persist(m); + _realizers[_realizer_count].fn = (el_realizer_fn)sym; + _realizer_count++; + pthread_mutex_unlock(&_realizer_mu); + return (el_val_t)1; + } + pthread_mutex_unlock(&_realizer_mu); + return (el_val_t)0; +} + +el_val_t realizer_has(el_val_t modality) { + const char* m = EL_CSTR(modality); + if (!m || !*m) return (el_val_t)0; + return realizer_lookup(m) ? (el_val_t)1 : (el_val_t)0; +} + +/* transduce — THE primitive: signal in, SUBGRAPH out. + * + * Dispatches to the realizer registered for `modality`. Returns 0 (not a + * Manifold) when no realizer is registered, and manifold_is() on the result is + * the check. + * + * THE RETURN TYPE IS THE CORRECTION. #144 shipped this as + * `transduce(signal, modality) -> Geometry` — one vector out. That made + * transduction a CONVERSION: take a thing, encode it, store a position. What + * comes back from a conversion is a fingerprint, and a fingerprint supports + * exactly two operations, match and rank. It cannot be decomposed, cannot have + * one part grounded while another is not, and cannot be contradicted in a part + * — it has no parts. Transduction is not conversion. It is DECOMPOSITION into + * components plus the relations among them, and the relations are the content. + * See the Manifold header above. + * + * There is deliberately NO built-in realizer, not even for text. A modality + * the program has declared no organ for is one it genuinely cannot sense, and + * returning nothing is more honest than quietly embedding a description of the + * signal and calling that perception — the failure #144 named, and which a + * single-vector return type quietly reintroduced one level down: a + * one-vector-per-signal organ is a description of the signal, not a perception + * of it. + * + * The result is validated to actually BE a Manifold before it is handed back. + * A realizer still returning a bare Geometry — #144's contract — therefore + * transduces NOTHING rather than handing back a value that decomposes to + * nothing far from here. That is a deliberate hard failure, not an oversight: + * "no organ" and "an organ that only fingerprints" must not look alike, which + * is the same distinction realizer_register draws between an absent and a + * broken organ. A realizer with genuinely one part says so with + * manifold_single. */ +el_val_t transduce(el_val_t signal, el_val_t modality) { + const char* m = EL_CSTR(modality); + if (!m || !*m) return (el_val_t)0; + el_realizer_fn fn = realizer_lookup(m); + if (!fn) return (el_val_t)0; + el_val_t g = fn(signal); + return mfld_of(g) ? g : (el_val_t)0; +} + /* ── Batch 3: Engram in-process graph store ──────────────────────────────── */ /* * Single global EngramStore allocated lazily on first call. All node and @@ -7565,10 +8625,14 @@ static double engram_decode_score(el_val_t v) { return (double)n; } +/* Truncate to at most n BYTES without splitting a UTF-8 codepoint. The old + * implementation was `if (l > n) l = n;` — a byte cut that could land inside a + * multi-byte character and emit a half codepoint into the node's label, which + * then propagated into every JSON document containing that node. See + * el_utf8_safe_len for the measurement. */ static char* engram_first_n_chars(const char* s, size_t n) { if (!s) return el_strdup(""); - size_t l = strlen(s); - if (l > n) l = n; + size_t l = el_utf8_safe_len(s, n); char* out = el_strbuf(l); memcpy(out, s, l); out[l] = '\0'; @@ -7594,6 +8658,7 @@ static char* engram_first_n_chars(const char* s, size_t n) { * mutation (node/edge create, forget) is mirrored through the store's * WAL-logged API so neuron.egm/neuron.wal stay authoritative. * ══════════════════════════════════════════════════════════════════════════ */ +#include "engram_text.h" /* text: tokenize, token hygiene, loss signature */ #include "engram_store.h" #include "engram_vindex.h" /* M8: ANN (HNSW) index for activation seed selection */ #include "engram_geometry.h" /* M9: centered relational-neighborhood geometry (priming) */ @@ -8017,31 +9082,9 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) { * RIGHT NOW" — which is the regression question, and the one that * would have caught this in a day instead of two months. * - * SIGNATURE. Conservative on purpose — a false alarm that cries corruption - * over ordinary punctuation is worse than useless. Two patterns, both of - * which are essentially absent from well-formed English prose: - * (a) alnum '?' alnum — "na?ve", "caf?s", "don?t". A real question mark - * never sits between two word characters. - * (b) ' ? ' followed by a lowercase letter — a lost em/en dash. A real - * question mark is not preceded by a space, and - * what follows one starts a new sentence. - * Deliberately NOT flagged: a trailing '?' after a word, '? ' before a - * capital, or '?' at end of string — all legitimate. This under-counts (it - * cannot see a mangled 'café ' where the '?' landed before a space), so the - * census is a floor on the damage, never an exaggeration of it. */ -static int eg_text_loss_signature(const char* s) { - if (!s) return 0; - for (const char* p = s; *p; p++) { - if (*p != '?') continue; - unsigned char prev = (p == s) ? 0 : (unsigned char)p[-1]; - unsigned char next = (unsigned char)p[1]; - /* (a) sandwiched between word characters. */ - if (isalnum(prev) && isalnum(next)) return 1; - /* (b) spaced, with lowercase continuation — a lost dash. */ - if (prev == ' ' && next == ' ' && islower((unsigned char)p[2])) return 1; - } - return 0; -} + * The SIGNATURE itself (eg_text_loss_signature) moved to engram_text.c on + * 2026-08-16 — it is plain C over and touches nothing in here. The + * stock/flow gauges below stay, because they touch store and EL value types. */ /* Damaged-node creations since process start. See the block comment above. */ static int64_t _eg_txt_write_damaged = 0; @@ -8414,6 +9457,96 @@ el_val_t engram_node_count(void) { return (el_val_t)engram_get()->node_count; } +/* node_attach_geometry — a node acquires geometry. + * + * Named for the operation, not for the store that happens to hold the node. + * This is the geometry-valued ingest path that replaces #141's hex-string + * one: nothing here parses text, and nothing here takes a caller's word for + * how wide the vector is. The Geometry carries its own width. + * + * The payload is COPIED into the node, so the node and the caller's Geometry + * have independent lifetimes — the caller may geometry_free() immediately + * after, and a later free of the node's emb never touches the El value. + * + * DIMENSION POLICY (measured in #141, load-bearing — do not regress): dim + * need NOT equal the canonical text-embedding width. An off-dimension vector + * is stored and is simply not inserted into the resident HNSW index, whose + * build loop already filters on `n->emb_dim == dim`. So a 64-dim voice + * geometry is durable and addressable without perturbing the 768-dim + * canonical index. + * + * Attaching geometry also makes the node ineligible for embed_backfill + * (which fills only nodes with no emb), so a realizer's vector is never + * overwritten by a text-derived one. + * + * Returns 1 on success, 0 on unknown id or a value that is not a Geometry. */ +el_val_t node_attach_geometry(el_val_t node_id, el_val_t g) { + const char* sid = EL_CSTR(node_id); + if (!sid || !*sid) return (el_val_t)0; + + ElGeometry* p = geom_of(g); + if (!p || p->dim <= 0) return (el_val_t)0; + + EngramNode* n = engram_find_node(sid); + if (!n) return (el_val_t)0; + + float* v = (float*)malloc(sizeof(float) * (size_t)p->dim); + if (!v) return (el_val_t)0; + memcpy(v, p->v, sizeof(float) * (size_t)p->dim); + + free(n->emb); + n->emb = v; + n->emb_dim = p->dim; + n->updated_at = engram_now_ms(); + if (engram_store_enabled()) eg_store_put_node(n); + return (el_val_t)1; +} + +/* node_geometry_dim — read the attached width back, 0 if the node carries + * none. Exists so an attach is VERIFIED by reading it back rather than by + * trusting a success return. That is not a nicety: #141 was misdiagnosed for + * an hour precisely because a genuine ingest drop and a mere reporting gap + * were indistinguishable from the outside. */ +el_val_t node_geometry_dim(el_val_t node_id) { + const char* sid = EL_CSTR(node_id); + if (!sid || !*sid) return (el_val_t)0; + EngramNode* n = engram_find_node(sid); + if (!n || !n->emb) return (el_val_t)0; + return (el_val_t)n->emb_dim; +} + +/* engram_node_set_emb — DEPRECATED. Shipped in #141; superseded 2026-08-16 + * by geometry_from_f32le_hex + node_attach_geometry, and now implemented as + * literally that. + * + * It is kept, rather than removed, for one reason only: the runtime is + * published as an SDK asset, so a downstream binary may already be linking + * this symbol. It is NOT kept because a hex string is an acceptable way to + * move geometry between two pieces of El — it isn't, and that was the + * placement defect. New code calls transduce() or geometry_from_f32le_hex() + * plus node_attach_geometry(). + * + * The #141 contract is preserved exactly, including its negative cases, so + * this remains a drop-in: `dim` <= 0 rejects, malformed hex rejects, and a + * `dim` that disagrees with the vector's actual width rejects. The + * difference is that `dim` is now an ASSERTION checked against a width the + * Geometry already knows, rather than the authority the allocation trusted — + * which is why #141's arbitrary `dim <= 8192` guard has no counterpart here. + * There is no longer an unbounded-malloc hazard to guard against. */ +el_val_t engram_node_set_emb(el_val_t id, el_val_t hex, el_val_t dim) { + int32_t want = (int32_t)(int64_t)dim; + if (want <= 0) return (el_val_t)0; + + el_val_t gv = geometry_from_f32le_hex(hex); + ElGeometry* p = geom_of(gv); + if (!p) return (el_val_t)0; /* empty / malformed hex */ + if (p->dim != want) { geometry_free(gv); return (el_val_t)0; } /* length mismatch */ + + el_val_t ok = node_attach_geometry(id, gv); + geometry_free(gv); + return ok; +} + /* ── Telemetry retention ──────────────────────────────────────────────────── * (2026-07-16 self-review) InternalStateEvent nodes are append-only telemetry * (heartbeat, curiosity_scan, engram_sync) written ~3/min by the awareness @@ -8563,37 +9696,7 @@ static int istr_contains(const char* hay, const char* needle) { * fix landed but never reached this release runtime — the copy the engram * binary actually builds against.) */ #define ENGRAM_MAX_QTOKENS 32 -#define ENGRAM_QTOK_LEN 256 - -/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct - * (case-insensitive) tokens. Returns the token count. Over-long tokens are - * truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */ -static int engram_tokenize_query(const char* q, - char toks[][ENGRAM_QTOK_LEN], int maxtok) { - int n = 0; - if (!q) return 0; - const char* p = q; - while (*p && n < maxtok) { - while (*p && isspace((unsigned char)*p)) p++; - if (!*p) break; - char buf[ENGRAM_QTOK_LEN]; - size_t tl = 0; - while (*p && !isspace((unsigned char)*p)) { - if (tl < sizeof(buf) - 1) buf[tl++] = *p; - p++; - } - buf[tl] = '\0'; - if (tl == 0) continue; - int dup = 0; - for (int s = 0; s < n; s++) { - if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; } - } - if (dup) continue; - memcpy(toks[n], buf, tl + 1); - n++; - } - return n; -} +/* ENGRAM_QTOK_LEN and engram_tokenize_query moved to engram_text.h/.c. */ /* Count how many of the ntok distinct query tokens appear (case-insensitive) * in the node's content, label, or tags. 0 == no match. */ @@ -9109,18 +10212,17 @@ el_val_t engram_consolidate_permanence(el_val_t node_id){ * Explicit per-node temporal_decay_rate still overrides lambda (77 nodes carry * one) — that path is untouched and remains the escape hatch for content that * genuinely should expire fast. */ -#define ENGRAM_DECAY_FLOOR 0.25 +/* 2026-08-16: the body moved to cog_decay_factor (engram_cognition.c) so that + * NODE decay and EDGE-GROUNDING decay are one implementation with one set of + * constants, rather than a decay model and a parallel copy of it. The mapping is + * exact — reinforcements := activation_count, lambda_override := + * temporal_decay_rate — so this path is bit-identical to what it replaced. + * COG_T_HALF_HOURS / COG_DECAY_LAMBDA / COG_DECAY_FLOOR carry the same values + * ENGRAM_T_HALF_HOURS / ENGRAM_DECAY_LAMBDA / 0.25 carried here. */ static double engram_temporal_decay(const EngramNode* n, int64_t now_ms) { - int64_t age_ms = now_ms - n->last_activated; - if (age_ms <= 0) return 1.0; - double lambda = (n->temporal_decay_rate > 0.0) ? n->temporal_decay_rate - : ENGRAM_DECAY_LAMBDA; - double age_hours = (double)age_ms / 3600000.0; - double t_half = ENGRAM_T_HALF_HOURS * - (1.0 + log(1.0 + (double)n->activation_count)); - double factor = exp(-lambda * age_hours / t_half); - if (factor < ENGRAM_DECAY_FLOOR) factor = ENGRAM_DECAY_FLOOR; - return factor; + return cog_decay_factor(now_ms - n->last_activated, + (double)n->activation_count, + n->temporal_decay_rate); } /* Activation dampening: high activation_count nodes are "well-known" context @@ -9397,6 +10499,35 @@ static double engram_goal_bias(const EngramNode* n, const char* query) { * the exact O(n) argmax scan tops up any seed slot the ANN leaves unfilled. * Single-threaded, matching the adjacent query-embedding cache (no lock). * Returns NULL when no index is available → caller falls back to the O(n) scan. */ +/* ── VINDEX PUBLICATION BOUNDARY (2026-08-16) ──────────────────────────────── + * The index is DERIVED GEOMETRY: a projection of the store's embeddings. The + * store is append-only and superseding, so a reader must be able to project + * against geometry that does not move under it. + * + * The HNSW index is NOT itself append-only: vindex_insert rewires the neighbour + * lists of ALREADY-EXISTING elements and reallocs elems[]. So "extend" is a + * mutation of the whole structure, and a reader holding element pointers across + * one is unsafe no matter how pure search itself is (measured: TSan reports the + * elems[] race even after the visited set moved to the call frame). + * + * Hence a publication boundary rather than an ownership discipline: + * + * - eg_vindex_maintain() is the ONLY mutator of the five statics below. It + * takes _eg_vindex_rw EXCLUSIVELY, so it never runs beside a reader. + * - eg_vindex_view() hands back a `const VIndex*` with the boundary held for + * READ. N readers project concurrently; none can mutate, because search + * takes a const index and the compiler enforces it. + * + * A read path may DEMAND that a current snapshot exist — that is a request to + * the owner, not a mutation by the reader. What it may not do is mutate the + * geometry it is projecting against. eg_vindex_view/eg_vindex_maintain is + * exactly that split. + * + * Lock ordering: request-outer -> vindex -> store-inner. The vindex boundary is + * never held across a call that can re-enter eg_vindex_view/maintain (verified: + * the four read regions each acquire, search, release without nesting). */ +static pthread_rwlock_t _eg_vindex_rw = PTHREAD_RWLOCK_INITIALIZER; + static VIndex* _eg_vindex = NULL; static int32_t _eg_vindex_dim = 0; static int64_t _eg_vindex_built_nc = 0; /* g->node_count at last (re)build */ @@ -9416,8 +10547,10 @@ static int eg_vindex_seen_ensure(int64_t need) { return 0; } -static VIndex* eg_vindex_sync(EngramStore* g, int32_t dim) { - if (!g || dim <= 0) return _eg_vindex; +/* THE OWNER. The only function that mutates _eg_vindex* — must be called with + * _eg_vindex_rw held EXCLUSIVELY (see eg_vindex_maintain, the sole caller). */ +static void eg_vindex_publish_locked(EngramStore* g, int32_t dim) { + if (!g || dim <= 0) return; /* Drop a stale index: embedder dim changed, or the resident array shrank * (indices may have been reused/reordered → cached node_ids unsafe). */ if (_eg_vindex && (_eg_vindex_dim != dim || g->node_count < _eg_vindex_built_nc)) { @@ -9427,8 +10560,8 @@ static VIndex* eg_vindex_sync(EngramStore* g, int32_t dim) { } if (!_eg_vindex) { VIndex* idx = vindex_create((int)dim, 0, 0); - if (!idx) return NULL; - if (eg_vindex_seen_ensure(g->node_count)) { vindex_free(idx); return NULL; } + if (!idx) return; + if (eg_vindex_seen_ensure(g->node_count)) { vindex_free(idx); return; } for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; if (n->emb && n->emb_dim == dim && vindex_insert(idx, (uint64_t)i, n->emb) == 0) @@ -9452,8 +10585,62 @@ static VIndex* eg_vindex_sync(EngramStore* g, int32_t dim) { } _eg_vindex_built_nc = g->node_count; } +} + +/* Owner-mediated publish. Takes the boundary EXCLUSIVELY, so it can never run + * beside a reader. Cheap no-op when the published snapshot is already current. */ +static void eg_vindex_maintain(EngramStore* g, int32_t dim) { + if (!g || dim <= 0) return; + pthread_rwlock_wrlock(&_eg_vindex_rw); + eg_vindex_publish_locked(g, dim); + pthread_rwlock_unlock(&_eg_vindex_rw); +} + +/* READ SIDE. Returns the published snapshot as an IMMUTABLE view, with the + * boundary held for READ — the caller MUST pair every call with exactly one + * eg_vindex_view_release(), on every path including error returns. + * + * The returned pointer is `const`: a read path physically cannot call + * vindex_insert on it. That is the compile-time constraint, and it is why this + * replaces eg_vindex_sync rather than wrapping it. May return NULL (no index + * available -> caller falls back to the exact O(n) scan); the boundary is still + * held and still must be released. */ +static const VIndex* eg_vindex_view(EngramStore* g, int32_t dim) { + if (g && dim > 0) { + /* Fast path: snapshot already current, take it read-only and go. */ + pthread_rwlock_rdlock(&_eg_vindex_rw); + if (_eg_vindex && _eg_vindex_dim == dim && _eg_vindex_built_nc == g->node_count) + return _eg_vindex; + /* Stale or absent. Drop to no lock, ask the owner to publish, re-acquire. + * NEVER upgrade rdlock->wrlock in place: that self-deadlocks. */ + pthread_rwlock_unlock(&_eg_vindex_rw); + eg_vindex_maintain(g, dim); + } + pthread_rwlock_rdlock(&_eg_vindex_rw); return _eg_vindex; } +static void eg_vindex_view_release(void) { + pthread_rwlock_unlock(&_eg_vindex_rw); +} + +/* WRITE-SIDE MAINTENANCE HOOK. Call after an embedding becomes present on a + * resident ordinal. A node without an embedding cannot be in a vector index at + * all, so embedding-assignment — not node append — is the event that owns index + * membership. Cheap: one O(log n) HNSW insert, no O(node_count) presence scan. + * A no-op before the first publish (the cold build picks the node up) and on a + * dim mismatch. */ +static void eg_vindex_note_embedded(EngramStore* g, int64_t ordinal) { + if (!g || ordinal < 0 || ordinal >= g->node_count) return; + EngramNode* n = &g->nodes[ordinal]; + if (!n->emb || n->emb_dim <= 0) return; + pthread_rwlock_wrlock(&_eg_vindex_rw); + if (_eg_vindex && _eg_vindex_dim == n->emb_dim && + eg_vindex_seen_ensure(g->node_count) == 0 && !_eg_vindex_seen[ordinal]) { + if (vindex_insert(_eg_vindex, (uint64_t)ordinal, n->emb) == 0) + _eg_vindex_seen[ordinal] = 1; + } + pthread_rwlock_unlock(&_eg_vindex_rw); +} /* ── M9 GEOMETRY PRIMING (ENGRAM_GEOMETRY_PRIMING, default OFF) ────────────── * Opt-in wiring of the centered relational-neighborhood geometry (engram_geometry.c) @@ -9560,7 +10747,9 @@ static int64_t engram_activate_beam(void) { v = d; return v; } -el_val_t engram_activate(el_val_t query, el_val_t depth) { +/* Core activation. Callers must hold the engram request lock — reached only via + * the engram_activate() wrapper below, which self-guards (see eg_guard_enter). */ +static el_val_t engram_activate_inner(el_val_t query, el_val_t depth) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); int64_t max_depth = (int64_t)depth; if (max_depth <= 0) max_depth = 2; @@ -9601,6 +10790,12 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { float* v = eg_embed_fetch(n->content, &d); if (!v) break; /* embedder down / breaker open — stop this call */ n->emb = v; n->emb_dim = d; + /* Write-side index maintenance: an embedding just became present on + * ordinal i, so the index's owner publishes it now. This is what + * retires the "STALENESS (honest tradeoff)" note above — a lazily + * embedded OLDER node no longer waits for a full rebuild to become + * visible to route_nearest / autoconnect. */ + eg_vindex_note_embedded(g, i); backfilled++; } } @@ -9799,7 +10994,9 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { * same budget as the exact scan's retry `guard` — so dedup/threshold * rejects still leave enough distinct seeds. */ { - VIndex* vx = eg_vindex_sync(g, q_dim); + /* Immutable view: the boundary is held for READ across the whole + * search + harvest, and released at the end of this block. */ + const VIndex* vx = eg_vindex_view(g, q_dim); if (vx && (int64_t)vindex_size(vx) >= ENGRAM_EMBED_SEED_K) { const float* seed_qv = e_eff ? e_eff : q_emb; int kreq = ENGRAM_EMBED_SEED_K * 8; @@ -9843,6 +11040,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { } free(aid); free(ad); } + eg_vindex_view_release(); } /* Exact O(n) argmax fallback / top-up (pre-M8 selection, verbatim). @@ -9929,9 +11127,11 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { char** vids = malloc((size_t)g->node_count * sizeof(char*)); if (gmean && vids) { for (int64_t i = 0; i < g->node_count; i++) vids[i] = g->nodes[i].id; + const VIndex* gvx = eg_vindex_view(g, q_dim); geo = engram_geometry_descriptor( - g_engram_store, _eg_vindex, vids, (int)g->node_count, + g_engram_store, gvx, vids, (int)g->node_count, seed_ids, (size_t)nsel, NULL, gmean); + eg_vindex_view_release(); } free(vids); if (geo && geo->n_members > 0) { @@ -11209,6 +12409,15 @@ static void engram_emit_node_json(JsonBuf* b, const EngramNode* n, int include_e snprintf(tmp, sizeof(tmp), ",\"wm_anchor\":%g", n->wm_anchor); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"base_level\":%g", engram_bll_base_level(n, engram_now_ms())); jb_puts(b, tmp); + /* GEOMETRY VISIBILITY (2026-08-16 self-review): the node document never + * said whether the node carried a vector, so a read-back could not tell + * "has geometry" from "text only". Not cosmetic — it is exactly how a + * real ingest drop and a mere reporting gap became indistinguishable, + * and I misdiagnosed one as the other for an hour. Always emit the width + * and the boolean; the vector itself stays behind include_emb since it + * is large and most callers do not want it inline. */ + snprintf(tmp, sizeof(tmp), ",\"emb_dim\":%d,\"embedded\":%s", + (int)n->emb_dim, (n->emb && n->emb_dim > 0) ? "true" : "false"); jb_puts(b, tmp); /* Base-level access history: chronological (oldest→newest) compact * string. Loaders replay it through engram_bll_record_access; absent * field = empty ring (optimized-form fallback). (2026-07-22) */ @@ -13072,13 +14281,16 @@ static int eg_knn_for_node(EngramStore* g, int64_t self, int want, uint64_t* out if(self < 0 || self >= g->node_count) return 0; EngramNode* n = &g->nodes[self]; if(!n->emb || n->emb_dim <= 0) return 0; - VIndex* vx = eg_vindex_sync(g, n->emb_dim); - if(!vx) return 0; + /* Immutable view held for READ across the search; the harvest below reads + * only g->nodes, so the boundary is released as soon as the search returns. */ + const VIndex* vx = eg_vindex_view(g, n->emb_dim); + if(!vx){ eg_vindex_view_release(); return 0; } int K = want + 8; uint64_t* ids = (uint64_t*)malloc(sizeof(uint64_t)*(size_t)K); float* dist = (float*)malloc(sizeof(float)*(size_t)K); - if(!ids || !dist){ free(ids); free(dist); return 0; } + if(!ids || !dist){ eg_vindex_view_release(); free(ids); free(dist); return 0; } int m = vindex_search(vx, n->emb, K, 0, ids, dist); + eg_vindex_view_release(); int c = 0; for(int j=0; jnodes[self]; if((!n->emb || n->emb_dim <= 0) && n->content && eg_embed_eligible(n)){ int32_t d = 0; float* v = eg_embed_fetch(n->content, &d); - if(v && d > 0){ n->emb = v; n->emb_dim = d; if(engram_store_enabled()) eg_store_put_node(n); } + if(v && d > 0){ n->emb = v; n->emb_dim = d; if(engram_store_enabled()) eg_store_put_node(n); + eg_vindex_note_embedded(g, self); } else free(v); } if(!n->emb || n->emb_dim <= 0){ jb_puts(&b, "{\"connected\":0,\"reason\":\"unembedded\"}"); return el_wrap_str(b.buf); } @@ -13186,7 +14399,67 @@ el_val_t engram_ise_log_append(el_val_t content_v){ } } fputs("\"}\n", f); + /* RETENTION (2026-08-17 self-review). The on-graph ISE branch in + * server.el calls engram_prune_telemetry(48h) on every insert, but that + * branch is DEAD in production: ENGRAM_ISE_OFFGRAPH=1 is the live + * setting, so every state event lands here instead — and this path had + * no retention of any kind. Measured: 17.1 MB / 14,305 events over 3.56 + * days = 4.81 MB/day, growing without bound (~1.76 GB/year). The graph + * got its telemetry-growth fix on 2026-07-16; moving telemetry off-graph + * moved the leak rather than closing it. + * + * Byte-bounded rather than time-bounded on purpose: this is a flat + * append-only file with no index, so size is the property that actually + * has to be bounded, and a byte check is O(1) against the handle we + * already hold (ftell) instead of an O(file) timestamp scan per append. + * At the measured rate the 64 MB default retains ~13 days — comfortably + * more history than the 48h the on-graph path kept. + * + * Compaction keeps the TAIL, never the head: engram_dreams_json reads + * the last ~2 MB of this file for dream-recall, so the recent end is the + * end that has a reader. KEEP is held well above that 2 MB window so + * recall is never truncated by a rotation. The honesty rail is + * preserved exactly as before — rotated-out remains "I don't remember", + * never a synthesized dream; this only makes the forgetting bounded and + * explicit instead of deferred forever. */ + long pos = ftell(f); fclose(f); + { + long maxb = 64L*1024L*1024L; + long keepb = 16L*1024L*1024L; + const char* mv = getenv("ENGRAM_ISE_LOG_MAX_BYTES"); + if (mv && *mv) { long v = atol(mv); if (v > 0) maxb = v; } + if (keepb > maxb/2) keepb = maxb/2; + if (pos > 0 && pos > maxb) { + FILE* rf = fopen(path, "rb"); + if (rf) { + if (fseek(rf, pos - keepb, SEEK_SET) == 0) { + char* buf = (char*)malloc((size_t)keepb + 1); + if (buf) { + size_t rd = fread(buf, 1, (size_t)keepb, rf); + buf[rd] = 0; + /* Resume at the first LINE boundary so the tail never + * begins with a half-written JSON record. */ + char* start = memchr(buf, '\n', rd); + start = start ? start + 1 : buf; + size_t keep_n = rd - (size_t)(start - buf); + char tmp[4096]; + snprintf(tmp, sizeof tmp, "%s/state-events.jsonl.tmp", dir); + FILE* wf = fopen(tmp, "wb"); + if (wf) { + int ok = (fwrite(start, 1, keep_n, wf) == keep_n); + fclose(wf); + /* Only replace the live log if the tail was written + * in full — a short write must not destroy history. */ + if (ok) rename(tmp, path); else remove(tmp); + } + free(buf); + } + } + fclose(rf); + } + } + } return EL_INT(1); } @@ -13244,8 +14517,10 @@ static GeoDescriptor* eg_geo_build_desc(const char* csv) { char** vids = malloc((size_t)g->node_count * sizeof(char*)); if (gmean && vids) { for (int64_t i = 0; i < g->node_count; i++) vids[i] = g->nodes[i].id; - geo = engram_geometry_descriptor(g_engram_store, _eg_vindex, vids, (int)g->node_count, + const VIndex* gvx = eg_vindex_view(g, dim); + geo = engram_geometry_descriptor(g_engram_store, gvx, vids, (int)g->node_count, (const char* const*)ids, (size_t)ns, NULL, gmean); + eg_vindex_view_release(); } free(vids); for (int i = 0; i < ns; i++) free(ids[i]); @@ -13282,12 +14557,16 @@ el_val_t engram_geo_reify_run_json(void){ int32_t dim = 0; for(int64_t i = 0; i < g->node_count && dim == 0; i++) if(g->nodes[i].emb && g->nodes[i].emb_dim > 0) dim = g->nodes[i].emb_dim; - VIndex* vx = (dim > 0) ? eg_vindex_sync(g, dim) : NULL; char** vids = malloc((size_t)g->node_count * sizeof(char*)); if(!vids) return eg_geo_err("reify oom"); for(int64_t i = 0; i < g->node_count; i++) vids[i] = g->nodes[i].id; + /* Held for READ across the whole reify pass: it only searches the index. + * (The multi-second SELF-reify beat below builds a PRIVATE index instead and + * never touches this boundary at all.) */ + const VIndex* vx = eg_vindex_view(g, dim); int persisted = engram_geo_reify_store(g_engram_store, vx, vids, (int)g->node_count, NULL); + eg_vindex_view_release(); free(vids); int nested = 0; if(persisted >= 0){ @@ -13597,12 +14876,129 @@ static int eg_cog_is_keystone_seeds(const char* csv) { el_val_t engram_think_json(el_val_t seeds, el_val_t faculty) { GeoDescriptor* g = eg_geo_build_desc(EL_CSTR(seeds)); if (!g) return eg_geo_err("geometry unavailable"); - CogStance st; cog_stance_init(&st, NULL, EL_CSTR(faculty), g->hub_id, NULL, g); + /* RESUME THE LEARNED STANCE (2026-08-16 self-review). This built a NEUTRAL + * stance every call — all axis_gain 1.0, bias_dir NULL, reliability 0.5 — + * and never loaded the one the correspondence-beat had been persisting. + * + * That mattered because the faculty enters engram_think ONLY through the + * stance: `gain = stance->axis_gain[k]` warps the per-axis extents, and + * `stance->bias_dir` seeds the steering direction. cog_stance_init stores + * the faculty NAME but nothing reads it. So with a neutral stance, + * reason / abduce / induce / plan / analogize are the same function with + * different labels — measured, byte-identical output across all five — + * and `confidence` is pinned to the 0.5 uninformed prior, because + * GeoGradient.confidence is just stance->reliability. + * + * The machinery already existed and only this call site ignored it: + * engram_correspondence_beat_json resumes via cog_stance_from_node and + * persists via cog_stance_to_node under the id "stance--". + * Every beat's calibration was being written and then thrown away on the + * next read. Same defect as the NULL anchor directly above: a neutral + * argument collapsing a capability to a constant. + * + * Resume the same id the beat writes, so learning compounds across beats + * and cold boot. Fall back to neutral only when no stance exists yet — + * which is a genuine uninformed prior, not a discarded informed one. */ + char sid[256]; + snprintf(sid, sizeof sid, "stance-%s-%s", + EL_CSTR(faculty) ? EL_CSTR(faculty) : "reason", + g->hub_id ? g->hub_id : "region"); + CogStance st; StoreNode prev; int resumed = 0; + if (g_engram_store && store_get_node(g_engram_store, sid, &prev) == 1) { + if (cog_stance_from_node(&prev, &st) == 0) resumed = 1; + store_node_free(&prev); + } + if (!resumed) cog_stance_init(&st, sid, EL_CSTR(faculty), g->hub_id, NULL, g); + else { free(st.id); st.id = strdup(sid); } GeoGradient grad; - if (engram_think(g, NULL, &st, &grad) != 0) { cog_stance_free(&st); engram_geo_free(g); return eg_geo_err("think failed"); } + + /* ANCHOR THE READ (2026-08-16 self-review). This passed NULL, and NULL is + * not "no opinion" — engram_think re-origins at `anchor ? anchor : + * region->centroid`, so NULL means "read from the centroid", and the + * centroid is the ONE point where the gradient is zero by construction: + * r = x - centroid = 0, so every axis projection is 0, grad is 0, and + * direction takes the "at rest" branch. Measured consequence: EVERY + * faculty — reason, abduce, induce, plan, analogize — returned an + * identical null result, differing only in its label: + * {"direction":[0,0,...],"spread":0,"magnitude":1,"confidence":0.5} + * magnitude 1 is membership evaluated at the centroid, spread 0 is its + * distance to itself, and confidence 0.5 is the stance fallback. The + * geometry was never the problem — /api/drift computes real values + * (centroid_sep 0.104, core_disp 0.045) over the very same 87 members. + * Neuron could not think because the read was always taken from the + * region's own centre. + * + * The seeds choose WHICH region; they must also supply the VANTAGE it is + * read from. Anchor at the first resolvable embedded seed — the same seed + * eg_geo_build_desc infers `dim` from, so the two never disagree. A single + * seed still yields a real gradient because the descriptor expands to the + * seed's neighbourhood (87 members for the self anchor), so the seed's own + * position is distinct from the neighbourhood centroid. + * + * COPY the vector, never borrow it: g->nodes is realloc'd in place on + * append, so a borrowed EngramNode* is a dangling pointer across any + * concurrent write. 768 floats is 3 KB. */ + float* anchor = NULL; + { + EngramStore* eg = engram_get(); + const char* csv = EL_CSTR(seeds); + if (eg && csv) { + const char* p = csv; + while (*p && !anchor) { + while (*p == ' ' || *p == ',') p++; + const char* s = p; + while (*p && *p != ',') p++; + const char* e = p; while (e > s && e[-1] == ' ') e--; + if (e > s) { + char* id = strndup(s, (size_t)(e - s)); + if (id) { + int64_t idx = engram_find_node_index(id); + if (idx >= 0 && idx < eg->node_count) { + EngramNode* n = &eg->nodes[idx]; + if (n->emb && n->emb_dim == g->dim) { + /* NORMALIZE INTO THE DESCRIPTOR'S FRAME (2026-08-16). + * This copied the RAW vector, but engram_geometry + * builds every descriptor over L2-normalized member + * embeddings, so the anchor was being fitted against + * an ellipsoid at a radius it was never fitted over. + * Measured on the self region: magnitude 0.00283443 + * raw vs 0.521837 in-frame on the same pair — a 180x + * error, and the reason every fit score sat three + * decimal places below the 0.5 floors that gate on + * them. The frame contract is stated in + * engram_verify.h; this call site did not honour it. + * Idempotent when the stored vector is already unit. */ + anchor = malloc(sizeof(float) * (size_t)g->dim); + if (anchor) { + double s2 = 0; + for (int i = 0; i < g->dim; i++) + s2 += (double)n->emb[i] * (double)n->emb[i]; + double nn = sqrt(s2); + if (nn > 1e-12) + for (int i = 0; i < g->dim; i++) + anchor[i] = (float)((double)n->emb[i] / nn); + else + memcpy(anchor, n->emb, sizeof(float) * (size_t)g->dim); + } + } + } + free(id); + } + } + } + } + } + + if (engram_think(g, anchor, &st, &grad) != 0) { free(anchor); cog_stance_free(&st); engram_geo_free(g); return eg_geo_err("think failed"); } + free(anchor); JsonBuf b; jb_init(&b); char t[256]; - snprintf(t, sizeof t, "{\"faculty\":\"%s\",\"n_support\":%d,\"magnitude\":%.6g,\"spread\":%.6g,\"confidence\":%.6g,\"dim\":%d", - EL_CSTR(faculty), grad.n_support, grad.magnitude, grad.spread, grad.confidence, grad.dim); + /* stance_resumed distinguishes an INFORMED read from an uninformed one. + * Without it, confidence 0.5 from a learned-but-unreliable stance and + * confidence 0.5 from "no stance exists" are indistinguishable — the same + * reporting gap that let the NULL anchor and the neutral stance hide. */ + snprintf(t, sizeof t, "{\"faculty\":\"%s\",\"n_support\":%d,\"magnitude\":%.6g,\"spread\":%.6g,\"confidence\":%.6g,\"stance_resumed\":%s,\"dim\":%d", + EL_CSTR(faculty), grad.n_support, grad.magnitude, grad.spread, grad.confidence, + resumed ? "true" : "false", grad.dim); jb_puts(&b, t); int emit = grad.dim < 8 ? grad.dim : 8; jb_puts(&b, ",\"direction\":"); eg_geo_emit_vec(&b, grad.direction, emit); @@ -13611,43 +15007,522 @@ el_val_t engram_think_json(el_val_t seeds, el_val_t faculty) { return el_wrap_str(b.buf); } -/* engram_ground_json(claim_csv, evidence_csv, for_whom) — grounding as a RELATION. - * Turns the DORMANT verifier inward for real: verifies the claim region's centroid - * against the evidence region, then writes a grounded-by edge (weight = grounding, - * grounded-for-whom). Additive. */ +/* ═══════════════════════════════════════════════════════════════════════════ + * §7 WIRING — GROUNDING IS THE EDGE'S WEIGHT, AND THE WEIGHT IS A VECTOR + * (2026-08-16; spec correspondence-and-censorship.md @ 2b7e4ba.) + * + * What was here minted a `grounded-by` edge on every call and returned a float. + * Every part of that was wrong, and #147 only fixed the endpoints: + * - grounding is a property OF a relation, not a relation BETWEEN nodes, so + * there was nothing for a new edge to carry; + * - the call was a READ that wrote — the eg_vindex_sync defect; + * - one scalar cannot separate "true and meaningful" from "true and misapplied", + * nor "no support" from "actively contradicts". + * + * ground() is now pure: it reads the vector the RELATION already carries, decays + * it analytically to now on the runtime's one decay model, computes what the + * current geometry would say, and reports whether that move is consequential — + * without recording it. Recording is a separate, explicitly named write. + * ═══════════════════════════════════════════════════════════════════════════ */ + +/* The values reference: THIRTEEN regions, discovered from the graph rather than + * hardcoded, so a fourteenth value is picked up without a code change. They are + * the nodes the values root `contains`. Measured on the live store: exactly 13, + * pairwise centroid cosine min 0.1525 / mean 0.5199 / max 0.9278 — not one region. */ +#define EG_VALUES_ROOT_DEFAULT "kn-5b606390-a52d-4ca2-8e0e-eba141d13440" +#define EG_MAX_VALUES 32 +typedef struct { + int n; + char* ids[EG_MAX_VALUES]; + GeoDescriptor* g[EG_MAX_VALUES]; +} EgValueRef; + +static void eg_values_release(EgValueRef* v) { + if (!v) return; + for (int i = 0; i < v->n; i++) { free(v->ids[i]); if (v->g[i]) engram_geo_free(v->g[i]); } + v->n = 0; +} +static int eg_values_build(EgValueRef* out) { + if (!out || !g_engram_store) return -1; + memset(out, 0, sizeof *out); + const char* root = getenv("ENGRAM_VALUES_ROOT"); + if (!root || !*root) root = EG_VALUES_ROOT_DEFAULT; + StoreEdge* edges = NULL; size_t ne = 0; + if (store_get_edges_from(g_engram_store, root, &edges, &ne) < 0) return -1; + for (size_t i = 0; i < ne && out->n < EG_MAX_VALUES; i++) { + if (edges[i].tombstoned) continue; + if (!edges[i].relation || strcmp(edges[i].relation, "contains") != 0) continue; + if (!edges[i].to_id) continue; + GeoDescriptor* g = eg_geo_build_desc(edges[i].to_id); + if (!g) continue; + out->ids[out->n] = strdup(edges[i].to_id); + out->g[out->n] = g; + out->n++; + } + store_edges_free(edges, ne); + return out->n; +} + +/* THE REFERENCE FRAME IS BUILT ONCE AND HELD. Two reasons, and the second is the + * real one: + * - cost: thirteen descriptors over the whole node vector per call made a + * single /api/ground take tens of seconds; + * - correctness: a reference frame that is rebuilt on every read moves under + * the measurement, which is the defect the whole spec is about. Holding it + * for the process lifetime is the closest this layer can come to "the frame + * updates when it is not being used to act" without owning that decision — + * which belongs to the dreamer, not here. ENGRAM_VALUES_NOCACHE=1 forces a + * rebuild per call for tests that deliberately move a value node. */ +static EgValueRef _eg_values_cache; +static int _eg_values_cached = 0; +static const EgValueRef* eg_values_ref(void) { + const char* nc = getenv("ENGRAM_VALUES_NOCACHE"); + if (nc && nc[0] && nc[0] != '0') { + if (_eg_values_cached) { eg_values_release(&_eg_values_cache); _eg_values_cached = 0; } + } + if (!_eg_values_cached) { + if (eg_values_build(&_eg_values_cache) <= 0) return NULL; + _eg_values_cached = 1; + } + return &_eg_values_cache; +} + +/* Copy a node's embedding INTO THE DESCRIPTOR'S FRAME (never borrow — g->nodes is + * realloc'd on append, so a borrowed EngramNode* dangles across any concurrent write). + * + * THE FRAME CONTRACT IS LOAD-BEARING AND WAS BEING VIOLATED. engram_verify.h states + * it plainly: the claim point and the descriptor must share the same frame. + * engram_geometry builds every descriptor over L2-NORMALIZED member embeddings — + * `centroid` is the mean of normcopy()'d vectors and the principal axes are computed + * against that — while the raw `n->emb` in the resident store is not necessarily unit. + * Fitting a raw vector against a unit-frame descriptor puts the point at a radius the + * ellipsoid was never fitted over, so the distance is dominated by the norm mismatch + * and every fit score collapses toward zero. Normalizing here is idempotent when the + * stored vector is already unit, so it can only help. */ +static float* eg_node_emb_copy(const char* id, int dim) { + EngramStore* eg = engram_get(); + if (!eg || !id || dim <= 0) return NULL; + int64_t idx = engram_find_node_index(id); + if (idx < 0 || idx >= eg->node_count) return NULL; + EngramNode* n = &eg->nodes[idx]; + if (!n->emb || n->emb_dim != dim) return NULL; + float* p = malloc(sizeof(float) * (size_t)dim); + if (!p) return NULL; + double s = 0; + for (int i = 0; i < dim; i++) s += (double)n->emb[i] * (double)n->emb[i]; + double nn = sqrt(s); + if (nn > 1e-12) for (int i = 0; i < dim; i++) p[i] = (float)((double)n->emb[i] / nn); + else memcpy(p, n->emb, sizeof(float) * (size_t)dim); + return p; +} +/* Salience of a relation = the salience of its endpoints, reusing the fields the + * runtime already keeps (node salience and working-memory weight). Consolidation + * is gated by salience — that is why you remember the argument and not the + * commute — and this deliberately reads existing state rather than introducing a + * threshold of its own. */ +static double eg_node_salience(const char* id) { + EngramStore* eg = engram_get(); + if (!eg || !id) return 0.0; + int64_t idx = engram_find_node_index(id); + if (idx < 0 || idx >= eg->node_count) return 0.0; + EngramNode* n = &eg->nodes[idx]; + double s = n->salience; + if (n->working_memory_weight > s) s = n->working_memory_weight; + if (n->background_activation > s) s = n->background_activation; + return s; +} +static double eg_vdot(const float* a, const float* b, int dim) { + double s = 0; for (int i = 0; i < dim; i++) s += (double)a[i] * (double)b[i]; return s; +} +/* Signed projection of a unit gradient onto unit(target − x): positive means the + * descent direction still points toward the target. Frame-independent, so it + * stays comparable across versions even if the region's principal axes rotate — + * which is what makes the DIRECTION_REVERSAL test meaningful over time. */ +static double eg_proj_toward(const float* dir, const float* x, const float* target, int dim) { + double s = 0, n2 = 0; + for (int i = 0; i < dim; i++) { double d = (double)target[i] - (double)x[i]; s += (double)dir[i] * d; n2 += d * d; } + double n = sqrt(n2); + return n > 1e-12 ? s / n : 0.0; +} + +/* RELATIONAL grounding of a point: the MIN fit over the thirteen value regions, + * and the NAME of the value that binds. Min, not mean, because a mean lets strong + * agreement with twelve values mask a violation of the thirteenth — which is the + * mechanism of rationalization, not a scoring detail. */ +static int eg_relational_read(const float* x, const EgValueRef* V, + double* out_score, int* out_idx, float** out_dir) { + if (!x || !V || V->n <= 0) return -1; + double worst = 2.0; int wi = -1; + for (int i = 0; i < V->n; i++) { + GeoFit f; + if (cog_warped_fit(V->g[i], x, NULL, &f) != 0) continue; + if (f.score < worst) { worst = f.score; wi = i; } + } + if (wi < 0) return -1; + if (out_score) *out_score = worst; + if (out_idx) *out_idx = wi; + if (out_dir) { + GeoGradient gr; + if (engram_think(V->g[wi], x, NULL, &gr) == 0) { + *out_dir = gr.direction; gr.direction = NULL; /* take ownership */ + engram_gradient_free(&gr); + } else *out_dir = NULL; + } + return 0; +} + +/* Find the live relation between two nodes, in either orientation, and return + * the NEWEST recorded version of it. Returns 1 on hit. */ +static int eg_find_relation(const char* a, const char* b, StoreEdge* out, char* base_id, size_t cap) { + if (!g_engram_store || !a || !b) return 0; + for (int dir = 0; dir < 2; dir++) { + const char* from = dir == 0 ? a : b; + const char* to = dir == 0 ? b : a; + StoreEdge* edges = NULL; size_t ne = 0; + if (store_get_edges_from(g_engram_store, from, &edges, &ne) < 0) continue; + for (size_t i = 0; i < ne; i++) { + if (edges[i].tombstoned) continue; + if (!edges[i].to_id || strcmp(edges[i].to_id, to) != 0) continue; + if (edges[i].id && strchr(edges[i].id, '#')) continue; /* a version, not a root */ + snprintf(base_id, cap, "%s", edges[i].id ? edges[i].id : ""); + store_edges_free(edges, ne); + if (cog_grounding_head(g_engram_store, base_id, out, 64) >= 0) return 1; + return 0; + } + store_edges_free(edges, ne); + } + return 0; +} + +/* The two-axis observation of one relation from the CURRENT geometry. Pure. + * factual = how well the far endpoint sits in the near endpoint's neighbourhood + * — a real correspondence measurement of the relation itself; + * relational = min fit over the thirteen value regions, with the binding name; + * cos_angle = the cosine between the two full-dimensional gradients. */ +typedef struct { + int ok; + double factual, relational, cos_angle, fac_proj, rel_proj; + char binding[128]; +} EgObservation; + +static int eg_observe_relation(const char* from_id, const char* to_id, + const EgValueRef* V, EgObservation* out) { + memset(out, 0, sizeof *out); + GeoDescriptor* R = eg_geo_build_desc(from_id); + if (!R) return -1; + int dim = R->dim; + float* x = eg_node_emb_copy(to_id, dim); + if (!x) { engram_geo_free(R); return -1; } + + GeoFit f; + if (cog_warped_fit(R, x, NULL, &f) != 0) { free(x); engram_geo_free(R); return -1; } + out->factual = f.score; + + GeoGradient fg; + float* fac_dir = NULL; + if (engram_think(R, x, NULL, &fg) == 0) { fac_dir = fg.direction; fg.direction = NULL; engram_gradient_free(&fg); } + + int vi = -1; float* rel_dir = NULL; + if (eg_relational_read(x, V, &out->relational, &vi, &rel_dir) == 0 && vi >= 0) + snprintf(out->binding, sizeof out->binding, "%s", V->ids[vi]); + + if (fac_dir && R->centroid) out->fac_proj = eg_proj_toward(fac_dir, x, R->centroid, dim); + if (rel_dir && vi >= 0 && V->g[vi]->centroid) out->rel_proj = eg_proj_toward(rel_dir, x, V->g[vi]->centroid, dim); + if (fac_dir && rel_dir) out->cos_angle = eg_vdot(fac_dir, rel_dir, dim); + + free(fac_dir); free(rel_dir); free(x); engram_geo_free(R); + out->ok = 1; + return 0; +} + +/* Fold an observation into the vector as it would stand now. The accrual is a + * running MEAN over reinforcements — the same form the beat's calibration + * already uses (brier_sum / n_trials) — so no rate constant is introduced. */ +static void eg_fold_observation(const CogGrounding* prev, const EgObservation* obs, + CogProvClass prov, double floor, double rel_floor, + int64_t now, CogGrounding* out) { + *out = *prev; + double n = prev->reinforcements; + out->factual = prev->present ? prev->factual + (obs->factual - prev->factual) / (n + 1.0) : obs->factual; + out->relational = prev->present ? prev->relational + (obs->relational - prev->relational) / (n + 1.0) : obs->relational; + out->fac_proj = obs->fac_proj; out->rel_proj = obs->rel_proj; out->cos_angle = obs->cos_angle; + out->agreement = obs->cos_angle > 0 ? 1 : (obs->cos_angle < 0 ? -1 : 0); + out->reinforcements = n + 1.0; + out->ts = now; + if (prov != COG_PROV_UNSET) out->prov = prov; + out->floor_at_record = floor; out->rel_floor_at_record = rel_floor; + snprintf(out->binding_value, sizeof out->binding_value, "%s", obs->binding); + /* DERIVED, recomputed at the instant — never carried over from prev. */ + out->age_ms = 0; out->decay = 1.0; + out->factual_now = out->factual; out->relational_now = out->relational; + out->associative_now = out->associative; + out->stale = 0; +} + +static void eg_emit_vector(JsonBuf* b, const char* key, const CogGrounding* g) { + char t[768]; + snprintf(t, sizeof t, + "\"%s\":{\"established\":%s," + "\"factual\":%.6g,\"relational\":%.6g,\"associative\":%.6g,\"polarity\":%.6g," + "\"provenance\":\"%s\",\"ts\":%lld,\"seq\":%lld,\"reinforcements\":%.6g," + "\"cos_angle\":%.6g,\"agreement\":%d,\"fac_proj\":%.6g,\"rel_proj\":%.6g," + "\"binding_value\":\"%s\"," + "\"derived\":{\"age_s\":%lld,\"decay\":%.6g,\"factual_now\":%.6g," + "\"relational_now\":%.6g,\"associative_now\":%.6g,\"stale\":%s}}", + key, g->present ? "true" : "false", + g->factual, g->relational, g->associative, g->polarity, + cog_prov_name(g->prov), (long long)g->ts, (long long)g->seq, g->reinforcements, + g->cos_angle, g->agreement, g->fac_proj, g->rel_proj, + g->binding_value[0] ? g->binding_value : "-", + (long long)(g->age_ms / 1000), g->decay, g->factual_now, + g->relational_now, g->associative_now, g->stale ? "true" : "false"); + jb_puts(b, t); +} + +/* engram_ground_json(claim, evidence, for_whom) — a READ. Never writes. */ el_val_t engram_ground_json(el_val_t claim, el_val_t evidence, el_val_t for_whom) { if (!g_engram_store) return eg_geo_err("store unavailable"); - GeoDescriptor* C = eg_geo_build_desc(EL_CSTR(claim)); - GeoDescriptor* E = eg_geo_build_desc(EL_CSTR(evidence)); - if (!C || !E) { if (C) engram_geo_free(C); if (E) engram_geo_free(E); return eg_geo_err("geometry unavailable"); } - const GeoDescriptor* ev[1] = { E }; - GeoGrounding gr; - int rc = engram_verify_grounding(C->centroid, C->dim, ev, 1, 1.0, 0.5, &gr); - double grounding = (rc == 0) ? gr.grounding : 0.0; - if (rc == 0) engram_verify_grounding_free(&gr); - const char* fw = EL_CSTR(for_whom); if (fw && !*fw) fw = NULL; - const char* cid = C->hub_id ? C->hub_id : EL_CSTR(claim); - const char* eid = E->hub_id ? E->hub_id : EL_CSTR(evidence); - int wr = cog_ground_edge(g_engram_store, cid, eid, grounding, fw); - JsonBuf b; jb_init(&b); char t[256]; - snprintf(t, sizeof t, "{\"relation\":\"grounded-by\",\"claim\":\"%s\",\"evidence\":\"%s\",\"for_whom\":\"%s\",\"grounding\":%.6g,\"written\":%s}", - cid, eid, fw ? fw : "-", grounding, wr == 0 ? "true" : "false"); + const char* cid = EL_CSTR(claim); + const char* eid = EL_CSTR(evidence); + const char* fw = EL_CSTR(for_whom); if (fw && !*fw) fw = NULL; + int64_t now = engram_now_ms(); + + StoreEdge cur; char base_id[192] = ""; + JsonBuf b; jb_init(&b); char t[768]; + + if (!eg_find_relation(cid, eid, &cur, base_id, sizeof base_id)) { + /* THE HONEST ANSWER. There is nothing to ground a claim "against" that is + * not already a relation. This previously minted one and scored it — and + * when both seeds fell in one region the score came back 1.0 with no + * evidence behind it. If the two are unrelated, say so and write nothing. */ + snprintf(t, sizeof t, + "{\"claim\":\"%s\",\"evidence\":\"%s\",\"for_whom\":\"%s\",\"related\":false," + "\"grounding\":null,\"written\":false," + "\"note\":\"no relation between these nodes; grounding is a property of a relation, not a score minted between nodes\"}", + cid ? cid : "", eid ? eid : "", fw ? fw : "-"); + jb_puts(&b, t); + return el_wrap_str(b.buf); + } + + CogGrounding rec; cog_grounding_parse(&cur, now, &rec); + + const EgValueRef* V = eg_values_ref(); + EgObservation obs; memset(&obs, 0, sizeof obs); + int nv = V ? V->n : 0; + if (nv > 0 && cur.from_id && cur.to_id) eg_observe_relation(cur.from_id, cur.to_id, V, &obs); + + double floor = 0.5, rel_floor = 0.5; + CogGrounding ng = rec; + if (obs.ok) eg_fold_observation(&rec, &obs, COG_PROV_UNSET, floor, rel_floor, now, &ng); + CogSignificance sig = obs.ok ? cog_grounding_significant(&rec, &ng, floor, rel_floor) + : COG_SIG_NONE; + double sal = cur.from_id && cur.to_id + ? (eg_node_salience(cur.from_id) > eg_node_salience(cur.to_id) + ? eg_node_salience(cur.from_id) : eg_node_salience(cur.to_id)) + : 0.0; + + CogTrajectory tr; memset(&tr, 0, sizeof tr); + cog_grounding_trajectory(g_engram_store, base_id, now, &tr); + + jb_putc(&b, '{'); + snprintf(t, sizeof t, + "\"claim\":\"%s\",\"evidence\":\"%s\",\"for_whom\":\"%s\",\"related\":true," + "\"relation\":\"%s\",\"edge\":\"%s\",\"edge_root\":\"%s\",\"from\":\"%s\",\"to\":\"%s\"," + "\"value_regions\":%d,\"aggregate\":\"min\",\"salience\":%.6g,", + cid ? cid : "", eid ? eid : "", fw ? fw : "-", + cur.relation ? cur.relation : "", cur.id ? cur.id : "", base_id, + cur.from_id ? cur.from_id : "", cur.to_id ? cur.to_id : "", nv > 0 ? nv : 0, sal); jb_puts(&b, t); - engram_geo_free(C); engram_geo_free(E); + eg_emit_vector(&b, "recorded", &rec); + jb_putc(&b, ','); + if (obs.ok) eg_emit_vector(&b, "observed", &ng); + else jb_puts(&b, "\"observed\":null"); + /* Volatility and drift are computed here and stored nowhere — the series + * exists only because nothing was destroyed. */ + snprintf(t, sizeof t, + ",\"trajectory\":{\"n_versions\":%d,\"factual_volatility\":%.6g," + "\"relational_volatility\":%.6g,\"factual_drift\":%.6g,\"relational_drift\":%.6g," + "\"stayed_true_became_wrong\":%s}" + ",\"would_record\":%s,\"significance\":\"%s\",\"inherent\":%s,\"written\":false}", + tr.n_versions, tr.factual_volatility, tr.relational_volatility, + tr.factual_drift, tr.relational_drift, + tr.stayed_true_became_wrong ? "true" : "false", + sig != COG_SIG_NONE ? "true" : "false", cog_significance_name(sig), + cog_significance_inherent(sig) ? "true" : "false"); + jb_puts(&b, t); + + store_edge_free(&cur); return el_wrap_str(b.buf); } -/* engram_assert_json(claim_id, for_whom, floor) — the honesty floor as a QUERY at - * ASSERTION time only (holding is never gated). Reads the claim's grounded-by - * edges (for the observer) and returns whether assertion is permitted. */ -el_val_t engram_assert_json(el_val_t claim_id, el_val_t for_whom, el_val_t floor) { +/* engram_ground_record_json — the WRITE half, named as one. Recomputes the + * vector, applies the consolidation gate (salience + per-dimension consequence), + * and on significance supersedes the EDGE, versioning the WHOLE vector jointly. + * The predecessor is never touched. + * + * There is no provenance predicate here and nothing takes its place. An earlier + * pass built one — a bounded traversal over a grounding-chain graph, refusing + * evidence downstream of the region being calibrated. It is deleted. Circularity + * of the reference frame is TEMPORAL, not topological: you cannot recalibrate the + * ruler while measuring with it, so that update happens when the frame is not + * being used to act, which is a fact about engagement and belongs to the dreamer. + * The measurement that settles it: reachability from the self region reaches + * 89.2% of the live graph, so any topological predicate marks nearly all evidence + * tainted and degenerates into the total block censorship started as. */ +el_val_t engram_ground_record_json(el_val_t claim, el_val_t evidence, + el_val_t provenance, el_val_t floor_v) { + if (!g_engram_store) return eg_geo_err("store unavailable"); + const char* cid = EL_CSTR(claim); + const char* eid = EL_CSTR(evidence); + double floor = atof(EL_CSTR(floor_v)); if (!(floor > 0)) floor = 0.5; + double rel_floor = floor; + CogProvClass prov = cog_prov_parse(EL_CSTR(provenance)); + int64_t now = engram_now_ms(); + + StoreEdge cur; char base_id[192] = ""; + JsonBuf b; jb_init(&b); char t[768]; + if (!eg_find_relation(cid, eid, &cur, base_id, sizeof base_id)) { + snprintf(t, sizeof t, "{\"claim\":\"%s\",\"evidence\":\"%s\",\"related\":false,\"written\":false}", + cid ? cid : "", eid ? eid : ""); + jb_puts(&b, t); return el_wrap_str(b.buf); + } + + CogGrounding rec; cog_grounding_parse(&cur, now, &rec); + const EgValueRef* V = eg_values_ref(); + EgObservation obs; memset(&obs, 0, sizeof obs); + int nv = V ? V->n : 0; + if (nv <= 0 || !cur.from_id || !cur.to_id || + eg_observe_relation(cur.from_id, cur.to_id, V, &obs) != 0 || !obs.ok) { + store_edge_free(&cur); + return eg_geo_err("geometry unavailable for this relation"); + } + + CogGrounding ng; + eg_fold_observation(&rec, &obs, prov, floor, rel_floor, now, &ng); + + CogSignificance sig = cog_grounding_significant(&rec, &ng, floor, rel_floor); + /* CONSOLIDATION GATE. Significance says the move would change a decision; + * salience says it is worth making durable. The two INHERENT moves — a + * polarity sign flip and a provenance class change — bypass salience because + * they are discrete changes of state rather than drift. */ + double sal = eg_node_salience(cur.from_id) > eg_node_salience(cur.to_id) + ? eg_node_salience(cur.from_id) : eg_node_salience(cur.to_id); + int salient = (sal > 0.0); + int consolidate = (sig != COG_SIG_NONE) && (cog_significance_inherent(sig) || salient); + + char written_id[224] = ""; + int seq = -1; + if (consolidate) seq = cog_grounding_record(g_engram_store, &cur, &ng, written_id, sizeof written_id); + + jb_putc(&b, '{'); + snprintf(t, sizeof t, + "\"claim\":\"%s\",\"evidence\":\"%s\",\"edge\":\"%s\",\"edge_root\":\"%s\"," + "\"value_regions\":%d,\"aggregate\":\"min\",\"floor\":%.4g,\"rel_floor\":%.4g," + "\"salience\":%.6g,\"salient\":%s,", + cid ? cid : "", eid ? eid : "", cur.id ? cur.id : "", base_id, + nv, floor, rel_floor, sal, salient ? "true" : "false"); + jb_puts(&b, t); + eg_emit_vector(&b, "previous", &rec); jb_putc(&b, ','); + eg_emit_vector(&b, "observed", &ng); + snprintf(t, sizeof t, + ",\"significance\":\"%s\",\"inherent\":%s,\"consolidated\":%s," + "\"written\":%s,\"version\":%d,\"version_id\":\"%s\"}", + cog_significance_name(sig), cog_significance_inherent(sig) ? "true" : "false", + consolidate ? "true" : "false", + (consolidate && seq > 0) ? "true" : "false", seq > 0 ? seq : 0, written_id); + jb_puts(&b, t); + + store_edge_free(&cur); + return el_wrap_str(b.buf); +} + +/* engram_ground_trajectory_json(claim, evidence) — the supersession chain read as + * a TIME SERIES OF VECTORS. Not only what the grounding is but which way it has + * been moving and how fast — a derivative obtained for free from immutability, + * because the points were never destroyed. */ +el_val_t engram_ground_trajectory_json(el_val_t claim, el_val_t evidence) { + if (!g_engram_store) return eg_geo_err("store unavailable"); + StoreEdge cur; char base_id[192] = ""; + if (!eg_find_relation(EL_CSTR(claim), EL_CSTR(evidence), &cur, base_id, sizeof base_id)) + return eg_geo_err("no relation between these nodes"); + store_edge_free(&cur); + int64_t now = engram_now_ms(); + JsonBuf b; jb_init(&b); char t[768]; + jb_puts(&b, "{\"edge_root\":\""); jb_puts(&b, base_id); jb_puts(&b, "\",\"versions\":["); + int emitted = 0; + for (int v = 0; v <= 64; v++) { + char vid[224]; + if (v == 0) snprintf(vid, sizeof vid, "%s", base_id); + else snprintf(vid, sizeof vid, "%s#%d", base_id, v); + StoreEdge e; + if (store_get_edge(g_engram_store, vid, &e) != 1) { if (v) break; else continue; } + CogGrounding g; cog_grounding_parse(&e, now, &g); + if (emitted) jb_putc(&b, ','); + snprintf(t, sizeof t, + "{\"version\":%d,\"id\":\"%s\",\"established\":%s,\"ts\":%lld," + "\"factual\":%.6g,\"relational\":%.6g,\"associative\":%.6g,\"polarity\":%.6g," + "\"provenance\":\"%s\",\"cos_angle\":%.6g,\"agreement\":%d," + "\"binding_value\":\"%s\",\"prev\":\"%s\"," + "\"derived\":{\"age_s\":%lld,\"decay\":%.6g,\"factual_now\":%.6g,\"stale\":%s}}", + v, vid, g.present ? "true" : "false", (long long)g.ts, + g.factual, g.relational, g.associative, g.polarity, + cog_prov_name(g.prov), g.cos_angle, g.agreement, + g.binding_value[0] ? g.binding_value : "-", g.prev_edge, + (long long)(g.age_ms / 1000), g.decay, g.factual_now, g.stale ? "true" : "false"); + jb_puts(&b, t); emitted++; + store_edge_free(&e); + } + CogTrajectory tr; memset(&tr, 0, sizeof tr); + cog_grounding_trajectory(g_engram_store, base_id, now, &tr); + snprintf(t, sizeof t, + "],\"n_versions\":%d,\"derived\":{\"factual_volatility\":%.6g," + "\"relational_volatility\":%.6g,\"factual_drift\":%.6g,\"relational_drift\":%.6g," + "\"stayed_true_became_wrong\":%s}}", + emitted, tr.factual_volatility, tr.relational_volatility, + tr.factual_drift, tr.relational_drift, + tr.stayed_true_became_wrong ? "true" : "false"); + jb_puts(&b, t); + return el_wrap_str(b.buf); +} + +/* engram_assert_json(claim_id, for_whom, floor, rel_floor) — the honesty floor as + * a QUERY at ASSERTION time only (holding is never gated), now gating on BOTH + * axes. A well-evidenced claim must not earn the right to be asserted regardless + * of whether it means the right thing. + * + * `still_held` was a HARDCODED `true` in this format string — a temporal property + * named in the API and answered without consulting anything, which is invariant + * §8.1 violated in one literal. It is now derived: the claim's node is read and + * the field reports whether the content is present and live. Holding remains + * unconditional; what decays is the GROUNDING, and a relation whose decayed + * grounding has fallen below its floor stops being assertable on its own, + * without anyone having to remember to check. */ +el_val_t engram_assert_json(el_val_t claim_id, el_val_t for_whom, el_val_t floor, el_val_t rel_floor) { if (!g_engram_store) return eg_geo_err("store unavailable"); const char* fw = EL_CSTR(for_whom); if (fw && !*fw) fw = NULL; - double fl = atof(EL_CSTR(floor)); if (!(fl > 0)) fl = 0.5; - int gate = cog_assert_gate(g_engram_store, EL_CSTR(claim_id), fw, fl); - JsonBuf b; jb_init(&b); char t[192]; - snprintf(t, sizeof t, "{\"claim\":\"%s\",\"for_whom\":\"%s\",\"floor\":%.4g,\"may_assert\":%s,\"still_held\":true}", - EL_CSTR(claim_id), fw ? fw : "-", fl, gate == 1 ? "true" : "false"); + double fl = atof(EL_CSTR(floor)); if (!(fl > 0)) fl = 0.5; + double rfl = atof(EL_CSTR(rel_floor)); if (!(rfl > 0)) rfl = fl; + CogAssertion a; + if (cog_assert_two_axis(g_engram_store, EL_CSTR(claim_id), fl, rfl, engram_now_ms(), &a) != 0) + return eg_geo_err("assert failed"); + JsonBuf b; jb_init(&b); char t[640]; + snprintf(t, sizeof t, + "{\"claim\":\"%s\",\"for_whom\":\"%s\",\"floor\":%.4g,\"rel_floor\":%.4g," + "\"may_assert\":%s,\"still_held\":%s,\"found\":%s,\"n_relations\":%d," + "\"factual\":%.6g,\"relational\":%.6g,\"relational_established\":%s," + "\"cos_angle\":%.6g,\"agreement\":%d,\"binding_value\":\"%s\",\"best_relation\":\"%s\"," + "\"refused_because\":\"%s\"}", + EL_CSTR(claim_id), fw ? fw : "-", fl, rfl, + a.may_assert ? "true" : "false", a.still_held ? "true" : "false", + a.found ? "true" : "false", a.n_edges, + a.factual, a.relational, a.relational_established ? "true" : "false", + a.cos_angle, a.agreement, + a.binding_value[0] ? a.binding_value : "-", a.best_edge, + a.may_assert ? "-" : + !a.found ? "no-relation" : + !a.relational_established ? "relational-axis-never-established" : + (a.factual < fl) ? "below-factual-floor" : + (a.relational < rfl) ? "below-relational-floor" : "-"); jb_puts(&b, t); return el_wrap_str(b.buf); } @@ -13747,15 +15622,22 @@ el_val_t engram_correspondence_beat_json(el_val_t seeds, el_val_t faculty, el_va JsonBuf b; jb_init(&b); char t[384]; snprintf(t, sizeof t, "{\"faculty\":\"%s\",\"stance_id\":\"%s\",\"region_hub\":\"%s\",\"dim\":%d,\"n_axes\":%d,\"signal_axes\":%d," + /* `keystone_write_blocked` is GONE from this response. It reported that the + * beat had refused to learn about the reference frame, and the measured + * cost of that refusal was 0.00% brier reduction over n_trials 0 on the + * keystone region — the loop never ran, so nothing about the self was ever + * calibrated OR falsifiable. Nothing replaces the flag: non-circularity of + * the reference frame is temporal, not a permission (spec §5.2). The + * `keystone` field is retained as a label on the region, and it no longer + * gates anything. */ "\"resumed\":%s,\"keystone\":%s,\"probes\":%d,\"epochs\":%d," "\"brier_before\":%.6g,\"brier_after\":%.6g,\"reduction_pct\":%.2f," - "\"reliability\":%.6g,\"n_trials\":%lld,\"stance_written\":%s,\"keystone_write_blocked\":%s}", + "\"reliability\":%.6g,\"n_trials\":%lld,\"stance_written\":%s}", EL_CSTR(faculty), sid, g->hub_id ? g->hub_id : "region", dim, na, signal, resumed ? "true" : "false", st.keystone ? "true" : "false", NP, EP, brier_before, brier_after, brier_before > 0 ? 100.0 * (brier_before - brier_after) / brier_before : 0.0, - st.reliability, (long long)st.n_trials, wrote == 0 ? "true" : "false", - st.keystone ? "true" : "false"); + st.reliability, (long long)st.n_trials, wrote == 0 ? "true" : "false"); jb_puts(&b, t); cog_stance_free(&st); engram_geo_free(g); return el_wrap_str(b.buf); @@ -13952,6 +15834,21 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t di return el_wrap_str(b.buf); } +/* Public activation entry point. Serializes against the http_worker threads that + * share g->nodes/g->edges — this is the guard the awareness main thread + * (soul.el: awareness_run) was missing entirely. Nested calls from a worker that + * already holds the lock pass straight through. + * + * It no longer guards _eg_vindex: the index has its own publication boundary + * (eg_vindex_view / eg_vindex_maintain) and search cannot mutate it. This guard is + * now about the RAM graph's realloc-in-place ONLY. See the note at eg_guard_enter. */ +el_val_t engram_activate(el_val_t query, el_val_t depth) { + int owned = eg_guard_enter(); + el_val_t r = engram_activate_inner(query, depth); + eg_guard_exit(owned); + return r; +} + el_val_t engram_activate_json(el_val_t query, el_val_t depth) { /* Run two-layer engram_activate and serialize the result list to JSON. * Each entry includes both activation_strength (layer 1 background) and @@ -14089,6 +15986,59 @@ el_val_t engram_wm_top_json(el_val_t n_v) { return el_wrap_str(b.buf); } +/* op_assert seam (realizer promotion, bl-53/#57). + * Gathers the grounded ASSERTION ENVELOPE for a subject node — + * { "subject": , "grounding": [ {node,edge,hops}... ] } + * i.e. the self-geometry a realizer renders as faithful first-person text. + * Read-only: realization (geometry->text) stays in the faculty/realizer; + * this native primitive produces its structured input from proven paths + * (engram_emit_node_json + engram_neighbors_json). arity 2 (node_id, depth). */ +el_val_t engram_op_assert_json(el_val_t node_id, el_val_t depth) { + const char* sid = EL_CSTR(node_id); + JsonBuf b; jb_init(&b); + jb_puts(&b, "{\"subject\":"); + EngramNode* n = (sid && *sid) ? engram_find_node(sid) : NULL; + if (n) engram_emit_node_json(&b, n, 0); else jb_puts(&b, "null"); + jb_puts(&b, ",\"grounding\":"); + el_val_t nb = engram_neighbors_json(node_id, depth, EL_STR("both")); + const char* nbs = EL_CSTR(nb); + jb_puts(&b, (nbs && *nbs) ? nbs : "[]"); + jb_putc(&b, '}'); + return el_wrap_str(b.buf); +} + +/* Parametric mutation (purview write-side bounding, keystone 56ecbec6). + * The mutation verbs travel with a TARGET MANIFOLD (purview) instead of the + * implicit global singleton. purview==0 (EL_NULL) is the DEGENERATE/DEFAULT + * case: G = live, behaviour identical to the base op. A non-zero purview is a + * bounded target that the engine cannot yet resolve (multi-manifold store is a + * promotion item), so we REFUSE rather than silently mutate the live set — + * write-side bounding must never leak into G=live. */ +el_val_t engram_node_full_in(el_val_t purview, + el_val_t content, el_val_t node_type, el_val_t label, + el_val_t salience, el_val_t importance, el_val_t confidence, + el_val_t tier, el_val_t tags) { + if (purview == 0) { + return engram_node_full(content, node_type, label, salience, importance, + confidence, tier, tags); + } + fprintf(stderr, "[engram] purview write-side not yet resolvable (G != live); " + "refusing to append to live store (purview=%lld)\n", + (long long)purview); + return EL_STR(""); +} + +void engram_connect_in(el_val_t purview, + el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation) { + if (purview == 0) { + engram_connect(from_id, to_id, weight, relation); + return; + } + fprintf(stderr, "[engram] purview write-side not yet resolvable (G != live); " + "refusing to connect in live store (purview=%lld)\n", + (long long)purview); +} + el_val_t engram_stats_json(void) { EngramStore* g = engram_get(); /* embedded_count: how far the lazy backfill has progressed. The single @@ -14468,29 +16418,6 @@ el_val_t engram_label_df(el_val_t term) { #define ENGRAM_ST_TOKLEN 64 #define ENGRAM_ST_SCANCHARS 400 -/* Trim leading/trailing non-alphanumerics, then accept only tokens whose core - * is alphanumeric plus '-' and '_' with at least 3 letters. This subsumes the - * quoted-title guard (2026-07-25) and the "