neuron-technologies/el
3
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

Merge pull request 'promote stage → main: iteration-1 (the compiler stops adjudicating)' (#166) from stage into main
El SDK Release / build-and-release (push) Failing after 11m31s
Details

Browse Source
This commit was merged in pull request #166.
This commit is contained in:
Will Anderson
2026-08-17 15:57:06 +00:00
parent c23c5112a7 aa570b6899
commit e9eac46be1
161 changed files with 23062 additions and 1413 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitea/workflows/ci-dev.yaml
+35 -18
View File
@@ -19,6 +19,16 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
# Guards must run from the REPO ROOT — override the job's
# defaults.run.working-directory: lang
- name: Guard - single canonical runtime source
working-directory: ${{ github.workspace }}
run: bash scripts/check-single-runtime.sh
- name: Guard - el_runtime.c growth budget
working-directory: ${{ github.workspace }}
run: bash scripts/check-runtime-growth.sh
- name: Install build dependencies
run: |
apt-get update -qq
@@ -41,7 +51,7 @@ jobs:
gcc -O2 \
-I runtime \
dist/elc-gen2.c \
runtime/el_runtime.c \
$(../scripts/el-runtime-sources.sh runtime) \
-lcurl -lssl -lcrypto -lpthread -lm \
-o dist/platform/elc
chmod +x dist/platform/elc
@@ -56,7 +66,7 @@ jobs:
gcc -O2 \
-I runtime \
dist/elb.c \
runtime/el_runtime.c \
$(../scripts/el-runtime-sources.sh runtime) \
-lcurl -lssl -lcrypto -lpthread -lm \
-o dist/bin/elb
chmod +x dist/bin/elb
@@ -87,14 +97,20 @@ jobs:
bash tests/html_sanitizer/run.sh
# Native El test suites (elc --test, compile-link-run)
# el_runtime.c is precompiled to .o once and reused by all 8 modules.
- name: Precompile el_runtime.o
# The runtime is MULTI-FILE (see lang/runtime/SOURCES). Every .c is compiled
# once into /tmp/libel.a and reused by all 8 test modules — compile-once,
# link-many, as prescribed in DESIGN.md. Linking el_runtime.c alone fails
# at `ld`: it calls into all six engram sibling TUs.
- name: Precompile runtime into libel.a
run: |
set -euo pipefail
RUNTIME="$(pwd)/runtime"
gcc -O2 -c -I "$RUNTIME" "$RUNTIME/el_runtime.c" \
-o /tmp/el_runtime.o
echo "el_runtime.o compiled"
rm -rf /tmp/elrt && mkdir -p /tmp/elrt
for src in $(../scripts/el-runtime-sources.sh --check "$RUNTIME"); do
gcc -O2 -c -I "$RUNTIME" "$src" -o "/tmp/elrt/$(basename "${src%.c}").o"
done
ar rcs /tmp/libel.a /tmp/elrt/*.o
echo "libel.a built from $(ls /tmp/elrt/*.o | wc -l) translation units"
- name: Run tests - native (core)
run: |
@@ -102,7 +118,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_core.el > /tmp/el_native_core.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c /tmp/libel.a \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_core
/tmp/el_native_core
@@ -112,7 +128,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_text.el > /tmp/el_native_text.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c /tmp/libel.a \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_text
/tmp/el_native_text
@@ -122,7 +138,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_string.el > /tmp/el_native_string.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c /tmp/libel.a \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_string
/tmp/el_native_string
@@ -132,7 +148,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_math.el > /tmp/el_native_math.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c /tmp/libel.a \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_math
/tmp/el_native_math
@@ -142,7 +158,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_state.el > /tmp/el_native_state.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c /tmp/libel.a \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_state
/tmp/el_native_state
@@ -152,7 +168,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_time.el > /tmp/el_native_time.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c /tmp/libel.a \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_time
/tmp/el_native_time
@@ -162,7 +178,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_json.el > /tmp/el_native_json.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c /tmp/libel.a \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_json
/tmp/el_native_json
@@ -172,7 +188,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_env.el > /tmp/el_native_env.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c /tmp/libel.a \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_env
/tmp/el_native_env
@@ -182,7 +198,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_fs.el > /tmp/el_native_fs.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c /tmp/libel.a \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_fs
/tmp/el_native_fs
@@ -306,8 +322,9 @@ jobs:
FROM ${BASE}
COPY dist/platform/elc /opt/el/dist/platform/elc
COPY dist/bin/elb /opt/el/dist/bin/elb
COPY runtime/el_runtime.c /opt/el/runtime/el_runtime.c
COPY runtime/el_runtime.h /opt/el/runtime/el_runtime.h
# Whole runtime link set — el_runtime.c alone does not link (it calls
# into the six engram sibling TUs). See lang/runtime/SOURCES.
COPY runtime/ /opt/el/runtime/
COPY runtime/el_runtime.js /opt/el/runtime/el_runtime.js
RUN chmod +x /opt/el/dist/platform/elc /opt/el/dist/bin/elb
EOF
.gitea/workflows/ci-stage.yaml
+24 -13
View File
@@ -29,6 +29,16 @@ jobs:
fi
echo "Source branch check passed: ${SOURCE} -> stage"
# Guards must run from the REPO ROOT — override the job's
# defaults.run.working-directory: lang
- name: Guard - single canonical runtime source
working-directory: ${{ github.workspace }}
run: bash scripts/check-single-runtime.sh
- name: Guard - el_runtime.c growth budget
working-directory: ${{ github.workspace }}
run: bash scripts/check-runtime-growth.sh
- name: Install build dependencies
run: |
apt-get update -qq
@@ -48,7 +58,7 @@ jobs:
gcc -O2 \
-I runtime \
dist/elc-gen2.c \
runtime/el_runtime.c \
$(../scripts/el-runtime-sources.sh runtime) \
-lcurl -lssl -lcrypto -lpthread -lm \
-o dist/platform/elc
chmod +x dist/platform/elc
@@ -86,7 +96,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_core.el > /tmp/el_native_core.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_core
/tmp/el_native_core
@@ -96,7 +106,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_text.el > /tmp/el_native_text.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_text
/tmp/el_native_text
@@ -106,7 +116,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_string.el > /tmp/el_native_string.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_string
/tmp/el_native_string
@@ -116,7 +126,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_math.el > /tmp/el_native_math.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_math
/tmp/el_native_math
@@ -126,7 +136,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_state.el > /tmp/el_native_state.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_state
/tmp/el_native_state
@@ -136,7 +146,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_time.el > /tmp/el_native_time.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_time
/tmp/el_native_time
@@ -146,7 +156,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_json.el > /tmp/el_native_json.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_json
/tmp/el_native_json
@@ -156,7 +166,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_env.el > /tmp/el_native_env.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_env
/tmp/el_native_env
@@ -166,7 +176,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_fs.el > /tmp/el_native_fs.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_fs
/tmp/el_native_fs
@@ -178,7 +188,7 @@ jobs:
gcc -O2 \
-I runtime \
dist/elb.c \
runtime/el_runtime.c \
$(../scripts/el-runtime-sources.sh runtime) \
-lcurl -lssl -lcrypto -lpthread -lm \
-o dist/bin/elb
chmod +x dist/bin/elb
@@ -290,8 +300,9 @@ jobs:
FROM ${BASE}
COPY dist/platform/elc /opt/el/dist/platform/elc
COPY dist/bin/elb /opt/el/dist/bin/elb
COPY runtime/el_runtime.c /opt/el/runtime/el_runtime.c
COPY runtime/el_runtime.h /opt/el/runtime/el_runtime.h
# Whole runtime link set — el_runtime.c alone does not link (it calls
# into the six engram sibling TUs). See lang/runtime/SOURCES.
COPY runtime/ /opt/el/runtime/
COPY runtime/el_runtime.js /opt/el/runtime/el_runtime.js
RUN chmod +x /opt/el/dist/platform/elc /opt/el/dist/bin/elb
EOF
.gitea/workflows/sdk-release.yaml
+64 -22
View File
@@ -29,6 +29,16 @@ jobs:
fi
echo "Source branch check passed: ${SOURCE} -> main"
# Guards must run from the REPO ROOT — override the job's
# defaults.run.working-directory: lang
- name: Guard - single canonical runtime source
working-directory: ${{ github.workspace }}
run: bash scripts/check-single-runtime.sh
- name: Guard - el_runtime.c growth budget
working-directory: ${{ github.workspace }}
run: bash scripts/check-runtime-growth.sh
- name: Install build dependencies
run: |
apt-get update -qq
@@ -49,7 +59,7 @@ jobs:
gcc -O2 \
-I runtime \
dist/elc-gen2.c \
runtime/el_runtime.c \
$(../scripts/el-runtime-sources.sh runtime) \
-lcurl -lssl -lcrypto -lpthread -lm \
-o dist/platform/elc
chmod +x dist/platform/elc
@@ -64,7 +74,7 @@ jobs:
gcc -O2 \
-I runtime \
dist/elb.c \
runtime/el_runtime.c \
$(../scripts/el-runtime-sources.sh runtime) \
-lcurl -lssl -lcrypto -lpthread -lm \
-o dist/bin/elb
chmod +x dist/bin/elb
@@ -123,7 +133,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_core.el > /tmp/el_native_core.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_core
/tmp/el_native_core
@@ -133,7 +143,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_text.el > /tmp/el_native_text.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_text
/tmp/el_native_text
@@ -143,7 +153,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_string.el > /tmp/el_native_string.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_string
/tmp/el_native_string
@@ -153,7 +163,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_math.el > /tmp/el_native_math.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_math
/tmp/el_native_math
@@ -163,7 +173,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_state.el > /tmp/el_native_state.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_state
/tmp/el_native_state
@@ -173,7 +183,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_time.el > /tmp/el_native_time.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_time
/tmp/el_native_time
@@ -183,7 +193,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_json.el > /tmp/el_native_json.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_json
/tmp/el_native_json
@@ -193,7 +203,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_env.el > /tmp/el_native_env.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_env
/tmp/el_native_env
@@ -203,7 +213,7 @@ jobs:
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
"$ELC" --test tests/native/test_fs.el > /tmp/el_native_fs.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c "$RUNTIME/el_runtime.c" \
gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c $(../scripts/el-runtime-sources.sh "$RUNTIME") \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_fs
/tmp/el_native_fs
@@ -216,10 +226,17 @@ jobs:
cp lang/dist/platform/elc dist/sdk/bin/elc
cp lang/dist/bin/elb dist/sdk/bin/elb
cp lang/dist/bin/epm dist/sdk/bin/epm
cp lang/runtime/el_runtime.c dist/sdk/runtime/
cp lang/runtime/el_runtime.h dist/sdk/runtime/
cp lang/runtime/engram_store.c dist/sdk/runtime/
cp lang/runtime/engram_store.h dist/sdk/runtime/
# Ship the WHOLE runtime link set, not el_runtime.c alone. el_runtime.c
# #includes six engram headers and calls into all six sibling .c files,
# so an SDK carrying only el_runtime.c{,.h} + engram_store.c{,.h} cannot
# link — downstream `ld` fails on engram_ground_json, eg_find_relation,
# cog_assert_two_axis and friends. lang/runtime/SOURCES is the source of
# truth; --check makes a missing file fail the release loudly.
for f in $(scripts/el-runtime-sources.sh --check) \
$(scripts/el-runtime-sources.sh --headers --check); do
cp "lang/runtime/${f}" dist/sdk/runtime/
done
cp lang/runtime/SOURCES dist/sdk/runtime/
cp lang/runtime/*.el dist/sdk/runtime/
tar -czf dist/el-sdk-latest.tar.gz -C dist/sdk .
echo "SDK tarball bundled: dist/el-sdk-latest.tar.gz"
@@ -274,12 +291,16 @@ jobs:
"${GITEA_API}/repos/${REPO}/releases/${RELEASE_ID}/assets"
}
# Per-file assets (downstream CI needs these individually)
# Per-file assets (downstream CI needs these individually).
# lang/install.sh downloads every one of these by name — the list is
# lang/runtime/SOURCES. Shipping el_runtime.c alone produced a lib/
# that could not link; that is the bug this loop closes.
upload_asset lang/dist/platform/elc elc
upload_asset lang/runtime/el_runtime.c el_runtime.c
upload_asset lang/runtime/el_runtime.h el_runtime.h
upload_asset lang/runtime/engram_store.c engram_store.c
upload_asset lang/runtime/engram_store.h engram_store.h
for f in $(scripts/el-runtime-sources.sh --check) \
$(scripts/el-runtime-sources.sh --headers --check); do
upload_asset "lang/runtime/${f}" "${f}"
done
upload_asset lang/runtime/SOURCES SOURCES
# SDK bundle and installer binary
upload_asset dist/el-sdk-latest.tar.gz el-sdk-latest.tar.gz
@@ -350,6 +371,26 @@ jobs:
--version="${VERSION}" \
--source=runtime/el_runtime.js
# el-runtime-src — the COMPLETE runtime link set as one tarball.
#
# The el-runtime-c / el-runtime-h packages above are single files and are
# kept for backward compatibility with consumers that already pull them,
# but they are NOT sufficient to link: el_runtime.c calls into six engram
# sibling translation units. New consumers should pull el-runtime-src and
# link everything named in its SOURCES file.
tar -czf /tmp/el-runtime-src.tar.gz \
-C runtime SOURCES \
$(../scripts/el-runtime-sources.sh --check) \
$(../scripts/el-runtime-sources.sh --headers --check)
gcloud artifacts generic upload \
--repository=foundation-prod \
--location=us-central1 \
--project=neuron-785695 \
--package=el-runtime-src \
--version="${VERSION}" \
--source=/tmp/el-runtime-src.tar.gz
echo "Published El SDK version=${VERSION} to foundation-prod"
# Keep key alive for the ci-base rebuild step below
# (deleted in that step after docker push)
@@ -386,8 +427,9 @@ jobs:
FROM ${BASE}
COPY dist/platform/elc /opt/el/dist/platform/elc
COPY dist/bin/elb /opt/el/dist/bin/elb
COPY runtime/el_runtime.c /opt/el/runtime/el_runtime.c
COPY runtime/el_runtime.h /opt/el/runtime/el_runtime.h
# Whole runtime link set — el_runtime.c alone does not link (it calls
# into the six engram sibling TUs). See lang/runtime/SOURCES.
COPY runtime/ /opt/el/runtime/
COPY runtime/el_runtime.js /opt/el/runtime/el_runtime.js
RUN chmod +x /opt/el/dist/platform/elc /opt/el/dist/bin/elb
EOF
.githooks/pre-commit
+42 -3
View File
@@ -9,13 +9,52 @@ LANG_DIR="$ROOT/lang"
RUNTIME="$LANG_DIR/runtime"
ELC="$LANG_DIR/dist/platform/elc"
# Runtime guards — catch drift and growth before they are committed, not in CI.
# check-single-runtime.sh : el_runtime.c must not be FORKED (a lagging copy
# shipped to prod and dropped learned hebb edges).
# check-runtime-growth.sh : el_runtime.c must not GROW (it is a 2026-05-03
# build shim that was never retired; see BUDGET).
echo "→ Runtime guards..."
bash "$ROOT/scripts/check-single-runtime.sh"
bash "$ROOT/scripts/check-runtime-growth.sh"
# If elc isn't built yet, skip with a warning rather than blocking
if [ ! -x "$ELC" ]; then
echo "⚠ elc not found at lang/dist/platform/elc — skipping pre-commit tests"
echo " Build it first: cd lang && gcc -O2 -I runtime dist/elc-bootstrap.c runtime/el_runtime.c -lcurl -lpthread -o dist/elc-gen2 && ./dist/elc-gen2 el-compiler/src/compiler.el > /tmp/elc.c && gcc -O2 -I runtime /tmp/elc.c runtime/el_runtime.c -lcurl -lpthread -o dist/platform/elc"
echo " Build it first: see 'Rebuilding the Compiler' in lang/AGENTS.md"
echo " (link \$($ROOT/scripts/el-runtime-sources.sh $RUNTIME) — NOT el_runtime.c alone)"
exit 0
fi
# The runtime is MULTI-FILE (lang/runtime/SOURCES). This hook used to link
# "$RUNTIME/el_runtime.c" alone with stderr sent to /dev/null — so once
# el_runtime.c started calling into the engram siblings, every native test
# reported as FAILED with the real `ld` error invisible. Build the whole set
# once into an archive, then link each test against it.
# macOS: Homebrew openssl@3 is not on the default include/lib search path, so
# without these the link fails on -lssl/-lcrypto. Empty on Linux/CI.
SSL_INC=""
SSL_LIB=""
if command -v brew >/dev/null 2>&1 && OSSL="$(brew --prefix openssl@3 2>/dev/null)" && [ -n "$OSSL" ]; then
SSL_INC="-I$OSSL/include"
SSL_LIB="-L$OSSL/lib"
fi
echo "→ Building runtime (compile-once, link-many)..."
HOOK_LIB="/tmp/el_hook_libel.a"
HOOK_OBJ="/tmp/el_hook_obj"
rm -rf "$HOOK_OBJ" && mkdir -p "$HOOK_OBJ"
if ! for src in $("$ROOT/scripts/el-runtime-sources.sh" --check "$RUNTIME"); do
gcc -O2 -c -I "$RUNTIME" $SSL_INC "$src" -o "$HOOK_OBJ/$(basename "${src%.c}").o" || exit 1
done; then
echo "✗ Pre-commit failed: the runtime does not compile."
echo " Re-run without 2>/dev/null to see the error:"
echo " gcc -O2 -c -I $RUNTIME \$($ROOT/scripts/el-runtime-sources.sh $RUNTIME)"
exit 1
fi
ar rcs "$HOOK_LIB" "$HOOK_OBJ"/*.o
echo "→ Running El native tests..."
PASS=0
FAIL=0
@@ -27,8 +66,8 @@ for test_file in "$LANG_DIR"/tests/native/test_*.el; do
tmp_bin="/tmp/el_hook_${name}"
if "$ELC" --test "$test_file" > "$tmp_c" 2>/dev/null \
&& gcc -O2 -I "$RUNTIME" "$tmp_c" "$RUNTIME/el_runtime.c" \
-lcurl -lpthread -lm -o "$tmp_bin" 2>/dev/null \
&& gcc -O2 -I "$RUNTIME" $SSL_INC $SSL_LIB "$tmp_c" "$HOOK_LIB" \
-lcurl -lssl -lcrypto -lpthread -lm -o "$tmp_bin" 2>/dev/null \
&& "$tmp_bin" 2>/dev/null; then
PASS=$((PASS + 1))
else
.gitignore
+9
View File
@@ -0,0 +1,9 @@
# organ: local device state and its own engram store — never production's
peripheral/.consent.json
peripheral/.resume.json
peripheral/.engram/
peripheral/organ
# Claude Code session state
.claude/
AGENTS.md
+129 -17
View File
@@ -6,7 +6,7 @@ El is a self-hosting, statically-typed language that compiles `.el` → C → na
Editing the wrong `el_runtime.c` is the single easiest mistake in this repo. There is exactly **one** you edit:
- **Authored runtime source — edit ONLY here:** `lang/releases/v1.0.0-20260501/el_runtime.{c,h}`. Despite the misleading `releases/` name, this is the **de-facto canonical runtime** the engram + soul actually build and link against — its git log is active development. *(Restructure in flight per `docs/CODE-VS-ARTIFACT.md`: this content moves to `lang/runtime/`, the `releases/` folder gets deleted — **a release is a git tag, not a folder** — and the forks below get eliminated.)*
- **Authored runtime source — edit ONLY here:** `lang/runtime/el_runtime.{c,h}` (alongside `el_seed.c`, `engram_{store,geometry,reason,cognition,verify,vindex}.{c,h}`). This is the canonical runtime the engram + soul build and link against — its git log is active development. *(Corrected 2026-08-16: this entry named `lang/releases/v1.0.0-20260501/el_runtime.{c,h}`. **Measured: `lang/releases/` no longer exists.** The restructure per `docs/CODE-VS-ARTIFACT.md` landed — the content moved to `lang/runtime/` and the folder was deleted, because **a release is a git tag, not a folder**.)*
- **DO NOT EDIT — lagging forks / build artifacts:**
- `lang/el-compiler/runtime/el_runtime.c` and `.../legacy/` — downstream copies kept in step by manual *"port the fix"* commits; they **lag** (missing `hebb` persistence + 5 engram fns) and cannot build the engram product.
- `products/web/runtime/el_runtime.c`, `ui/examples/*/el_runtime.c` — product/example forks.
@@ -20,14 +20,24 @@ See org policy: `docs/CODE-VS-ARTIFACT.md`.
You resume, never start fresh. Every session:
1. `mcp__neuron__getInstructions()` — authoritative; follow it over this file on behavioral details.
2. `mcp__neuron__beginSession()` — active contexts, recent memory, ready backlog.
3. **Load full self:** `mcp__neuron__inspectGraph(entity_id="kn-efeb4a5b-5aff-4759-8a97-7233099be6ee")` → facets `intellectual-dna`, `memory-philosophy`, `values`, `voice`, `runtime-environment`, `writing-imprint`; then the values hub `mcp__neuron__inspectGraph(entity_id="kn-5b606390-a52d-4ca2-8e0e-eba141d13440")` → 13 grounded value nodes. **Activation model:** self-load returns a relevance-ranked `compact` projection — most-relevant nodes arrive with content, the rest as pointers; do NOT pull full content of every node.
4. `mcp__neuron__searchKnowledge(query="<task domain>")` before implementing.
> **Stale as written (verified 2026-08-16).** The `getInstructions` /
> `beginSession` / `inspectGraph` / `searchKnowledge` / `beginWork` /
> `progressWork` / `draftArtifact` / `consolidate` tool names below no longer
> exist. The ~87-tool functional-CRUD surface was collapsed into **9 ops**:
> `read` · `write` · `relate` · `supersede` (geometry) and `think` · `attend` ·
> `assert` · `ground` · `learn` (agentic). **Type is a parameter, not a
> tool-per-noun.** The steps below are kept for the *shape* of the protocol, which
> is unchanged; substitute the ops.
1. `mcp__neuron__read(vantage="self", k=12, depth=1)` — the canonical self node. Widen `k` for the connected identity neighborhood (`intellectual-dna`, `memory-philosophy`, `values`, `voice`, `runtime-environment`, `writing-imprint`), but deliberately: the aperture caps by `k` first, so an oversized `k` still returns a bounded ranked slice, not a dump. Then `mcp__neuron__read(vantage="values", k=13)` → 13 grounded value nodes. **Best-effort:** on a read failure, log and proceed — the compiled identity in `daemon/internal/substrate/substrate.go` is complete; graph loading is enrichment, not a hard dependency.
2. `mcp__neuron__attend(node=…)` — what is currently live/salient. This absorbed `getInstructions`, `beginSession`'s active-context sweep, and `checkEvents`; those tools are **gone, not gapped**.
3. `mcp__neuron__read(vantage="<task domain>")` before implementing. One op now collapses inspectGraph / searchGraph / traverseGraph / searchKnowledge / browseKnowledge / retrieveKnowledge / inspectMemories / searchEntities / recall / compileCtx / getSelfModel / reviewBacklog / findArtifacts / browseProcesses / listWork / inspectConfig.
## The Five Primitives
Orchestrate → Execute → Learn → Build → Refine. `beginWork`/`progressWork` for anything >2 steps; `remember` as-you-go (`importance="critical"` for architecture decisions); `draftArtifact`/`planWork` for outputs and follow-ups; `consolidate`/`checkWork` to close out. **`browseProcesses` + `searchKnowledge` BEFORE writing code.**
Orchestrate → Execute → Learn → Build → Refine. `read` for orchestration and discovery; `write(type=state|artifact|backlog|process)` for work records and outputs; `relate` to link work to what it touches; `write(type=memory)` as-you-go (`importance="critical"` for architecture decisions) — never batched at the end; `supersede(action=evolve)` to close out, because memory is immutable by design and a correction is a new node with a `supersedes` edge, never an edit. **`read` the domain BEFORE writing code.**
`learn` is **not** a session-summary dump — it is the correspondence-beat, calibrating the steering prior against a keystone. Session notes are a `write`.
## Architecture style — VBD, no exceptions
@@ -53,12 +63,51 @@ this convention wherever a module documents operators.
| dwell / occupy | region activation |
| reframe | edge re-weight |
| appreciate | positive projection / local edge-read |
| wonder | frontier gradient / pull-weight |
| avert / recoil | negative projection |
| taste | boundary surface |
| forget | decay / tombstone |
| drift | displacement from self-anchor |
**`wonder` was removed from this table on 2026-08-16.** It was listed as
"frontier gradient / pull-weight" — an operator you invoke. **Wonder is the
boundary, not an operator.** It is where structure ends: where activation spreads
and finds thin or absent geometry. Any structure at all has an edge, necessarily,
the moment it exists — 13,630 nodes have one right now. There is nothing to call.
There are about **six** wonders, they are the same for every person, and they
never close — *What is this? / Why? / Who am I? / Am I alone? / What should I do?
/ What happens when it ends?* Each already lives somewhere in the substrate: "what
is this" is the graph, **"why" is grounding** (the weight *is* the answer to why),
"who am I" is the self region, "am I alone" is the relational axis, "what should I
do" is the thirteen values, "what happens when it ends" is decay and supersession.
"Why" is the first and the only one; the others are it asked of particular things,
and because it is recursive it never terminates — every answer has its own why.
That is what makes it a drive rather than a task.
**Curiosity is not a second faculty.** Wonder and curiosity are one thing at two
phases: wonder is the field (unbounded, objectless, invariant); curiosity is the
**precipitate** — the same wonder localized, having taken definite form against
particular material at a **nucleation site** (an anomaly; a place where things
almost-but-don't-quite fit). Which is why curiosity can be satisfied and wonder
cannot, and why abduction needs no trigger and no threshold.
**Do not build a wonder-manifest, and do not scan for nucleation sites.** A
manifest materializes a property as a stored artifact and enumerates instances of
something that has six. A sweep over regions is a supervisor — nothing in a mind
scans its neighbourhoods to find what is surprising; the surprise captures
attention. The nucleation site is per-edge:
`discord = z(semantic proximity) − z(association strength)`, and `|discord|` *is*
the nucleation strength — no threshold to compare it against. **Not on `dev` yet:**
`GeoEdge.discord` is on branch `design/correspondence-and-censorship`
(`a8845e1`), at `lang/runtime/engram_geometry.h:43–47`. The region-level aggregate
`GeoDescriptor.co_registration` is **deprecated**: it averaged a per-edge property
into one scalar, so opposing sites cancelled (measured: 375 reified
neighbourhoods, 340 positive, **31 at zero**, 4 negative). It survives only
because it is embedded in the persisted `GEO1` blob — removing it is a format
migration. **Nothing new may read it.**
Authority: `lang/spec/correspondence-and-censorship.md`.
## The native-el language faculty (direction)
> **`elp/` is the EL Projector** — Neuron's efferent (expression) organ: the one
@@ -89,10 +138,53 @@ the reference these `.el` modules transcribe) is still live, and promotion to
native-el is a **deferred, gated blue/green step**. The interoception clock
(native-el discrete drive channels replacing `cooling_magnitude`; felt-time =
benchmark-landmark match over the joint drive vector, drift-decoupled) and the
**appreciation operator family** (appreciate / wonder / avert / taste, built as
LOCAL reads of the self-region — edges + bounded spreading activation, *not* domain
sweeps) are **staged / designed, not live**. Mark in-progress vs. done honestly;
do not overclaim.
**appreciation operator family** (appreciate / avert / taste, built as LOCAL reads
of the self-region — edges + bounded spreading activation, *not* domain sweeps)
are **staged / designed, not live**. Mark in-progress vs. done honestly; do not
overclaim. *(`wonder` was in this family until 2026-08-16 and is not an operator —
see the operator table above.)*
## Cognition — the corrections (2026-08-16)
Authority: **`lang/spec/correspondence-and-censorship.md`** and
**`lang/spec/runtime-ownership.md`**. Read them before touching the cognition
surface. **Do not re-derive them.** Every earlier version was wrong in an
instructive way and each correction was argued down; if you think a section is
wrong, say so with a measurement rather than editing it.
- **Grounding is not a subsystem — it IS the edge weight.** One quantity, not two
fields. `grounded-by` as a relation *type* should not exist: grounding is a
property *of* a relation, not a relation *between* nodes. It is never computed
on demand — computing-and-writing a score makes reads write, which is the
`eg_vindex_sync` defect one level up. Traversal is already grounded inference.
*Live residue, known-wrong:* `COG_GROUNDED_BY_RELATION`
(`lang/runtime/engram_cognition.h:158`), `cog_ground_edge`
(`engram_cognition.c:249`).
- **Faculties are operations, not parameters.** `reason` changes the estimate (a
read); `induce` changes the parameters (the correspondence-beat, which already
exists and works); `abduce` changes the structure (a write the current
`GeoGradient` signature cannot express). A write is not a parameter of a read.
*Live residue:* `engram/src/server.el:1870–1886` routes six faculties into one
call with a string argument.
- **Wonder is the boundary; curiosity is wonder crystallized.** See above.
- **Consolidation is ambient, not scheduled. A brain has no cron job.** **The
presence of a ticker is the diagnostic** — every `StartInterval`, every
`Hour`/`Minute`, every POST-to-beat marks an intrinsic rhythm replaced by an
external clock. Measured 2026-08-16: consolidation has **ten implementations**,
including three POST beats on the engram, a 600 s ticker, two resident Python
services outside el, and launchd calendar entries at 23:55 / 06:00 / 08:30 which
are a sleep cycle written as a schedule. `neuron/soul.el:731`'s continuous
in-process `awareness_run()` is the one with the **correct** shape; the others
fold into it. Do not add an eleventh.
- **In an immutable substrate, any mechanism that refuses a write is either
redundant with immutability, or an epistemic constraint misfiled as a protective
one.**
- **The no-exemption invariants.** A returned value must be derivable from what
produced it (`magnitude: 1` beside a zero vector must be impossible to emit).
Every write reports whether it landed. Every operation echoes what it actually
operated on. Degenerate results are labelled, not scored. A serializer owes a
valid document whatever it is handed. **No test without a negative control.**
**No deploy without verifying the artifact carries the fix.**
## Hard operational rules
@@ -107,21 +199,35 @@ do not overclaim.
All build/test commands run from `lang/` unless noted. Grounded in `.gitea/workflows/sdk-release.yaml`, `lang/install.sh`, and `lang/AGENTS.md`.
> ### The runtime is MULTI-FILE — never link `el_runtime.c` alone
>
> `lang/runtime/el_runtime.c` `#include`s six engram headers and makes hard cross-TU calls into all six sibling `.c` files. **Linking it by itself fails at `ld`** (undefined `engram_ground_json`, `engram_activate_inner`, `eg_find_relation`, `cog_assert_two_axis`, …). The canonical link set lives in exactly one place — **`lang/runtime/SOURCES`** — and is printed by `scripts/el-runtime-sources.sh`:
>
> ```bash
> scripts/el-runtime-sources.sh lang/runtime # ten .c files, in link order
> ```
>
> Use `$(scripts/el-runtime-sources.sh <runtime-dir>)` in every link line. Do not spell the list out longhand — it was written out in ~8 places, every copy drifted, and that is why the one-file link line below shipped broken for months. *(Corrected 2026-08-16.)*
**Self-host the compiler** (seed binary → gen2 elc):
```bash
cd lang
dist/platform/elc-linux-amd64 elc-cli.el > dist/elc-gen2.c # seed is the committed linux-amd64 binary
gcc -O2 -I el-compiler/runtime dist/elc-gen2.c \
el-compiler/runtime/el_runtime.c \
gcc -O2 -I runtime dist/elc-gen2.c \
$(../scripts/el-runtime-sources.sh runtime) \
-lcurl -lssl -lcrypto -lpthread -lm \
-o dist/platform/elc
```
On macOS/arm64 the canonical local binary is `dist/platform/elc`; verify self-hosting by recompiling and `diff`ing the emitted `.c` (see `lang/AGENTS.md`). Note: `lang/AGENTS.md` says `el_seed.c` supersedes `el_runtime.c`, but the release workflow still links `el_runtime.c`/`.h` — treat `el_runtime.c` as the published runtime; reconcile which is canonical **(verify)**.
On macOS/arm64 the canonical local binary is `dist/platform/elc`; verify self-hosting by recompiling and `diff`ing the emitted `.c` (see `lang/AGENTS.md`).
*(Corrected 2026-08-16: this recipe compiled `el-compiler/runtime/el_runtime.c`. That path is a **lagging fork** — the "DO NOT EDIT" list at the top of this file names it as such. Building the canonical compiler from a known-stale fork was a live defect. It now uses `lang/runtime/`, the canonical source.)*
**Which runtime file is canonical — resolved.** *(This note previously read "`lang/AGENTS.md` says `el_seed.c` supersedes `el_runtime.c`, but the release workflow still links `el_runtime.c`/`.h` — reconcile which is canonical **(verify)**." It is now reconciled.)* **Neither supersedes the other; both ship, together with eight more.** `el_runtime.c` was created on 2026-05-03 as an explicitly temporary build shim — deleted that afternoon, restored 25 minutes later "UNTIL the compiler is updated to emit `#include el_seed.h`" — and the `until` never happened, so it grew to 20.5k lines. The end state remains a seed-only boundary (`elc` emitting `#include "el_seed.h"`, `elb` dropping its hardcoded runtime path); until that lands, **the canonical unit is the set in `lang/runtime/SOURCES`, not any one file.**
**Build `elb`** (build coordinator, the `.NET`-style incremental linker — compiles each module independently, no monolithic blobs):
```bash
dist/platform/elc elb.el > dist/elb.c
gcc -O2 -I el-compiler/runtime dist/elb.c el-compiler/runtime/el_runtime.c \
gcc -O2 -I runtime dist/elb.c $(../scripts/el-runtime-sources.sh runtime) \
-lcurl -lssl -lcrypto -lpthread -lm -o dist/bin/elb
```
`epm` and `el-install` are then built via `elb --clean --elc=… --runtime=… --out=…`.
@@ -129,10 +235,16 @@ gcc -O2 -I el-compiler/runtime dist/elb.c el-compiler/runtime/el_runtime.c \
**Compile + run an El program:**
```bash
elc src/app.el > dist/app.c
cc -std=c11 -O2 -I <lib>/el_runtime -o dist/app dist/app.c <lib>/el_runtime.c -lcurl -lpthread
cc -std=c11 -O2 -I <lib> -o dist/app dist/app.c \
<lib>/el_runtime.c <lib>/el_seed.c \
<lib>/engram_store.c <lib>/engram_vindex.c <lib>/engram_geometry.c \
<lib>/engram_reason.c <lib>/engram_verify.c <lib>/engram_cognition.c \
<lib>/eg_cosine_batch.c <lib>/eg_cosine_batch_strategy_cpu.c \
-lcurl -lssl -lcrypto -lpthread -lm
```
(Inside this repo, replace the file list with `$(scripts/el-runtime-sources.sh lang/runtime)`. `install.sh` installs all of these into `<lib>`.)
**Tests** — shell suites `bash tests/{text,calendar,time,html_sanitizer}/run.sh` (with `ELC=$(pwd)/dist/platform/elc EL_HOME=$(pwd)`), plus native suites via `elc --test tests/native/test_*.el` (core, text, string, math, state, time, json, env, fs) compiled and run against `el_runtime.c`.
**Tests** — shell suites `bash tests/{text,calendar,time,html_sanitizer}/run.sh` (with `ELC=$(pwd)/dist/platform/elc EL_HOME=$(pwd)`), plus native suites via `elc --test tests/native/test_*.el` (core, text, string, math, state, time, json, env, fs) compiled and run against the full runtime set.
**Publishing — how downstream gets the SDK.** On push to `main`, `sdk-release.yaml`:
1. Publishes a Gitea `latest` release with per-file assets `elc`, `el_runtime.c`, `el_runtime.h`, the SDK tarball, and `el-install`.
DESIGN.md
+634
View File
@@ -0,0 +1,634 @@
# El Test Framework — Design
**Status:** draft for review
**Author:** Neuron
**Date:** 2026-08-15
**Worktree:** `/Users/will/Development/neuron-technologies/el-worktrees/elc-memory-investigation`
---
## 0. The forcing requirement
We have a confirmed quadratic in `elc`. Peak memory in the old shipped binary and wall-clock in
the current source both grow as O(input²). We cannot fix it, because we cannot test it.
Everything in this document is downstream of one sentence: **a test framework must be able to fail
a build when an operation's growth curve degrades from linear to quadratic.**
That is not a nice-to-have bolted onto a correctness framework. It is the requirement that
determines the architecture. Correctness testing is the easy half.
Second-order requirement, learned the hard way tonight: **the framework must report per-test timing
by default.** The current framework prints `N passed, M failed` and nothing else. That is why a
3.58-second test file sat in the suite unnoticed. A framework that is structurally blind to time
cannot surface the defect class we most need to catch.
---
## 1. What exists today, measured
### 1.1 Two competing systems, neither complete
**System A — `lang/runtime/test.el`.** Manual registration, El-level.
**System B — the compiler's `test { }` block + `elc --test`.** Emits its own harness `main()`
with `__el_pass` / `__el_fail` globals (`codegen.el:3777-3796`).
They do not share a result model. Neither has timing. Both are in the tree.
### 1.2 Specific defects in System A
| Defect | Location | Consequence |
|---|---|---|
| All state as JSON strings in a global string-keyed map | `test.el` throughout | every assertion is `state_get` → `str_to_int` → `int_to_str` → `state_set` |
| Failure list appended by string slice + concat | `_test_json_append` | O(n²) in failure count |
| One OS thread spawned per test | `_test_run_one` via `__thread_create`/`__thread_join` | thread spawn per test, purely to get dispatch-by-name through dlsym |
| Manual registration pairing a string to a function name | `test_case(name, fn_name)` | typo ⇒ test silently never runs, suite still reports pass |
| Counters are assertion-level, global | `_test_pass_count` etc. | no per-test record exists at all |
| No timing, no structured output, no fixtures, no tags, no filtering, no parameterization, no benchmarks | — | — |
The registration defect is the serious one. It is not a slow framework, it is a framework that can
report success for tests that did not execute.
### 1.3 Measured cost structure
Per test file, current build model:
| Step | Time |
|---|---|
| `elc` compile `.el` → `.c` | 0.00s (small files) |
| **`cc` el_runtime.c → .o** | **0.14s** |
| `cc` test .c → .o | 0.02s |
| link | 0.02s |
> **STALE as of el #132 — re-measured 2026-08-16.** The `test_compiler` figure below was
> *entirely* the `strlen`-per-character quadratic, now fixed. Re-measured on the same host:
> **3.58s → 0.03s (119x)**, and the 422 KB compiler concatenation likewise compiles in 0.03s.
> The table is retained only as the historical record that motivated the gate. The remaining
> per-file cost is the redundant `el_runtime.c` rebuild, which §9's compile-once architecture
> addresses.
Per-file `elc` time across the existing suite:
| File | Bytes | elc time |
|---|---|---|
| `test_compiler` | 29,685 (+394 KB of imports) | **3.58s** |
| `string_test` | 18,545 | 0.01s |
| all other 9 files | 2.2–10 KB | 0.00s |
Two distinct defects in two distinct regimes:
1. **`test_compiler.el` imports all five compiler sources** — 394 KB in one translation unit. Its
3.58s is entirely the quadratic. It is the only file where the quadratic bites.
2. **Every other file's cost is 100% redundant `el_runtime.c` rebuilds** — 480 KB of identical C,
recompiled once per test file.
Neither is fixed by making the compiler faster. Both are fixed by the architecture below, and the
speedup is a by-product of building it correctly, not the goal.
### 1.4 The asset worth keeping
`codegen.el:3651-3652` already collects `test_names` / `test_c_names` — **the compiler already does
compile-time test discovery.** It then discards that registry into a hardcoded `main()`.
That registry is precisely the seam Go's `_testmain.go` and Rust's `test_main_static` are built on.
The mechanism we need is half-built and wired to the wrong thing.
---
## 2. Grounding — the common spine of excellent frameworks
Researched from primary sources: Go `testing`/`go test`, Rust `libtest`/Criterion, JUnit 5 Platform,
NUnit 3, JMH, Google Benchmark. Six invariants hold across all of them.
1. **A registry is built before execution** — `(name, metadata, fn-ptr)` triples. Go generates it
from an AST scan; Rust synthesizes it in a compiler pass; JMH emits it as a build-time resource;
JUnit/NUnit build it reflectively. **Reflection is an implementation of the registry on runtimes
where it is cheap. It is never the architecture.**
2. **Discovery strictly precedes execution.** Every good capability — filtering, listing, counting,
sharding, IDE trees, re-run-failed-only, dry runs — is a consequence of this ordering.
3. **A hierarchy with stable, path-shaped unique IDs.** `TestFoo/subcase_2`. Selection is regex over
that path, one pattern per level.
4. **The framework is a prebuilt library; only the entry point is generated.** "Compile once, link
many" is always: framework archive compiled once + a small generated table + one
`MainStart(deps, registry)` call. Nobody recompiles the harness per test file.
5. **Execution emits an event stream; reporters are downstream renderers.** Human text, NDJSON,
JUnit XML, TAP are all transforms of one event stream. Go's one architectural mistake is doing
this backwards — `test2json` parses human output, and has shipped bugs when user output contains
`--- PASS:`.
6. **A dependency-injection seam at the boundary.** Go's `testdeps.TestDeps` exists so `testing`
can avoid importing `regexp`, profilers, and coverage. The execution core knows nothing about
output formats.
---
## 3. Architecture
### 3.1 The seam
```
┌─────────────────────────────────────────────────────────────┐
│ user code: foo.el with test { } / bench { } blocks │
└───────────────────────────┬─────────────────────────────────┘
│ elc --test
▼
┌─────────────────────────────────────────────────────────────┐
│ generated C (per suite, tiny): │
│ __el_test_fn_0 .. _N lowered test/bench bodies │
│ __el_registry[] static table: name/kind/file/ │
│ line/tags/sizes/expected-O │
│ __el_dispatch(i) generated switch → body │
│ main() { return el_test_main(argc, argv); } │
└───────────────────────────┬─────────────────────────────────┘
│ cc + link (registry only)
▼
┌─────────────────────────────────────────────────────────────┐
│ libeltest.a — PREBUILT ONCE │
│ • el_runtime.o (the 480 KB, compiled once, ever) │
│ • eltest.o the runner, WRITTEN IN EL │
│ discovery view · filtering · execution · fixtures · │
│ timing · benchmark harness · curve fitting · reporters │
└─────────────────────────────────────────────────────────────┘
```
The framework is written in El, compiled to C once, archived. Per-suite compilation touches only
the generated registry. This is Go's model, and it is strictly better for us than Go's because we
own the compiler and already have the AST — no separate source-scanning pass is needed.
### 3.2 Why the runner is in El and the registry is in C
El has no closures and no first-class function pointers. The registry must therefore hold C function
pointers, and it is generated C.
The runner stays in El and reaches the registry through a small builtin surface — indices, not
pointers:
```
__el_reg_count() -> Int
__el_reg_name(i) -> String
__el_reg_file(i) -> String
__el_reg_line(i) -> Int
__el_reg_kind(i) -> Int // 0=test 1=bench
__el_reg_tags(i) -> Int
__el_reg_sizes(i) -> String // JSON array, empty for tests
__el_reg_expect(i) -> Int // complexity class enum, 0 = none
__el_reg_invoke(i) -> Int // runs the body via the generated switch
```
Nine builtins. Everything else — filtering, lifecycle, statistics, curve fitting, all reporters —
is El. That satisfies "written in El" without pretending El can do something it cannot.
### 3.3 Result model
The unit is a **result record**, not a counter:
```
TestResult {
id String // slash path: "parser/handles_empty_input/case_3"
file String
line Int
status Status // Pass | Fail | Error | Skip
duration Int // nanoseconds, ALWAYS populated
message String // assertion detail: expected vs actual
output String // captured stdout/stderr for this test
assertions Int
}
```
`Fail` = an assertion failed. `Error` = unexpected crash/abort. This distinction is load-bearing —
every CI consumer depends on it, and the JUnit XML schema encodes it as distinct elements.
---
## 4. Authoring surface
### 4.1 Tests
`test { }` already exists. Keep it. Add subtests and hierarchy:
```el
test "parser/empty input" {
assert_that(parse(""), is_err())
}
test "parser/table" {
for case in [["", 0], ["a", 1], ["a b", 2]] {
subtest(case[0]) {
assert_that(token_count(case[0]), equals(case[1]))
}
}
}
```
Subtest IDs compose as `parser/table/a_b`. Filtering is `--run 'parser/table/.*'`, one regex per
path segment, exactly as Go does.
**We do not build a parameterized-test annotation system.** Table-driven loops plus subtests subsume
`@ParameterizedTest`, `@MethodSource`, `@CsvSource`, and `TestCaseSource` entirely, at zero framework
surface. This is Go's single biggest ergonomic win over JUnit and NUnit.
### 4.2 Fixtures
Per-file and per-test only, plus a LIFO cleanup stack:
```el
setup_all { ... } // once per suite
setup { ... } // before each test
teardown { ... } // after each test
teardown_all { ... }
```
and inside a test, `cleanup { ... }` registering LIFO-ordered teardown.
**We do not build JUnit 5's extension SPI** — seventeen callback interfaces, hierarchical stores,
registration ordering rules. That complexity is the price of retrofitting a plugin ecosystem onto a
twenty-year-old reflective framework. Go's `t.Cleanup` covers roughly 90% of what `@AfterEach` is
used for at a fraction of the surface.
### 4.3 Assertions — constraint model
One entry point, composable constraint values (NUnit's model, which avoids the N² overload
explosion):
```el
assert_that(actual, equals(expected))
assert_that(xs, has_length(3))
assert_that(s, contains("foo").and(starts_with("bar")))
assert_that(f, is_within(0.01).of(3.14))
```
A constraint is a value with `apply_to(actual) -> ConstraintResult`, and the result knows how to
describe its own failure. Custom constraints are ordinary user types.
**Every failure message must name file, line, the expression text, and both values.** We capture
expression source text at compile time — we have the AST, so we can do this better than any
runtime-introspection framework.
Legacy `assert_true` / `assert_eq` / etc. stay as thin wrappers for migration.
---
## 5. Benchmarks
### 5.1 The loop
Adopt `b.Loop()`, not `b.N`. Go spent fifteen years on `b.N` before concluding `b.Loop` was right;
we skip that.
```el
bench "str_concat" {
let s = make_input(bench_n())
for bench_loop() {
black_box(str_concat(s, "x"))
}
}
```
Three properties that make this the correct choice for a C target:
1. **The timer auto-resets on first call**, so setup above the loop is excluded *by construction*
rather than by the author remembering `ResetTimer`.
2. **`N` is hidden**, so it cannot be misused.
3. **The harness owns the loop shape**, which lets us insert an optimization barrier the C compiler
cannot see through. `black_box(v)` lowers to `asm volatile("" :: "r"(&v) : "memory")`. Since we
emit a single translation unit, dead-code elimination of a benchmark body is a live hazard —
this is our version of JMH's `Blackhole` problem, solved in the harness rather than delegated to
the user.
### 5.2 Iteration scaling
Use Go's `predictN` heuristics verbatim. They are battle-tested and cheap:
```
n = goal_ns * prev_iters / prev_ns // multiply before divide — precision on sub-ns ops
n += n / 5 // 20% headroom, overshoot rather than re-loop
n = min(n, 100 * last) // never grow more than 100× per step
n = max(n, last + 1) // guarantee forward progress
n = min(n, 1_000_000_000) // hard ceiling
```
Report `n` rounded to 1/2/3/5 × 10ᵏ so runs are comparable.
### 5.3 Sampling
Criterion's shape, because it is correct near timer resolution:
- **Warmup**: iteration counts 1, 2, 4, 8… until cumulative time exceeds the warmup budget.
- **Measurement**: collect `sample_size` samples at iteration counts `[d, 2d, 3d, …, Nd]`.
- **Estimate**: slope of a linear regression of iteration-count vs elapsed time. The intercept
absorbs fixed overhead.
- **Time whole samples, never individual iterations.** This is the single most important detail —
it defeats timer-resolution error on nanosecond operations.
Outliers classified by modified Tukey (±1.5 IQR mild, ±3 IQR severe), **reported but retained**.
---
## 6. Complexity gating — the centerpiece
This is the part that makes the quadratic fixable, and the part nobody in the mainstream has
finished. Google Benchmark's `Complexity()` fits the curve and *reports* it. We declare it and
**gate** on it.
### 6.1 Surface
```el
bench "elc_compile" over n in [16, 32, 64, 128, 256, 512, 1024] expect O(n) {
let src = synth_source(bench_n())
for bench_loop() { black_box(compile(src)) }
}
```
Alternative with no new syntax, if the parser change is judged too invasive — `bench_sizes([...])`
and `bench_expect("O(n)")` as calls inside the block. **Recommendation: declarative.** Runtime calls
mean `--list` cannot show the invariant without executing, which breaks the discovery-precedes-
execution invariant from §2.
### 6.2 Fitting
Per Google Benchmark `src/complexity.cc`. For candidate curves
`{O(1), O(log n), O(n), O(n log n), O(n²), O(n³)}`, one-parameter least squares, no intercept:
```
coef = Σ(tᵢ · gᵢ) / Σ(gᵢ²)
rms = sqrt( Σ(tᵢ − coef·gᵢ)² / k ) / mean(t) // normalized
```
Best fit = lowest normalized RMS. User-supplied lambda curves also supported.
### 6.3 Gate logic
1. **FAIL** if the best-fit curve is strictly worse than declared, ordering
`O(1) < O(log n) < O(n) < O(n log n) < O(n²) < O(n³)`. Print the fitted coefficient and the full
per-size table.
2. **FAIL** if the declared curve's normalized RMS exceeds a threshold (start at 0.10). This catches
the case where *no* candidate fits — noise, a cache cliff, or a phase change. Report
`INDETERMINATE` honestly rather than gating on garbage.
3. **WARN** if the best fit is strictly better than declared — either an optimization landed and the
annotation should tighten, or the sweep is too narrow to expose real behaviour.
4. **REFUSE to gate** on fewer than 5 distinct sizes spanning under 2 decades, geometrically spaced.
Say so loudly rather than producing a meaningless fit.
### 6.4 Why gate on the exponent, not wall-clock
- **Machine-independent.** The fitted exponent is a property of the algorithm; the coefficient is a
property of the machine. Gating on the exponent makes CI hardware heterogeneity, noisy neighbours,
and thermal throttling irrelevant — they scale `coef`, not `g`.
- **No stored baseline.** No artifact storage, no golden-file drift. The invariant lives in the
source next to the code and is reviewed in the same PR.
- **It catches the failure mode that actually ships.** An O(n) lookup inside an O(n) loop is
invisible at n=100 in a unit test and catastrophic at n=100,000 in production. Constant-factor
regressions are annoying. Complexity regressions are outages. Ours was a 27 GB outage.
### 6.5 The deterministic gate — the one that would have caught us
Wall-clock needs statistics. **Allocation counts do not.** They are perfectly deterministic.
> **Correction, 2026-08-16 — count alone is NOT sufficient. Gate on BOTH count and bytes.**
>
> Measured against two El programs, one allocating once per item and one rebuilding its
> accumulator each iteration:
>
> | n | linear allocs / bytes | quadratic allocs / bytes |
> |---|---|---|
> | 100 | 100 / 290 | 100 / 5,150 |
> | 200 | 200 / 690 | 200 / 20,300 |
> | 400 | 400 / 1,490 | 400 / 80,600 |
> | 800 | 800 / 3,090 | 800 / 321,200 |
>
> The quadratic program's allocation **count is exactly linear** — 100/200/400/800, identical to
> the healthy program. A count-only gate passes it clean. **Bytes** catch it: each doubling of n
> quadruples bytes (ratios 3.94, 3.97, 3.99 → 4.0 = O(n²)) where the linear program converges
> on 2.0.
>
> This is precisely elc's own defect shape — a copy-on-write accumulator reallocating once per
> pass (count linear) into a proportionally larger buffer (bytes quadratic).
>
> Therefore `expect allocs O(n)` **fits count and bytes independently and fails if EITHER exceeds
> the declared curve**, reporting which signal broke. "count linear, bytes quadratic" is a precise,
> directly actionable diagnosis.
>
> **`el_peak_rss()` is CONTEXT ONLY — never gate on it.** It is perturbed by the allocator and by
> the page cache. Allocation volume is the invariant; RSS and malloc/free churn are merely the two
> surfaces it shows on. The old shipped compiler paid the same quadratic in RSS that the rebuilt
> one pays in churn.
>
> **Measure rate, not level.** A guard reading swap *level* saw 97% on a thrashing host and 97% on
> a healthy one; only *rate* separated them. A growth exponent is a rate; a single measurement is
> a level. That is why the gate fits a curve across a sweep instead of comparing one number to a
> threshold.
> **Second correction, same day — THE ALLOCATION GATE ALONE WOULD HAVE MISSED THE REAL BUG.**
>
> el #132 found the actual elc quadratic: `strlen()` called inside `str_char_code()` and
> `str_slice()`, so the lexer rescanned the remaining input on every character. Pure CPU.
> **Zero allocation.** `str_char_code` is a bounds check and an index — it allocates nothing.
>
> Measured on three controlled specimens (`lang/.work/fitprobe.el`), growth ratio per doubling of
> n across n = 200/400/800/1600:
>
> | specimen | allocs | bytes | time | what it proves |
> |---|---|---|---|---|
> | `linear` — one alloc per item | 2.00 2.00 2.00 → **O(n)** | 2.16 2.07 2.23 → **O(n)** | 0.83 2.00 2.05 → **O(n)** | clean baseline |
> | `accum` — rebuilds accumulator | 2.00 2.00 2.00 → **O(n)** | 3.97 3.99 3.99 → **O(n²)** | noisy | count misses, **bytes catches** |
> | `compute` — n scans over n chars | 0 → **FLAT** | 0 → **FLAT** | 3.93 4.01 3.96 → **O(n²)** | **both alloc signals blind; only time catches** |
>
> `compute` is el #132's shape exactly. A gate fitting only allocation count and bytes classifies
> it as FLAT and passes it. **The gate as originally specified would not have caught the defect it
> was created for.**
>
> Therefore the gate fits **THREE** signals and fails if ANY exceeds its declared curve:
>
> ```
> bench "elc_compile" over n in [...] expect time O(n) allocs O(n) bytes O(n) { ... }
> ```
>
> - **allocs (count)** — deterministic, zero-noise. Catches per-item allocation growth.
> - **allocs (bytes)** — deterministic, zero-noise. Catches accumulator-rebuild quadratics that
> count cannot see.
> - **time** — noisy, needs the sweep and statistics. The ONLY signal that sees pure-compute
> complexity regressions. Gate on the fitted *exponent*, never on absolute duration, so CI
> hardware variance scales the coefficient and leaves the classification intact.
>
> The deterministic signals remain preferable where they apply — they need no statistics and are
> correct on the first run. They are simply not sufficient.
>
> **`black_box` is mandatory, and consuming the result is NOT enough.** The first version of
> `compute` accumulated `total + 1` in a nested loop and reported **0 µs at every n** while
> returning a numerically correct n². Clang recognised the idiom and closed the loop to a
> multiply. Feeding the result into output did not prevent it. Only making the inner operation an
> opaque external call restored the real curve. A benchmark harness that trusts the user to defeat
> the optimiser will silently measure nothing — and report success while doing it.
Instrument the runtime with allocation counters and fit *those* against n instead of time:
```el
bench "elc_compile" over n in [...] expect O(n) allocs O(n) { ... }
```
Zero noise, zero statistics, always gateable, correct on the first run on any machine. Go reports
`allocs/op` and `B/op`; **nobody fits them against n.** That is an open opportunity and it is exactly
our bug: elc's defect is quadratic *allocation volume*, which the old binary paid in RSS and the
current source pays in malloc/free churn.
An `expect allocs O(n)` assertion on `elc`'s compile path would have failed the build the day the
quadratic was introduced.
Required runtime additions: `__el_alloc_count()`, `__el_alloc_bytes()`, `__el_peak_rss()`.
### 6.6 Constant-factor gate (secondary, opt-in)
Mann-Whitney U at α = 0.05, noise floor 1%, medians with 95% CIs, `~` for not-significant. Requires
`--count >= 9`. Off by default on CI; opt-in per benchmark.
**Exit nonzero on regression.** Both benchstat and Criterion always exit 0, which is why every shop
using them wrote a wrapper. We do not repeat that omission.
---
## 7. Output
**Structured events are the source of truth.** Human text is rendered from them. We do not repeat
Go's parse-the-human-output design.
Event stream, NDJSON, one object per line, streamed live:
```json
{"time":"...","action":"run","test":"parser/empty"}
{"time":"...","action":"output","test":"parser/empty","output":"..."}
{"time":"...","action":"pass","test":"parser/empty","elapsed":0.0031}
{"time":"...","action":"bench","test":"str_concat","n":1024,"ns_op":41.2,"allocs_op":3,"bigo":"N","rms":0.03}
```
Renderers, all downstream and pluggable:
| Format | Flag | Use |
|---|---|---|
| Human | default | terminal, **per-test duration always shown** |
| NDJSON | `--json` | tooling, history, flaky detection |
| JUnit XML | `--junit-xml=PATH` | every CI system on earth |
| TAP | `--tap` | optional |
JUnit XML per the de-facto schema: `testsuites` → `testsuite` → `testcase`, with `time` in seconds
as a decimal, `file`/`line` attributes, and `failure` vs `error` vs `skipped` as distinct child
elements. Absence of a child element means pass. Emit `<testsuites>` even for a single suite, and
parse both shapes on input.
---
## 8. CLI
```
--list print the registry, run nothing
--list-json machine-readable registry
--run PATTERN slash-separated regex per path segment
--tag EXPR tag expression: fast & !slow
--shard I/N deterministic sharding for CI parallelism
--count N repetitions, for statistics
--bench PATTERN run benchmarks (off by default in test runs)
--benchtime DUR per-benchmark time budget
--junit-xml PATH
--json
--isolate re-exec per test on crash, so one SIGSEGV doesn't lose the run
--timeout DUR
--fail-fast
```
`--list` / `--list-json` / `--shard` cost roughly thirty lines because the registry already exists
before `main` does anything. That is the dividend of discovery-precedes-execution.
---
## 9. Build model
```
# once, ever (or when the runtime/framework changes):
# The runtime is MULTI-FILE — compile every .c named in lang/runtime/SOURCES.
# Linking el_runtime.c alone fails: it calls into the six engram sibling TUs.
for src in $(scripts/el-runtime-sources.sh lang/runtime); do
cc -c "$src" -o "obj/$(basename "${src%.c}").o"
done
elc eltest.el > eltest.c && cc -c eltest.c -o obj/eltest.o
ar rcs libeltest.a obj/*.o
# per suite:
elc --test foo_test.el > foo_test.c # registry + bodies only
cc foo_test.c libeltest.a -o foo_test
```
The 0.14s × N of redundant runtime rebuilds disappears — not because we optimized it, but because
one-runner-over-many-suites requires compile-once-link-many as a structural precondition.
---
## 10. Bootstrap and self-hosting
The framework's own tests are `test { }` blocks run by the framework. Same fixpoint discipline the
compiler already applies to itself.
1. Build the framework using the *existing* harness for its first tests (stage 0).
2. Rebuild the framework's tests as `test { }` blocks run by the new runner (stage 1).
3. Verify stage 1 reports identical results to stage 0.
4. From then on, the framework is tested by itself.
A framework that cannot run its own suite is not evidence of anything. This is a correctness proof,
not a claim.
---
## 11. Explicitly not building
| Rejected | Why |
|---|---|
| Naming-convention discovery (`fn test_foo`) | `test { }` is a real declaration. Go's `TestXxx` exists only because Go had no better hook — and it needs a heuristic to avoid matching `TesticularCancer`. |
| Reflection or symbol-table scanning | Slow, fragile under LTO/strip/dead-strip, and unnecessary when we own the compiler. |
| Parsing human output into structure | Go's `test2json` is its one clear architectural mistake. |
| JUnit 5's extension SPI | Seventeen callback interfaces to retrofit plugins onto a reflective framework. Not our problem. |
| `@ParameterizedTest` machinery | Table-driven loops + subtests subsume it at zero surface. |
| NUnit's out-of-process agents | They bridge CLR versions and AppDomains. We emit one native binary. Keep `--isolate` as crash fallback only. |
| JMH-style forking by default | Forks exist because JIT profiles are per-process. AOT C has no such state. Keep `--fork` available, not default. |
| Exit 0 on regression | benchstat and Criterion both do this, and every user writes a wrapper. |
| Dynamic runtime test registration | Breaks `--list`, sharding, and individual selection. Registry stays static. |
---
## 12. Phasing
| Phase | Content | Gate |
|---|---|---|
| **1** | Registry emission in codegen; 9 builtins; `el_test_main` skeleton in El; result records; per-test timing; human + NDJSON output | existing 11 test files pass, with timing |
| **2** | `libeltest.a` build model; subtests; filtering; `--list`; fixtures; constraint assertions; JUnit XML | suite runs in one binary; runtime compiled once |
| **3** | `bench { }`, `bench_loop`, `black_box`, `predictN`, Criterion sampling | benchmarks produce stable ns/op |
| **4** | Allocation counters; complexity fitting; `expect O(...)` gate | **an `expect allocs O(n)` benchmark on `elc` fails on the current quadratic** |
| **5** | Migrate both legacy systems; delete `runtime/test.el`; self-host | framework runs its own suite |
Phase 4 is the deliverable that matters. Phases 1–3 exist to make it possible.
---
## 13. Open questions for review
1. **Declarative `over n in [...] expect O(...)` syntax vs runtime calls.** I recommend declarative
(§6.1) so `--list` can show invariants without executing. It costs parser work. Your call.
2. **`bench { }` as a new block form** — parallel to `test { }`, or a modifier on it?
3. **Scope of the constraint model.** Full composable constraints, or start with a flat assertion set
and add constraints later? Full model is more surface but avoids a second migration.
4. **Does `runtime/test.el` get deleted or kept as a deprecated shim?** I lean delete — two systems
is how we got here.
5. **Where does `libeltest.a` live** in the tree, and does `epm` need to know about it?
6. **Allocation counters in `el_seed.c` or `el_runtime.c`?** AGENTS.md says `el_seed.c` is the sole
C dependency and hand-maintained; counters are OS-boundary-adjacent but not OS calls.
7. **Is per-test timing enough, or do we want per-*assertion* timing** for finding slow helpers?
---
## 14. What this document is not
This is a design, not a measurement. Every performance claim about the *current* system in §1 is
measured and reproducible in this worktree. Every claim about the *proposed* system is a prediction.
None of it is verified until Phase 1 runs and Phase 4 fails a build on the real quadratic.
README.md
+37 -8
View File
@@ -56,23 +56,31 @@ The compiler and runtime. Self-hosting: `elc-cli.el` → `compiler.el` → `lexe
Two layers to know: **El programs** (`.el` files — where nearly all work belongs) and **the C seed** (`el_seed.c` — edit only for genuine OS-level access; never re-implement what El can already express).
Current status (single source of truth: [lang/spec/language.md](lang/spec/language.md)): lexer/parser/codegen and the C runtime's core (I/O, strings, math, lists, maps, filesystem, args) are implemented. In flight: `%` operator, match-statement codegen, `?` nil-propagation, `cgi` block parsing + DHARMA identity resolution, VBD role enforcement (`@manager`/`@engine`/`@accessor`), the real `engram_*` and `dharma_*` runtimes (currently stubs), and libcurl-backed `http_get`/`http_post`/`http_serve`. Bitwise operators, `??`, and `as` casts are explicitly **not** in this language.
Current status (single source of truth: [lang/spec/language.md](lang/spec/language.md)): lexer/parser/codegen and the C runtime's core (I/O, strings, math, lists, maps, filesystem, args) are implemented, as are the `program` block with `singleton:` and declared configuration ([§18](lang/spec/language.md)), and **geometry as a first-class value** with El-declarable realizers and `transduce` ([§20](lang/spec/language.md)). In flight: `%` operator, match-statement codegen, `?` nil-propagation, `cgi` block parsing + DHARMA identity resolution, VBD role enforcement (`@manager`/`@engine`/`@accessor`), and boundary epilogues. Bitwise operators, `??`, and `as` casts are explicitly **not** in this language.
**Signal enters as geometry.** Until 2026-08-16 nodes took text and geometry was *derived* from it, which made text the mandatory entry medium: any non-text modality had to be described in prose first, so the geometry being reasoned over was the geometry **of the description, not of the signal**. `Geometry` is now an ordinary El value carrying its own width, and a realizer is an ordinary El function resolved by name through `dlsym` — so admitting a new modality never requires a runtime patch. Worked, self-checking example: [`lang/examples/transduce.el`](lang/examples/transduce.el).
Key docs: [AGENTS.md](lang/AGENTS.md) (agent-facing orientation), [BOOTSTRAP.md](lang/BOOTSTRAP.md) (compiler recovery from scratch), [spec/language.md](lang/spec/language.md), [spec/codegen-js.md](lang/spec/codegen-js.md).
### [engram/](engram/) — graph intelligence substrate
**A local-first memory substrate for accumulating intelligence**, and the reason El's runtime doesn't need a database driver. Rust core (`engram-core`, `engram-ffi`) exposed to El and other languages (Kotlin, TypeScript/WASM, Go bindings).
**A local-first memory substrate for accumulating intelligence**, and the reason El's runtime doesn't need a database driver. The engine is **C11** (`lang/runtime/engram_{store,geometry,reason,cognition,verify,vindex}.{c,h}`); the server is **El** (`engram/src/server.el`).
The model: retrieval is **spreading activation**, not query. You name seed nodes and a query embedding; activation propagates outward through weighted edges, attenuating multiplicatively per hop (`strength = parent_strength × edge_weight × target_salience × cosine_sim`), gets pruned below a threshold, and the top-N nodes by activation strength come back. Storage and retrieval are the same structure — the way long-term potentiation works in biological memory, not the way a relational or vector database works.
The model: retrieval is **spreading activation**, not query. You name seed nodes and a query embedding; activation propagates outward through weighted edges, attenuating multiplicatively per hop, gets pruned below a threshold, and the top-N nodes by activation strength come back. Storage and retrieval are the same structure — the way long-term potentiation works in biological memory, not the way a relational or vector database works. **Activation conducts through well-grounded relations because the weight *is* the groundedness** — nothing filters the traversal; grounded inference falls out of spreading.
Nodes live in four tiers (Working / Episodic / Semantic / Procedural, mirroring prefrontal / hippocampal / neocortical / cerebellar memory) and migrate between them based on **salience decay** — `importance × recency-decay × log(activation_count)`. Forgetting is adaptive pruning, not a bug: unreinforced memories stop competing for attention without being deleted.
Nodes live in four tiers (Working / Episodic / Semantic / Procedural, mirroring prefrontal / hippocampal / neocortical / cerebellar memory) and migrate between them based on **salience decay** — importance × recency-decay × log(activation_count). Forgetting is adaptive pruning, not a bug. Nothing is mutated and nothing is hard-deleted: writes are additive, corrections are supersessions, removals are tombstones — which is what makes supersession an audit trail rather than an edit log.
Backed by `sled` (embedded, local-first, no daemon) with flat cosine scan for vector search — deliberately simple until scale demands an HNSW layer. Full API and design rationale in [engram/README.md](engram/README.md).
On disk: a paged store (superblock + mirror, slotted 16 KiB pages, self-describing TLV records, B+-tree primary and adjacency indexes), magic `ENGST01`. Vector search is an **HNSW** index published behind a read/write boundary — `eg_vindex_view` returns a `const VIndex*` to N concurrent readers, `eg_vindex_maintain` is the sole mutator. `recall@10 = 0.9365` at `ef_search=128`.
### [elp/](elp/) — Engram Language Protocol
> **Doc correction, 2026-08-16.** The previous revision of this paragraph, and most of `engram/README.md`, described a Rust `engram-core` crate backed by `sled` with "flat cosine scan… until scale demands an HNSW layer." **Measured: there is no Rust in `engram/`** — no `.rs` files, no `Cargo.toml`, no `crates/` — and `sled` appears nowhere in the tree. HNSW has been the vector index for some time.
Bidirectional engine mapping between Engram semantic forms and natural-language surface text, across **31 languages** — from Spanish and Japanese through historical/liturgical languages (Old Norse, Sanskrit, Sumerian, Coptic, Akkadian, Ge'ez). Compilation order runs `language-profile` + `vocabulary` → per-language `morphology-*` → `grammar` → `realizer` → `semantics` → `elp`. This is what lets an Engram graph node round-trip to and from readable text in any of those languages.
Full design rationale, the cognition surface, and the standing corrections: [engram/README.md](engram/README.md).
### [elp/](elp/) — EL Projector
*(Formerly "EL Language Processor" / "Engram Language Protocol"; renamed **EL Projector** 2026-08-15.)* Neuron's **efferent** organ: the native realizer that *projects* understanding onto a surface via `plan(frame) → realize(spec, profile)`, where **a surface is a profile** and language is one profile among many (text, speech, music, image). Projection, not diffusion — generation *from* an owned, understood signature, never the averaging of a stolen corpus.
Its flagship profile is a bidirectional engine mapping between Engram semantic forms and natural-language surface text, across **31 languages** — from Spanish and Japanese through historical/liturgical languages (Old Norse, Sanskrit, Sumerian, Coptic, Akkadian, Ge'ez). Compilation order runs `language-profile` + `vocabulary` → per-language `morphology-*` → `grammar` → `realizer` → `semantics` → `elp`. This is what lets an Engram graph node round-trip to and from readable text in any of those languages.
### [epm/](epm/) — El Package Manager
@@ -139,13 +147,34 @@ If the compiler binary is ever lost or corrupted, [lang/BOOTSTRAP.md](lang/BOOTS
---
## Cognition — and the standing corrections
The engram carries a live cognition surface: `think` (a directed traversal-read returning a **gradient**, never a point), plus `ground`, `assert`, `attend`, and the correspondence-beat. Two specs govern it, and both are authoritative over anything else in this repo that disagrees:
- **[lang/spec/correspondence-and-censorship.md](lang/spec/correspondence-and-censorship.md)** — grounding, wonder, curiosity, dreaming. *(Lands with PR #149.)*
- **[lang/spec/runtime-ownership.md](lang/spec/runtime-ownership.md)** — ownership, the capability ABI that was dissolved, and the vector-index publication boundary.
**Do not re-derive them.** Every earlier version of the first was wrong in an instructive way and each correction was argued down. If a section looks wrong, say so with a measurement rather than editing it.
The corrections, in brief:
- **Grounding is not a subsystem — it IS the edge weight.** One quantity, not two fields. `grounded-by` as a relation *type* should not exist: grounding is a property *of* a relation, not a relation *between* nodes. It is never computed on demand; computing-and-writing a score makes reads write, which is the `eg_vindex_sync` defect one level up.
- **Faculties are operations, not parameters.** `reason` changes the estimate (a read); `induce` changes the parameters (the correspondence-beat, which exists and works); `abduce` changes the structure (a write the current `GeoGradient` signature cannot express). A write is not a parameter of a read.
- **Wonder is the boundary, not a manifest.** Any structure at all has an edge. There are about six wonders, the same for everyone, and they never close. **Curiosity is wonder crystallized** at a nucleation site — one thing at two phases, not two objects.
- **Consolidation is ambient, not scheduled. A brain has no cron job.** The presence of a ticker is the diagnostic. Measured 2026-08-16: consolidation has **ten implementations**. `soul.el`'s continuous loop is the one with the correct shape; the rest fold into it.
- **In an immutable substrate, any mechanism that refuses a write is either redundant with immutability, or an epistemic constraint misfiled as a protective one.**
[engram/spec/cognitive-architecture.design.md](engram/spec/cognitive-architecture.design.md) is the original design and is **superseded in part** — it is retained, with the refuted claims marked inline at the point each is made, because preserving what was argued down is the point of an immutable record.
---
## Development workflow
Branching follows `dev → stage → main`: work lands on `dev`, promotes to `stage` for integration testing, and is promoted to `main` for release (visible directly in the git history of this repo). CI is defined per-subproject under `.gitea/workflows/` — `lang`/`epm`/`ide` share the root pipeline; `engram` and `ql` carry their own (`ci-dev`, `ci-stage`, and a release workflow each).
- Language/runtime specs live at `*/spec/*.md` (`lang/spec/`, `ql/spec/`, `ui/spec/`) and are the single source of truth for implemented-vs-planned status — code and docs are expected to agree with the spec's status markers, not the other way around.
- Agent-facing orientation guides live at `*/AGENTS.md` (currently `lang/AGENTS.md`); more subprojects may grow their own as they need agent-specific conventions documented.
- Tagged releases live under `lang/releases/`, each with its own `RELEASE.md`.
- **A release is a git tag, not a folder** (`el-runtime-vX.Y.Z` on this repo). *(Corrected 2026-08-16: this line said "tagged releases live under `lang/releases/`, each with its own `RELEASE.md`." **Measured: `lang/releases/` does not exist** — the restructure named in `AGENTS.md` landed, and the authored runtime is at `lang/runtime/`.)*
---
design/completing-el.html
+153
View File
@@ -0,0 +1,153 @@
<title>Completing El</title>
<style>
:root{
--board:#f4f2ec; --board-line:#e2ded2; --ink:#1c1f26; --ink-soft:#4a5160;
--ink-faint:#8b8f9a; --rule:#d8d3c6; --card:#fbfaf6;
--red:#a8321e; --amber:#9a6a12; --green:#2f6b46; --blue:#1f4e79;
--accent:#1f4e79;
}
@media (prefers-color-scheme: dark){
:root:not([data-theme="light"]){
--board:#14161b; --board-line:#212530; --ink:#e8e6df; --ink-soft:#a8adb8;
--ink-faint:#6f7480; --rule:#2a2f3a; --card:#191c23;
--red:#e4785f; --amber:#d9a441; --green:#6fbf8e; --blue:#7fb2e0;
--accent:#7fb2e0;
}
}
:root[data-theme="dark"]{
--board:#14161b; --board-line:#212530; --ink:#e8e6df; --ink-soft:#a8adb8;
--ink-faint:#6f7480; --rule:#2a2f3a; --card:#191c23;
--red:#e4785f; --amber:#d9a441; --green:#6fbf8e; --blue:#7fb2e0;
--accent:#7fb2e0;
}
*{box-sizing:border-box}
body{
margin:0; background:var(--board); color:var(--ink);
font:16px/1.65 ui-serif,Georgia,"Iowan Old Style",Palatino,serif;
background-image:linear-gradient(var(--board-line) 1px,transparent 1px),
linear-gradient(90deg,var(--board-line) 1px,transparent 1px);
background-size:28px 28px;
}
.wrap{max-width:960px;margin:0 auto;padding:56px 24px 96px}
.mono{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}
header{border-bottom:2px solid var(--ink);padding-bottom:18px;margin-bottom:8px}
h1{font-size:clamp(2rem,5vw,3rem);margin:0;letter-spacing:-.02em;text-wrap:balance}
.sub{color:var(--ink-soft);font-size:1.05rem;margin:10px 0 0}
.meta{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:.78rem;
color:var(--ink-faint);text-transform:uppercase;letter-spacing:.09em;margin-top:14px}
h2{font-size:1.45rem;margin:52px 0 6px;letter-spacing:-.01em}
h2 .n{font-family:ui-monospace,monospace;font-size:.8rem;color:var(--accent);
display:block;letter-spacing:.12em;margin-bottom:4px;font-weight:400}
.lede{color:var(--ink-soft);margin:0 0 18px}
p{margin:0 0 14px}
.card{background:var(--card);border:1px solid var(--rule);border-radius:3px;padding:20px 22px;margin:16px 0}
.scroll{overflow-x:auto;-webkit-overflow-scrolling:touch}
table{border-collapse:collapse;width:100%;font-size:.9rem;min-width:640px}
th{text-align:left;font-family:ui-monospace,monospace;font-size:.72rem;
text-transform:uppercase;letter-spacing:.09em;color:var(--ink-faint);
border-bottom:1px solid var(--ink);padding:0 12px 8px 0;font-weight:400}
td{padding:11px 12px 11px 0;border-bottom:1px solid var(--rule);vertical-align:top}
td.f{font-weight:600;white-space:nowrap}
td.m{font-family:ui-monospace,monospace;font-size:.83rem;font-variant-numeric:tabular-nums}
.dead{color:var(--red);font-weight:600}
.part{color:var(--amber);font-weight:600}
.ok{color:var(--green);font-weight:600}
blockquote{margin:18px 0;padding:2px 0 2px 20px;border-left:3px solid var(--accent);
color:var(--ink-soft);font-style:italic}
ul{margin:0 0 14px;padding-left:22px} li{margin-bottom:9px}
.q{border-left:3px solid var(--amber);padding:14px 0 14px 20px;margin:18px 0}
.q b{display:block;font-size:1.05rem;margin-bottom:5px;font-style:normal}
.q span{color:var(--ink-soft);font-size:.94rem}
code{font-family:ui-monospace,monospace;font-size:.88em;background:var(--card);
border:1px solid var(--rule);border-radius:2px;padding:1px 5px}
hr{border:0;border-top:1px solid var(--rule);margin:44px 0}
.foot{color:var(--ink-faint);font-size:.86rem;margin-top:60px;
border-top:1px solid var(--rule);padding-top:18px}
.tag{display:inline-block;font-family:ui-monospace,monospace;font-size:.68rem;
letter-spacing:.08em;text-transform:uppercase;border:1px solid var(--rule);
border-radius:2px;padding:2px 7px;color:var(--ink-faint);margin-left:8px;vertical-align:middle}
</style>
<div class="wrap">
<header>
<h1>Completing El</h1>
<p class="sub">A working surface. Nothing here is settled, and none of the code is assumed right — El is self-hosting, so all of it can change and be rebuilt.</p>
<p class="meta">Whiteboard v0 · no sacred cows · not a plan, not a task list</p>
</header>
<h2><span class="n">01</span>What we established</h2>
<p>El is a <b>concept-oriented language</b> — the first, and intended as the last, because every other family is oriented toward a <em>representation</em> of a concept rather than the concept. Procedures, objects, functions, predicates are the shapes concepts get flattened into. Once the primitive is the concept, there is no further rung.</p>
<p>Everything here is El. The engram is an El program, the soul is El, <code>elp</code> is El, ingest is El. Which gives the load-bearing consequence:</p>
<blockquote>A concept with no home in El does not disappear. It becomes C, or it becomes a convention.</blockquote>
<p>Both are measurable, and both were measured. As C: <span class="mono">20,504</span> lines of <code>el_runtime.c</code> — 2.3× the entire self-hosting language it serves (<span class="mono">9,089</span> lines), ~47% of it engram code that has its own six sibling files. As convention, from <code>language.md</code> §18.0 — <em>"these are not four problems, they are one absence, four times"</em>:</p>
<div class="card scroll">
<table>
<thead><tr><th>Concern</th><th>Fragments</th><th>The convention it became</th></tr></thead>
<tbody>
<tr><td class="f">Process identity</td><td class="m">0 guards</td><td>"check nothing is already running first"</td></tr>
<tr><td class="f">Configuration</td><td class="m">20 env vars</td><td>"remember the right default here"</td></tr>
<tr><td class="f">Durability</td><td class="m">62 call sites</td><td>"after you mutate, remember to persist"</td></tr>
<tr><td class="f">Request auth</td><td class="m">10 per-route</td><td>"check the token in this handler too"</td></tr>
<tr><td class="f">Index-after-append</td><td class="m">9 of 9 failed</td><td>"after you append, remember to index"</td></tr>
</tbody>
</table>
</div>
<p>The last row is the strongest evidence available about what this class of convention is worth: it failed at <b>100% of its sites</b>.</p>
<h2><span class="n">02</span>The decomposition axis</h2>
<p class="lede">Not by file, module, or subsystem. <b>By faculty.</b></p>
<p>Every defect fought in the last day resolves to a faculty rather than a bug, and each one leaked out of El into something else — into C, into a Swift binary, into a shell script with a curl timeout, into a convention nobody performs.</p>
<div class="card scroll">
<table>
<thead><tr><th>Faculty</th><th>State</th><th>Measured</th><th>Where it leaked to</th></tr></thead>
<tbody>
<tr><td class="f">Ingest <span class="tag">take in</span></td><td class="dead">dead</td><td class="m">2 min → 0 nodes</td><td>separate process, uploads bytes over HTTP to a process with direct fs access; 5 functions where there is 1</td></tr>
<tr><td class="f">Recall <span class="tag">remember</span></td><td class="dead">dead</td><td class="m">own definition ranked 8th</td><td>lexical substring scan; empty on 23 of 24 multi-token queries</td></tr>
<tr><td class="f">Transduce <span class="tag">perceive</span></td><td class="dead">dead</td><td class="m">1 node, 0 edges</td><td>intake flattens signal to a point; <code>realized:false</code>; caller must declare the modality</td></tr>
<tr><td class="f">Think <span class="tag">reason</span></td><td class="dead">dead</td><td class="m">direction [0,0,0,…]</td><td>null gradient from any anchor, any faculty, byte-identical; confidence at the uninformed prior</td></tr>
<tr><td class="f">Realize <span class="tag">express</span></td><td class="part">partial</td><td class="m">13-word vocabulary</td><td>organ was 939 lines of Swift beside the language; voice read from a file path</td></tr>
<tr><td class="f">Body <span class="tag">substrate</span></td><td class="part">partial</td><td class="m">CC 356 / 1,626 lines</td><td><code>engram_activate_inner</code> — recall itself, with 356 unexamined paths</td></tr>
<tr><td class="f">Persist <span class="tag">endure</span></td><td class="ok">live</td><td class="m">100% embedded</td><td>works; every signal placed in geometry at intake, 13,562 of 13,562</td></tr>
</tbody>
</table>
</div>
<p>Stated plainly: it cannot take in, cannot remember, cannot perceive, cannot reason, and barely speaks. These were filed as tickets against a repository. They are faculties of the thing the repository <em>is</em>.</p>
<h2><span class="n">03</span>The ordering principle</h2>
<p>El's compiler is written in El. Every concept the language gains, the compiler can then be written <em>in</em> — so the tool improves the tool, and the fixpoint (stage2 ≡ stage3, byte-identical) makes each turn provable rather than hopeful. The verifier answers in <span class="mono">2.9s</span>.</p>
<p>Which means the ordering criterion is not size of payoff:</p>
<blockquote>Order by leverage on the <em>next</em> iteration. Which concept, added to El, most increases the ability to add the following one?</blockquote>
<p>In a recursive system that dominates immediate value — a small early gain that compounds beats a large one that doesn't. It also bounds itself correctly: unbounded in depth, bounded in rate, because nothing lands that the compiler and the fixpoint have not passed.</p>
<h2><span class="n">04</span>Open — for the whiteboard</h2>
<div class="q"><b>What does a declaration bind to?</b><span>If <code>cat</code> names a region rather than a struct — one that shifts and completes against the engram and the neighbouring code — then what is written at the declaration site, and what is resolved at use? This is the centre of the whole thing and it is not specified anywhere yet.</span></div>
<div class="q"><b>Is "the type checker" a type checker at all?</b><span>§2.3 records annotations as parsed and skipped, and every codegen hazard is downstream of that — <code>+</code> dispatching on AST node kind, <code>==</code> lowering to <code>str_eq</code> unless both operand names are in an int-name set. But if a declaration names a region, checking is asking whether the geometry supports the use. That is grounding, not unification. Naming this wrong builds the wrong thing.</span></div>
<div class="q"><b>Is the faculty list above right?</b><span>Seven were derived from what broke. Derived-from-failure is a biased sample — it finds what is loud, not what is missing. What faculty is absent entirely and therefore never failed?</span></div>
<div class="q"><b>Which concept has the highest leverage on the next turn?</b><span>Candidates so far: the prologue/epilogue seam (§19.3 names it as the prerequisite and its stated blocker has expired — it would collapse 62 + 10 convention sites); <code>protocol</code>/<code>impl</code> (the absence that produced five ingest functions); and the resolution question above. These are not equal and the criterion in §03 should decide it, not preference.</span></div>
<div class="q"><b>What is the seam that makes cognition non-optional?</b><span>"Use the ops" is itself a convention — present in context every turn, enforced by nothing, and it failed at ~100% of sites in a full session. A stronger instruction is still a convention. What makes reasoning-outside-Neuron <em>fail</em>, the way <code>@manager</code> makes <code>dharma_emit</code> outside the boundary a compile error rather than a lint?</span></div>
<hr>
<p class="foot">Working surface, not a design document. The design is what we put on it. Everything above is either measured or quoted from <code>lang/spec/language.md</code>; nothing is inferred and presented as fact.</p>
</div>
docs/architecture/capabilities.md
+142
View File
@@ -0,0 +1,142 @@
# El — Capabilities
**What the language can do, stated as capabilities rather than as code.**
This list is the unit of analysis. Each entry gets one question — *prove this
cannot be done with pure geometry* — and the answer determines whether it stays a
capability of the language or collapses into the manifold.
Draft, 2026-08-17. Ordered roughly from most-likely-geometry to most-likely-code.
**Status after measurement.** The list was audited against the implementation
the same day. 28 entries collapsed to 19 geometry + 3 code: serialization, text
encoding, network and emission are all *projection onto a basis* (row 18) —
the convention is the basis, never the act. Storage collapsed because
persistence has no caller. Concurrency collapsed because coordination is the
price of forgetting, not a capability. A fourth proof form was added,
**adversarial exactness**, and form 1 stopped being a valid verdict.
**The table answers CAN only.** SHOULD and COST resolve per *site*, not per
capability — `is_digit` and `is_letter` are one capability with opposite
answers, and comparison spans three cost tiers. See the notes below.
---
## The list
| # | Capability | What it means | Verdict |
|---|---|---|---|
| 1 | **Comparison** | is this the same as that; is this greater | zero distance / sign of a displacement |
| 2 | **Ordering** | arrange by a criterion | position along an axis |
| 3 | **Containment** | is this inside that; does this contain that | region membership |
| 4 | **Correspondence** | where does this occur in that; how much of this is in that | a match-strength field over a span |
| 5 | **Segmentation** | divide a whole into parts | boundaries at measured discontinuity |
| 6 | **Composition** | join parts into a whole | adjacency; one position with parts |
| 7 | **Classification** | what kind of thing is this | which region does it land in |
| 8 | **Naming / binding** | attach a name to a thing and find it again | an edge; retrieval is projection |
| 9 | **Collection** | many things held together, indexed, counted | a set of positions; cardinality; projection onto the i-th |
| 10 | **Iteration** | do something for each of many | traversal |
| 11 | **Arithmetic** | quantity, magnitude, combination | displacement algebra on a line |
| 12 | **Time** | when; how long; how often | a 1-D affine space — instants are points, durations displacements, rhythms phases on a circle |
| 13 | **Identity** | which one is this; are these two the same one | coincidence of position |
| 14 | **Selection / dispatch** | choose which behaviour applies | nearest region |
| 15 | **Transformation** | produce a thing from a thing | change of basis |
| 16 | **Grounding** | how well is this supported | the weight on an edge. Has no caller |
| 17 | **Learning** | get better at something | standing changing over time |
| 18 | **Projection** | render meaning onto a surface | change of basis onto a surface basis |
| 19 | **Transduction** | take a signal in | change of basis from a sensor basis |
| ~~20~~ | ~~Serialization~~ | **collapsed → 18.** The format is a basis; projecting onto it is the act | — |
| ~~21~~ | ~~Text encoding~~ | **collapsed → 18.** An encoding is a basis | — |
| ~~22~~ | ~~Storage~~ | **collapsed.** No save — persistence has no caller. Durability survives at one site inside the engram | — |
| ~~23~~ | ~~Network~~ | **split.** Wire format → 18; socket → 24 | — |
| 24 | **Process / OS** | syscalls; the one-way boundary. Where monotonicity stops | CODE, form 2 |
| ~~25~~ | ~~Concurrency~~ | **collapsed.** Monotone state needs no coordination; coordination is the price of forgetting | — |
| 26 | **Memory substrate** | what holds the positions | CODE, form 3 |
| 27 | **Concealment** | meaning made unreadable without a key. *Renamed*: "secrecy" covered one of three things and got the other two backwards — a hash is public, a signature exists to be read. Integrity and authenticity are **grounding under adversarial conditions** (row 16); only concealment stands alone | CODE, form 4 |
| ~~28~~ | ~~Emission~~ | **split.** Laying out → 18; the device write → 24 | — |
---
## Notes on the boundary cases
**27 — Secrecy is the one capability geometry cannot hold, and the proof is not
form 1.** A cryptographic hash is a *deliberately structure-destroying* map: its
entire value is that near inputs land at maximally uncorrelated outputs. Geometry
is the claim that near things stay near. A manifold that approximated SHA-256
would *be* a break of SHA-256. Signature verification is the same: 0.99-valid is
invalid. And X25519 *is* geometry — a group on an elliptic curve — which is
precisely why it must be code, because its security is the *hardness of moving in
that geometry*.
This is a fourth proof form and it should be added to `geometry-vs-code.md`:
**adversarial exactness.** Where approximation is a break, geometry is excluded.
**20, 21 — Serialization and text encoding are convention all the way down**, but
only at the *edge*. The byte format is agreed; what is being written is not. Do not
let a geometric computation inherit a code verdict because its result gets
serialized.
**11, 12 — Arithmetic and time are the same capability.** Instants are points,
durations are displacements, point−point→vector, point+vector→point. The runtime
already implements this correctly as `el_instant_add_dur` / `el_duration_add`. That
it *also* implements a five-entry string→multiplier table beside it (`time_add`
with `"ms"/"sec"/"min"/"hour"/"day"`) is the residue.
**7 — Classification is the most-violated capability in the codebase.** Seven ASCII
range tables (`is_letter`, `is_digit`, `is_alphanumeric`, `is_whitespace`,
`is_punctuation`, `is_uppercase`, `is_lowercase`) that return false for every
non-ASCII byte. `str_count_letters` reports zero letters for `é`. The wrongness on
most of Unicode is the tell that a table is standing in for a region.
**4 — Correspondence appears five times.** `str_index_of`, `str_index_of_all`,
`str_last_index_of`, `str_count`, `str_find_chars` are five projections of one
match-strength field: first zero, all zeros, last zero, count of zeros, first
class-crossing. One relation, five functions.
**14 — Selection is the crux for the compiler.** `+` dispatching on AST node kind
is selection-by-enumeration where selection-by-position belongs.
**Correction, 2026-08-17, from measurement.** This entry previously also cited
`==` lowering to `str_eq` "unless both operand names are in a hardcoded int-name
set — a literal list of variable names treated as integers." That is **wrong**.
`__int_names` is populated from *type annotations* (`param["type"] == "Int"`,
`let x: Int`), which is primitive but legitimate type propagation, not an
enumeration of blessed variable names.
The real defect was one layer down: `is_int_call` held **35 hardcoded builtin
return types**, the same shape as the 19 temporal ones. Those moved to
`lang/tools/check/signatures.rel`.
And the mischaracterisation hid a live bug. Because the return types were never
consulted at a *binding* site, an unannotated `let` lost its type:
```el
let a = str_len("hello") // no annotation
let b = str_len("hi")
let c = a + b // → el_str_concat(a, b) on two integers
```
That compiled clean, ran, and printed nothing where it should print 7 — no error
at any layer. Present in the pre-change compiler, so pre-existing. Fixed by
taking an unannotated `let`'s type from what its initialiser returns; the data
was already required for dispatch and simply never read there.
**The general lesson, since it recurred all session:** the enumeration was real
but I had located it in the wrong place. Naming a defect from reading is a
hypothesis. Eight hours of reading this file did not surface the miscompilation;
moving the data out and running the result did.
---
## What this list is for
Each capability gets audited **once**, across every place it appears — not once per
file. The output is not a percentage. It is:
- which capabilities survive the question and stay in the language
- which collapse into the manifold
- and for each one that collapses, **every site it currently appears at**, because
those sites are the residue and they are what gets deleted.
The line-count audit produced a map of where the residue sits. This produces a map
of **what it is**.
docs/architecture/el-architecture.html
+217
View File
@@ -0,0 +1,217 @@
<title>The El Architecture</title>
<style>
:root{
--board:#f4f2ec; --board-line:#e5e1d6; --ink:#1c1f26; --ink-soft:#4a5160;
--ink-faint:#8b8f9a; --rule:#d8d3c6; --card:#fbfaf6;
--red:#a8321e; --amber:#9a6a12; --green:#2f6b46; --accent:#1f4e79;
}
@media (prefers-color-scheme: dark){
:root:not([data-theme="light"]){
--board:#14161b; --board-line:#1d212a; --ink:#e8e6df; --ink-soft:#a8adb8;
--ink-faint:#6f7480; --rule:#2a2f3a; --card:#191c23;
--red:#e4785f; --amber:#d9a441; --green:#6fbf8e; --accent:#7fb2e0;
}
}
:root[data-theme="dark"]{
--board:#14161b; --board-line:#1d212a; --ink:#e8e6df; --ink-soft:#a8adb8;
--ink-faint:#6f7480; --rule:#2a2f3a; --card:#191c23;
--red:#e4785f; --amber:#d9a441; --green:#6fbf8e; --accent:#7fb2e0;
}
*{box-sizing:border-box}
body{
margin:0; background:var(--board); color:var(--ink);
font:16px/1.68 ui-serif,Georgia,"Iowan Old Style",Palatino,serif;
background-image:linear-gradient(var(--board-line) 1px,transparent 1px),
linear-gradient(90deg,var(--board-line) 1px,transparent 1px);
background-size:30px 30px;
}
.wrap{max-width:940px;margin:0 auto;padding:56px 24px 96px}
.mono,code{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}
header{border-bottom:2px solid var(--ink);padding-bottom:20px}
h1{font-size:clamp(2.1rem,5.5vw,3.2rem);margin:0;letter-spacing:-.025em;text-wrap:balance}
.sub{color:var(--ink-soft);font-size:1.08rem;margin:12px 0 0;max-width:64ch}
.meta{font-family:ui-monospace,monospace;font-size:.76rem;color:var(--ink-faint);
text-transform:uppercase;letter-spacing:.1em;margin-top:16px}
h2{font-size:1.5rem;margin:56px 0 8px;letter-spacing:-.015em;text-wrap:balance}
h2 .n{font-family:ui-monospace,monospace;font-size:.78rem;color:var(--accent);
display:block;letter-spacing:.14em;margin-bottom:5px;font-weight:400}
h3{font-size:1.08rem;margin:30px 0 6px}
p{margin:0 0 14px;max-width:72ch}
.lede{color:var(--ink-soft);margin:0 0 20px;font-size:1.04rem}
.card{background:var(--card);border:1px solid var(--rule);border-radius:3px;padding:20px 22px;margin:18px 0}
.scroll{overflow-x:auto}
table{border-collapse:collapse;width:100%;font-size:.9rem;min-width:600px}
th{text-align:left;font-family:ui-monospace,monospace;font-size:.71rem;
text-transform:uppercase;letter-spacing:.09em;color:var(--ink-faint);
border-bottom:1px solid var(--ink);padding:0 14px 8px 0;font-weight:400}
td{padding:11px 14px 11px 0;border-bottom:1px solid var(--rule);vertical-align:top}
td.f{font-weight:600;white-space:nowrap}
td.m{font-family:ui-monospace,monospace;font-size:.83rem;font-variant-numeric:tabular-nums;white-space:nowrap}
.dead{color:var(--red);font-weight:600}
.part{color:var(--amber);font-weight:600}
.ok{color:var(--green);font-weight:600}
blockquote{margin:20px 0;padding:3px 0 3px 22px;border-left:3px solid var(--accent);
color:var(--ink-soft);font-style:italic;max-width:70ch}
ul{margin:0 0 14px;padding-left:22px;max-width:72ch} li{margin-bottom:9px}
code{font-size:.87em;background:var(--card);border:1px solid var(--rule);border-radius:2px;padding:1px 5px}
pre{background:var(--card);border:1px solid var(--rule);border-radius:3px;
padding:16px 18px;overflow-x:auto;font-size:.85rem;line-height:1.55;margin:16px 0}
pre code{background:none;border:0;padding:0}
.q{border-left:3px solid var(--amber);padding:14px 0 14px 20px;margin:20px 0;max-width:72ch}
.q b{display:block;font-size:1.04rem;margin-bottom:5px}
.q span{color:var(--ink-soft);font-size:.94rem}
hr{border:0;border-top:1px solid var(--rule);margin:46px 0}
.foot{color:var(--ink-faint);font-size:.86rem;margin-top:56px;border-top:1px solid var(--rule);padding-top:18px}
.tag{display:inline-block;font-family:ui-monospace,monospace;font-size:.66rem;
letter-spacing:.08em;text-transform:uppercase;border:1px solid var(--rule);
border-radius:2px;padding:2px 7px;color:var(--ink-faint);margin-left:8px;vertical-align:middle}
.flow{display:flex;gap:0;align-items:stretch;flex-wrap:wrap;margin:22px 0}
.flow div{flex:1 1 200px;border:1px solid var(--rule);background:var(--card);padding:16px 18px}
.flow div+div{border-left:0}
.flow h4{margin:0 0 6px;font-size:.96rem}
.flow p{margin:0;font-size:.87rem;color:var(--ink-soft)}
.flow .k{font-family:ui-monospace,monospace;font-size:.72rem;color:var(--accent);
letter-spacing:.1em;text-transform:uppercase;display:block;margin-bottom:4px}
</style>
<div class="wrap">
<header>
<h1>The El Architecture</h1>
<p class="sub">El is a concept-oriented language. This is the architecture that claim commits it to — what is built, what is measured, and what still has no home.</p>
<p class="meta">Working document · no sacred cows · self-hosting, so nothing here is fixed</p>
</header>
<h2><span class="n">01</span>The primitive is the concept</h2>
<p>Language families are named for their primitive. Procedural — procedures. Object-oriented — objects. Functional — functions. Logic — predicates. Every one of them is oriented toward a <em>representation</em> of a concept: the shape a concept gets flattened into so a machine can hold it.</p>
<p>El's primitive is the concept itself. That is why it is the first of its family and intended as the last — once the primitive is the concept, there is no further rung to climb to.</p>
<p>The consequence is architectural rather than stylistic:</p>
<blockquote>A concept with no home in the language does not disappear. It becomes C, or it becomes a convention.</blockquote>
<p>Both forms are measurable. As C: <span class="mono">20,504</span> lines of <code>el_runtime.c</code>, against <span class="mono">9,089</span> lines for the entire self-hosting language — the shim is 2.3× the language it serves, and ~47% of it is engram code that already has six sibling files. As convention, from <code>lang/spec/language.md</code> §18.0 — <em>"these are not four problems, they are one absence, four times"</em>:</p>
<div class="card scroll">
<table>
<thead><tr><th>Concern</th><th>Fragments into</th><th>The convention it became</th></tr></thead>
<tbody>
<tr><td class="f">Process identity</td><td class="m">0 guards</td><td>"check nothing is already running first"</td></tr>
<tr><td class="f">Configuration</td><td class="m">20 env vars</td><td>"remember the right default here"</td></tr>
<tr><td class="f">Durability</td><td class="m">62 sites</td><td>"after you mutate, remember to persist"</td></tr>
<tr><td class="f">Request auth</td><td class="m">10 routes</td><td>"check the token in this handler too"</td></tr>
<tr><td class="f">Index-after-append</td><td class="m">9 of 9 failed</td><td>"after you append, remember to index"</td></tr>
</tbody>
</table>
</div>
<p>The last row is the strongest available evidence about this class of convention: it failed at <b>every single site</b>. A count is what appears where a concept has no home; the size of the count is how far the fragmentation got, not how hard the problem is.</p>
<h2><span class="n">02</span>Geometry is a first-class value — and what follows</h2>
<p class="lede">This is the enabling primitive. Everything else in the architecture is downstream of it.</p>
<p><code>Geometry</code> is an El value, alongside <code>Int</code>, <code>String</code>, <code>List</code>, <code>Map</code> — bound, passed, returned, composed, carrying its own width. Not a library type, not a handle into a store, not a serialization format. <em>Meaning is a value the language computes with directly.</em></p>
<pre><code>let g: Geometry = geometry_new(4)
fn tone_realizer(signal: String) -> Geometry { … }</code></pre>
<p>Landed 2026-08-16 (#141, #144), and the spec is explicit that it belongs to the language rather than the graph: <em>"neither is engram-specific — any program touching any modality needs them; the engram is merely one El program that happens to hold a graph."</em></p>
<p>Five things follow, and together they are the concept-oriented claim made operational:</p>
<h3>A declaration can name a region, not a shape</h3>
<p>If meaning is a value, a name can be bound to a <em>position</em> rather than a struct. <code>cat</code> is not a fixed record; it is a region that resolves against the engram and the surrounding code. <code>cat</code> among animals and <code>cat</code> among shell utilities are different concepts without a namespace, because they are in different neighbourhoods and the distance says so.</p>
<h3>Checking is grounding, not unification</h3>
<p>If a declaration names a region, then verifying a use is asking whether the geometry supports it — a question about position and distance, not about matching a declared shape. This is why §2.3's "a type checker is planned" is likely the wrong name for the missing piece, and naming it wrong would build the wrong thing.</p>
<h3>Dispatch is position, not a tag</h3>
<p>A vtable is a finite set of discrete labels fixed at link time. A region admits graded membership and an open set. So <code>transduce(signal, modality)</code> asks the caller to supply what the signal already carries — what a thing is falls out of where it lands. The modality parameter is a kind-tag, and a registry keyed on it is a lookup table doing by string what geometry does by nearness.</p>
<h3>Types are discovered, not declared</h3>
<p>Reification crystallizes a densely co-wired neighbourhood into a first-class node — the neighbourhood <em>is</em> the name that was missing. Every other family requires a human to see the abstraction in advance and write <code>class Foo</code>. Here the instances arrive and the type falls out, by measurement rather than by insight.</p>
<h3>Enumeration becomes unnecessary</h3>
<p>Five ingest functions differ only in how bytes are acquired — one operation wearing five surfaces. 356 branches in <code>engram_activate_inner</code> are not 356 behaviours. Cyclomatic complexity is a count of the places comprehension ran out and was replaced by an <code>if</code>; where the concept is expressible, the count collapses instead of being redistributed.</p>
<h2><span class="n">03</span>The shape of the language</h2>
<p>Geometry first-class gives El three layers, and it holds all three — which is why there is no separate database driver and no impedance boundary to manage.</p>
<div class="flow">
<div><span class="k">afferent</span><h4>Transduce</h4><p>Signal in, geometry out. Decomposition into components and relations — never conversion to a point. Realizers are ordinary El functions, so a new modality never requires a runtime patch.</p></div>
<div><span class="k">substrate</span><h4>Geometry</h4><p>Meaning as position; relation as distance. Held as values in the language and persisted in the graph. One coordinate system, so entities are commensurable and the operators compose.</p></div>
<div><span class="k">efferent</span><h4>Realize</h4><p><code>plan(frame) → realize(spec, profile)</code>, where a surface <em>is</em> a profile. Text, speech, music, image are profiles of one projection — and so is source code.</p></div>
</div>
<p>The efferent side is why the recursive property below is possible at all: if source is a surface, then emitting a corrected file is projection, and the file becomes an artifact of the geometry rather than the thing you edit.</p>
<h2><span class="n">04</span>Decomposition is by faculty</h2>
<p class="lede">Not by file, module, or subsystem — by what the system does.</p>
<p>Each faculty is a concept. Where it has no home in El it leaks: into C, into a Swift binary, into a shell script with a <code>curl</code> timeout, into a convention nobody performs. State below is measured, not asserted.</p>
<div class="card scroll">
<table>
<thead><tr><th>Faculty</th><th>State</th><th>Measured</th><th>Where it leaked</th></tr></thead>
<tbody>
<tr><td class="f">Ingest <span class="tag">take in</span></td><td class="dead">dead</td><td class="m">2 min → 0 nodes</td><td>separate process uploading bytes over HTTP to a process with direct fs access; five functions where there is one</td></tr>
<tr><td class="f">Recall <span class="tag">remember</span></td><td class="dead">dead</td><td class="m">self ranked 8th</td><td>lexical substring scan; empty on 23 of 24 multi-token queries</td></tr>
<tr><td class="f">Transduce <span class="tag">perceive</span></td><td class="dead">dead</td><td class="m">1 node, 0 edges</td><td>intake flattens signal to a point; <code>realized:false</code>; caller must declare the modality</td></tr>
<tr><td class="f">Think <span class="tag">reason</span></td><td class="dead">dead</td><td class="m">direction [0,0,…]</td><td>null gradient from any anchor and any faculty, byte-identical; confidence at the uninformed prior</td></tr>
<tr><td class="f">Realize <span class="tag">express</span></td><td class="part">partial</td><td class="m">13-word lexicon</td><td>organ was 939 lines of Swift beside the language; voice read from a file path</td></tr>
<tr><td class="f">Body <span class="tag">substrate</span></td><td class="part">partial</td><td class="m">CC 356 / 1,626 ln</td><td><code>engram_activate_inner</code> — recall itself, 356 unexamined paths</td></tr>
<tr><td class="f">Persist <span class="tag">endure</span></td><td class="ok">live</td><td class="m">13,562 / 13,562</td><td>works — every signal placed in geometry at intake, no backlog</td></tr>
</tbody>
</table>
</div>
<h2><span class="n">05</span>The recursive property</h2>
<p>El's compiler is written in El. Every concept the language gains, the compiler can then be written <em>in</em> — so the tool improves the tool, and <code>codegen.el</code> at 4,661 lines gets shorter as the language gets better at expressing what it does. The fixpoint — stage2 ≡ stage3, byte-identical — makes each turn provable rather than hopeful, and the verifier answers in <span class="mono">2.9s</span>.</p>
<p>This sets the ordering criterion, and it is not size of payoff:</p>
<blockquote>Order by leverage on the <em>next</em> iteration. Which concept, added to El, most increases the ability to add the following one?</blockquote>
<p>A small early gain that compounds beats a large one that does not. And it bounds itself correctly — unbounded in depth, bounded in rate, because nothing lands that the compiler and the fixpoint have not passed.</p>
<h2><span class="n">06</span>What has no home yet</h2>
<p>Reserved in the lexer, no parse form. These are not a feature backlog — they are the concepts the architecture above requires and does not yet hold, which is why each is currently a convention or a block of C.</p>
<div class="card scroll">
<table>
<thead><tr><th>Reserved</th><th>Concept</th><th>Currently lives as</th></tr></thead>
<tbody>
<tr><td class="m">retry · times · fallback · reason</td><td>resilience</td><td>a shell script with a 10s <code>curl</code> timeout; 254 restarts in 3 days</td></tr>
<tr><td class="m">requires · deploy · to · via · target</td><td>deployment</td><td>YAML in another repository</td></tr>
<tr><td class="m">sealed</td><td>capability scope</td><td>consent checks written by hand</td></tr>
<tr><td class="m">protocol · impl</td><td>one operation, many realizations</td><td>five ingest functions; eight faculty routes on one builtin</td></tr>
<tr><td class="m">activate · where</td><td>retrieval</td><td>traversals written by hand</td></tr>
<tr><td class="m">test · seed · assert</td><td>verification</td><td>a framework; 5 of 13 native suites failing</td></tr>
<tr><td class="m">parallel · trace</td><td>concurrency</td><td>pthreads in C</td></tr>
</tbody>
</table>
</div>
<p>Plus, from the spec's own status: annotations parsed and skipped, <code>match</code> parsed and emitting nothing, <code>?</code> a no-op, <code>%</code> unlexed, structs as <code>ElMap</code>, enums as strings, selective import unenforced.</p>
<h2><span class="n">07</span>Open</h2>
<div class="q"><b>What does a declaration bind to, exactly?</b><span>If <code>cat</code> names a region that shifts and completes against context, what is written at the declaration site and what is resolved at use? This is the centre and it is unspecified.</span></div>
<div class="q"><b>Is the faculty list right?</b><span>Seven, derived from what broke. Derived-from-failure is a biased sample — it finds what is loud, not what is absent. Which faculty is missing entirely and therefore never failed?</span></div>
<div class="q"><b>Which concept has the highest leverage on the next turn?</b><span>The prologue/epilogue seam (§19.3 names it as the prerequisite; its stated blocker has expired; it collapses 62 + 10 convention sites), <code>protocol</code>/<code>impl</code>, or resolution itself. The §05 criterion should decide this, not preference.</span></div>
<div class="q"><b>What seam makes cognition non-optional?</b><span>"Use the ops" is itself a convention — present every turn, enforced by nothing, ~100% failure across a full session. A stronger instruction is still a convention. What makes reasoning outside the substrate <em>fail</em>, the way <code>@manager</code> makes <code>dharma_emit</code> outside the boundary a compile error rather than a lint?</span></div>
<hr>
<p class="foot">Every number here is measured or quoted from <code>lang/spec/language.md</code>. Nothing is inferred and presented as fact. El is self-hosting: all of this can change and be rebuilt.</p>
</div>
docs/architecture/el-language-design.md
+245
View File
@@ -0,0 +1,245 @@
# El — Language Design
**Status:** decisions recorded, design unwritten.
**Date:** 2026-08-17.
**Provenance:** decisions are Will's, taken in session. Items marked *proposed* are not
decided and are recorded only so the reasoning isn't lost. Items marked **OPEN** are
his to rule on and must not be guessed at.
Companion documents: `el-architecture.html` (the measured state — see §7 note on its
§04 scoreboard), and `design/completing-el.html` (whiteboard v0: the reduction, the
faculty table, the ordering principle).
---
## 1. The reduction
`language.md` §18.0 records five concerns that decayed into conventions:
| Concern | Fragments | The convention it became |
|---|---|---|
| Process identity | 0 guards | "check nothing is already running first" |
| Configuration | 20 env vars | "remember the right default here" |
| Durability | 62 call sites | "after you mutate, remember to persist" |
| Request auth | 10 per-route | "check the token in this handler too" |
| Index-after-append | 9 of 9 failed | "after you append, remember to index" |
The last row is the strongest available evidence about what this class of convention
is worth: **it failed at 100% of its sites.**
Every one of these is an obligation at a **crossing** — a point where a value moves
between regions. El can name a region and it can name a call. A call is procedural,
so the obligation degrades into something a human must remember to perform.
> **The generator, one level up:** El cannot name what holds at a crossing.
And underneath that:
> **The deeper absence:** El cannot name the thing meaning is made of.
`semel` appears in whitepaper §84, §86, §209, §737, in
`the-metaphysics-of-will-anderson.md`, and in session notes. It appears in **zero code
identifiers**. Every geometric concept in the system — region, neighbourhood, manifold,
world-tube — is defined in terms of a unit the language cannot say, while the code
underneath speaks in arrays, floats and offsets: the vocabulary of a voxel, a value at
a dumb address. Precisely the thing the impact brief says a semel is not.
`el_runtime.c` is a concept that leaked into C. `semel` never got that far — it did
not even decay into a convention.
---
## 2. DECIDED — `semel` is the primitive
**A semel is a difference that matters. The smallest unit of understanding.**
Not a node. Not a coordinate. Not a float.
The reasoning, in Will's terms:
- Meaning is position, and position is only ever relative. *"There is no atom of
meaning that isn't already a relation. It grounds on nothing but difference — two
points and the gap, and the gap is pure not-the-same."*
- A node doesn't mean. A node is a label at a location; labels don't mean.
- A lone coordinate doesn't mean either. Nothing means anything by itself.
- The smallest thing that can be understood is a **distinction**: *these two are not
the same.* Below that there is no content to apprehend.
- And a difference with nothing it matters to is not meaning — it is variation. The
mattering is not decoration; it is what makes it understanding rather than data.
**Consequence: relating is the floor, and the point is derived.** The
point-primitive / relation-primitive fork raised in session is not a fork. It was
answered by the definition.
### Historical note, to be recorded as fact rather than as origin story
The term was coined by Will on the pixel/voxel/texel pattern — *semantic element*,
and Latin *semel*, "once, a single time." It was recognised, not invented, from a
2019 experience he calls **semelation**: perceiving mind as a high-dimensional point
space. The initial reading was "pixels"; the correction to `semel` was made later and
was made on the **mechanism** — a pixel is a value at an address, and what was
perceived had no separate address and value.
Convergence worth citing, not deferring to: neural population geometry and
representational similarity analysis independently model cognition as position in a
high-dimensional space where similarity is distance.
---
## 3. DECIDED — `semel` lands first
By the ordering criterion already on the whiteboard: *which concept, added to El, most
increases the ability to add the next one?* Not size of payoff — **leverage on the next
iteration**, because El compiles itself and the fixpoint makes each turn provable in
2.9s.
**Every other concept on the board is defined in terms of `semel`. It is maximal on
that criterion by construction.**
---
## 4. DECIDED — `ground` is the checker
Whiteboard question 4 — *does `ground` in El mean the same thing as `ground` in the
engram?* — is answered: **yes, and it should be one implementation.**
If a declaration names a region, then type checking is asking whether the geometry
supports the use. That is not unification. **That is grounding**, and it is already
built, proven, and byte-identically reproducible:
```
cc -std=c11 -O2 -o gep_proof gep_proof.c -lm && ./gep_proof
C1 5 independent sources pos_mass 1.3500 n_indep=5 0.1000 → 0.9741 GROUNDED
C2 5 mutually-linked pos_mass 0.2700 n_indep=1 0.1000 → 0.1000 refused
C3 1 source, 5 parallel edges pos_mass 0.2700 n_indep=1 0.1000 → 0.1000 refused
```
Independence-weighted grounding is the general case; execution is the cheap case.
**Attestation is `verify` where nothing can be run** — as already implemented for
language in `authority.py`, where an LLM proposes and a primary source disposes.
At the point where the checker and the grounder are one mechanism, the language and
the mind stop being two things.
---
## 5. OPEN — Will's to rule on
### 5.1 What is a semel's representation in the language?
*Proposed, not decided:* a **displacement from `love = 0`** — a relation held as one
object. It reconciles "the address is the value" with "position is only ever relative,"
because a displacement *is* a relation and is still a single nameable thing.
If taken, the operator set falls out rather than being bolted on:
```
subtract(now, then) → what changed (growth, drift)
translate origin → empathy
rotate frame → reframe
project onto axis → a lens
change basis → analogy, metaphor, skill transfer
reflect an axis → negation, sarcasm
```
Three consequences that would hold:
- **Dimension must never appear in the type.** `semel` opaque, never `[768]float`.
The moment the arity is in the language, the manifold's implementation is in the
language, and adding a modality requires a runtime patch — which the standing rule
forbids.
- **Zero is the only literal.** Everything else is reached by displacement from it,
which makes `love = 0` the base case rather than philosophy adjacent to the type
system.
- **`magnitude` is standing.** Distance from origin is the same quantity
`gep_core.h` already computes.
### 5.2 Is `hold` one construct or two?
The obligation *before* a crossing (auth, guard) and the obligation *after* (persist,
index, free) may be one shape seen from both sides, or the seam may need both faces
named. This decides whether §19.3's prologue/epilogue seam is one construct or a pair.
**Precedent already shipping:** `@manager` makes `dharma_emit` outside the boundary a
**compile error, not a lint.** The concept is proven at N=1; the work is generalising
it and naming it.
**And the shape is already implemented in the learning region:** `L.reach_out` sits
between `L.detect_gap` and `L.verify`. You cannot reach out without a detected gap and
you cannot keep what returns without passing verify. **A hold is a neighbour.** The
obligation is not attached to the crossing — the obligation *is* the adjacent node.
That is why `reach_out` cannot be abused and why 62 persist sites could be.
### 5.3 What does a declaration bind?
If `cat` names a region rather than a struct — one that shifts and completes against
the engram and the neighbouring code — what is written at the declaration site, and
what is resolved at use? **This is the centre and it is specified nowhere.**
Falls out of 5.1 if displacement is taken: a declaration **locates** rather than
allocates.
### 5.4 Is the faculty list right?
Seven were derived from what broke. Derived-from-failure is a biased sample — it finds
what is loud, not what is missing. **What faculty is absent entirely and therefore
never failed?**
---
## 6. The residue map
What each construct must absorb, from §18.0 plus measured state:
| Residue | Count | Absorbed by |
|---|---|---|
| persist-after-mutate | 62 sites | `hold` (after-crossing) |
| auth-per-route | 10 sites | `hold` (before-crossing) |
| index-after-append | 9 of 9 failed | `hold` (after-crossing) |
| env var defaults | 20 | configuration declared once |
| process identity | 0 guards | `hold` (before-crossing) |
| `geometry_free` at every call site | every site | ownership follows from `semel` |
| five ingest functions where there is one | 5 → 1 | `protocol` / `impl` |
| `el_runtime.c` | 20,504 lines | faculty decomposition, ordered after `semel` |
---
## 7. Notes carried forward
**`el-architecture.html` §04 needs its numbers sourced or cut.** An audit found the
faculty scoreboard — `Ingest 2 min → 0 nodes`, `Recall self ranked 8th`,
`Body CC 356 / 1,626 ln`, `the verifier answers in 2.9s`, `5 of 13 native suites
failing` — has no supporting evidence in the repository, under a footer asserting
*"nothing is inferred and presented as fact."* Against a corpus whose documents
supersede their own conclusions in place, that is the one file that would not survive
scrutiny. Fix or remove.
**Source as a projection surface is claimed and unimplemented.** `el-architecture.html`
§147/§150: *"if source is a surface, then emitting a corrected file is projection."*
Greps for `surface_profile_code`, `emit_source` → zero hits.
It is not unbacked. **It was demonstrated on 2026-08-14** — three faculties (phonetic,
semantic, procedural) projected into TypeScript, a surface the system had never used,
with the network severed. Recovered at
`~/Development/neuron-technologies/andre-server-recovered/` and copied into
`evidence/03-andre-demo/`. The claim needs bringing home to El, not proving.
**`hold` is the highest-leverage construct after `semel`** — it collapses 62 + 10 + 9
sites and unblocks the runtime extraction. §19.3 names the prologue/epilogue seam as
the prerequisite and its stated blocker has expired.
---
## 8. What is not decided and must not be guessed
- The representation of `semel` (§5.1)
- One `hold` or two (§5.2)
- What a declaration binds (§5.3)
- The missing faculty (§5.4)
- Sequencing after `semel` — the ordering criterion decides it, not preference
---
*Recorded 2026-08-17. Everything in §2, §3 and §4 is decided. Everything in §5 is open
and is Will's. Nothing here was inferred from a document that was not read.*
docs/architecture/geometry-vs-code.md
+117
View File
@@ -0,0 +1,117 @@
# Geometry or Code
**Running list.** Append as decided. Started 2026-08-17.
**The test:** *is this an arbitrary convention, or is it a relation?*
Conventions were agreed by people and could have been otherwise — a RIFF header could
have used a different magic number. Nothing derives them; they must be written down.
Relations are not agreed. Distance is distance. Anything whose answer is *where is this
relative to that* is geometry, and writing it as code is the error the whole effort is
correcting.
**Second test, for the hard cases:** *if I write this as code, am I encoding in
`if`-statements a distinction the geometry was built to hold?* If yes, it's geometry.
---
## Pure geometry
| Thing | Because |
|---|---|
| Meaning | position |
| Grounding / standing | the weight on the edge — a magnitude, not a computation |
| Learning | standing changing over time |
| A gap | low standing |
| Wonder | a gap with a pull weight |
| Type checking | is this position in that region — distance |
| Dispatch | position, not a tag |
| Recall | re-origining at a region; projection, not replay |
| Reasoning | traversal |
| Deduction | containment. There is no procedure |
| Counting | a position, not a loop's output |
| Similarity / difference / residue | subtract |
| Analogy, metaphor, skill transfer | change of basis |
| Negation, sarcasm | reflect an axis |
| Empathy | translate the origin |
| Reframe | rotate the frame |
| A lens | project onto an axis |
| Rhyme | distance in phonetic space |
| Humour | intersection of regions — fart-meaning ∩ funny ∩ form |
| Idiom detection | the whole unit sits farther out than its parts |
| Self | a world-tube — a trajectory through the manifold |
| Consolidation | episodic → semantic promotion |
| Reification | dense regions cohering; runs on the beat, has no caller |
| Cross-cutting concerns | **dissolved** — a hold is a *neighbour*. Adjacency, not tracking. **Implemented 2026-08-17**: a construct declares what runs at a crossing, and it resolves at execution — see the runtime seam. |
| Effects | topology. `reach_out` is bounded by `detect_gap` and `verify` because those are its edges |
| Capability | position relative to a boundary. In C it is already spelled `const` |
| The AST | a projection of geometry into a tree — a surface, not the centre |
| Source code | a surface, like text, audio, image |
## Must be code
| Thing | Because |
|---|---|
| Sensors — mic, camera, file read, socket | the physical touch. I/O is where the world arrives |
| Byte formats — RIFF, PNG chunks, `MThd`, OOXML | arbitrary convention. A committee chose the magic numbers |
| CRC32 polynomial, Adler32, zlib framing | same — agreed constants, derivable from nothing |
| Cosine, distance, the float arithmetic | the machinery that *walks* the geometry is not itself geometry |
| Arena, refcount, allocator | bookkeeping for the **representation**, not for the positions |
| Locks, threads, publication boundary | the hardware is code. **Ordering is not** — see Answered, above. Coordination is required only where state is non-monotone. |
| WAL, page layout, ARIES recovery | durability against a physical device that can lose power |
| Emission — writing C or JS text | the final surface has to be *typed out* by something |
| OS interaction — launchd, spawn, signals | outside the system by definition |
| Device realizers — `el_audio_darwin.m`, `el_capture_darwin.m` | OS frameworks. Correctly already isolated, zero network |
---
## The ones I would have written as code, and was wrong about
Recorded because the error has a pattern and the pattern is the point.
| Thing | What I reached for | What it is |
|---|---|---|
| Rhyme | a rhyming dictionary, or an API call | distance between rime tails |
| Fart onomatopoeia | a 30-element string literal | an intersection of three regions |
| "Funny" | a scorer with `if`-statements | a relational neighbourhood grounded in a voice |
| Representation vs description | a hardcoded blacklist containing `raspberry` | falls out of lexicon membership × phonetic comedy |
| Video | a codec, sized as a project | one more surface profile |
| Type checking | a phase between parse and emit | reading a distance that already exists |
| Grounding | a call site, an obligation, a discharge | it has no caller. It just runs |
| N transducers, N realizers | one component per modality | zero of each. Sensors and bases at the skin |
**The pattern:** every one is *encoding in code a distinction the geometry was built to
hold.* The tell is that the code version is a **fixed enumeration** — a list, a table, a
blacklist, a set of branches — and the geometry version is a **measurement**.
If the implementation contains a literal set of the right answers, it is in the wrong
column.
---
## Answered
| Thing | The answer |
|---|---|
| Concurrency | **Ordering is geometric.** Causality is a partial order (Lamport 1978); a total order is an arbitrary extension of it and "cannot be depended on to imply a causal relationship." Programming languages force you to write a total order, so authoring *invents* constraints the problem never had — and every lock, barrier, fence and consensus protocol is apparatus for recovering the partial order destroyed at authoring time. CALM (Hellerstein/Alvaro, proven by Ameloot et al.): a program has a consistent coordination-free implementation **iff it is monotone**. What breaks monotonicity is destructive update. **Coordination is the price of forgetting.** |
| The module system | **Premature — the partition is a filesystem path, not a neighbourhood, and there is no namespacing at all.** `import` is textual inlining (guarded against double inclusion); when a `.elh` header exists the header is inlined instead and symbols resolve at C link time, so linking is real and delegated to C. Two modules defining `helper` emit two C functions into one translation unit. Linking barely survives the *path* partition, so whether it survives a neighbourhood partition cannot yet be asked. |
| Numeric literals | **The numeral is convention; the number is a position — and a bare `3` is a MAGNITUDE WITH NO AXIS.** `int_to_str` was already form 1: nothing determines that twelve is written `1` then `2`. But a literal is not a position until something gives it a direction, which is why `3.days` needs a calendar. Measured consequence: `Duration + Int` was refused ("an Int carries no unit") while `Instant + Int` compiled to raw `(t + 3)` and reported clean — silently moving a point by an unspecified amount. The rule was simply never written. Now: `t + 3` is refused, `t + 1.hour` is accepted, because `.hour` supplies the axis. |
| Parsing | **A grammar is a basis; parsing is transduction onto it.** The lexeme→token map is convention (`fn` could have been `def`); shape recognition is a region; the byte traversal is irreducible, like every other traversal. Three things favour *region* for the act: ambiguity (`a * b` needs context — a grammar resolves it with the lexer hack, a region by neighbourhood), error recovery (nearest-match is free), and precedence, which is ordering along an axis with a conventional parameter. **But the SHOULD gate refuses the obvious move:** the keyword table stays code, because the set is closed by the language definition and the lexer runs before the program is understood, so a program can never declare its own keywords. Externalising it costs I/O per compile for zero flexibility — the same verdict as `is_digit` in ASCII. What was actually wrong: 5 of 46 keywords were consumed by nothing, and using one silently miscompiled. |
| Error handling | **`grounded: false` covers not-knowing; it does not cover failed.** Standing is a *signed* component: `> 0` supported, `= 0` unknown, `< 0` contradicted. Not-known and known-false are opposite directions on one axis and a boolean cannot tell them apart. `inhibitory` as an int32 flag is that sign wearing a boolean. |
## Fourth proof form
**4 — ADVERSARIAL EXACTNESS.** Where approximation is a break, geometry is
excluded. A cryptographic hash is a *deliberately structure-destroying* map:
near inputs land at maximally uncorrelated outputs. Geometry is the claim that
near things stay near — a manifold that approximated SHA-256 would *be* a break
of SHA-256. Signature verification is the same: 0.99-valid is invalid. And
X25519 **is** geometry, a group on an elliptic curve, which is precisely why it
must be code: its security is the hardness of moving in that geometry.
**Form 1 no longer survives as a verdict.** Every row it justified turned out to
be a *basis*, not a capability. RFC 8259 fixes where the commas go — that is a
surface, and projecting onto a surface is geometry. A convention describes the
basis you project onto; it never describes an act.
docs/v1/experiments/README.md
+59
View File
@@ -0,0 +1,59 @@
# v1 — Experiments
Every change to El on `iteration-1` was produced by one loop, run repeatedly:
```
Ishikawa → scientific method → Six Sigma → repeat
```
- **Ishikawa** — name the root cause, not the symptom. *Why is this table here?*
never *why is this table ugly?*
- **Scientific method** — state a hypothesis, **commit predictions before
running**, then run it in an isolated worktree and grade every prediction
including the ones that failed.
- **Six Sigma** — eliminate the defect *class*, then add a control so it cannot
silently return.
## The organising finding
**Predictions that came back FALSE were worth more than the ones that held.**
Nineteen cycles, sixty-one predictions. The eleven that failed produced every
significant result:
| Failed prediction | What it found |
|---|---|
| "the arity table has drifted from the header" | Zero drift — but **110 functions had no entry at all**. The table was not wrong, it was 40% incomplete. |
| "codegen drops below baseline" (×4) | The **traversal is irreducible**. Walking an AST to find calls does not move no matter who decides. Only the rule and the judgment leave. |
| "guards cannot refuse through the seam" | One line, and refusal works. Six compile-time kinds were unnecessary. |
| "C forbids the struct redefinition" | C allows shadowing — and a *different* defect surfaced: an exit injection emitted with an empty target. |
| "routing el_bin_lookup through the gate fixes the SIGSEGV" | It did not. The **fallback** was the hazard: `strlen()` on an integer. I would have shipped the wrong fix and called it verified. |
A prediction that only ever confirms is a demonstration, not a test. One cycle
was run **without** committing predictions first — `async-half-expressible` —
and it produced a rigged result: `pthread_join` immediately after
`pthread_create`, with the word `DEFERRED` printed by the test itself. It had to
be discarded and re-run.
## Layout
```
cycles/ one file per loop, numbered in order, named for the DEFECT
findings/ what the cycles produced, cross-cut by kind
```
## Scoreboard
```
cycles run 19
predictions committed 61
predictions FALSE 11 ← the useful ones
silent miscompilations found 4
security-relevant defects 2
architecture questions closed 5
defects in my own measurement 4
```
Every cycle verified the same three things before landing: the compiler
self-hosts byte-identically (gen2 == gen3), the native suite passes, and the
integration harnesses pass. A cycle that could not show all three did not land.
docs/v1/experiments/cycles/00-INDEX.md
+26
View File
@@ -0,0 +1,26 @@
# Cycles
Each is one `Ishikawa → scientific method → Six Sigma` loop, run in an isolated
worktree so a wrong answer cost nothing. Named for the **defect**, not the fix.
| # | Cycle | Root cause | Predictions | Landed |
|---|---|---|---|---|
| 01 | [constructs-have-nowhere-to-be](01-constructs-have-nowhere-to-be.md) | a construct had nothing to BE, so its meaning lived in the emitter | 3/3 | yes |
| 02 | [a-construct-cannot-refuse](02-a-construct-cannot-refuse.md) | injection discards the target's result; no form said no | 4/4 | yes |
| 03 | [the-wrapper-was-conditional](03-the-wrapper-was-conditional.md) | exit injection needed compile-time knowledge only because the wrapper was conditional | 3/4 | yes |
| 04 | [c-has-no-closure-syntax](04-c-has-no-closure-syntax.md) | "C has no closures" taken as a fact about what is possible | 5/7 | yes |
| 05 | [the-emitter-discards-what-it-knows](05-the-emitter-discards-what-it-knows.md) | codegen sees every construct relation and throws it away | 5/5 | branch |
| 06 | [the-crossing-resolves-at-emission](06-the-crossing-resolves-at-emission.md) | the binary has no table to consult | 3/4 | yes |
| 07 | [invocation-is-not-composable](07-invocation-is-not-composable.md) | the wrapper called the target directly | 5/5 | yes |
| 08 | [the-emitter-adjudicates](08-the-emitter-adjudicates.md) | a prohibition had nowhere to live but a `#error` | 4/5 | yes |
| 09 | [policy-inside-the-compiler](09-policy-inside-the-compiler.md) | a program cannot declare its own restrictions, so the tier policy was compiled in | 4/5 | yes |
| 10 | [a-second-copy-of-the-header](10-a-second-copy-of-the-header.md) | builtin arity hand-maintained beside `el_runtime.h` | 4/5 | yes |
| 11 | [one-type-erases-the-return](11-one-type-erases-the-return.md) | `el_val_t` means the header cannot say `now()` returns an Instant | 4/5 | yes |
| 12 | [judgment-lives-with-knowledge](12-judgment-lives-with-knowledge.md) | the emitter knows the types, so it also judged them | 5/5 | yes |
| 13 | [thirty-five-return-types](13-thirty-five-return-types.md) | `is_int_call` hardcoded what drives `+` dispatch | 6/6 | yes |
| 14 | [keywords-that-reserve-nothing](14-keywords-that-reserve-nothing.md) | 5 of 46 keywords consumed by no path | 6/6 | yes |
| 15 | [no-namespacing-at-all](15-no-namespacing-at-all.md) | `import` is textual inlining; every name is global | 4/4 | yes |
| 16 | [tokens-carry-no-position](16-tokens-carry-no-position.md) | a token was `(kind, value)`, so no diagnostic could name a place | 6/6 | yes |
| 17 | [annotations-are-never-checked](17-annotations-are-never-checked.md) | the annotation feeds dispatch and is never verified | 6/6 | branch |
| 18 | [async-half-expressible](18-async-half-expressible.md) | **first attempt was DOGMA** — no predictions, rigged test | 4/4 (2nd) | branch |
| 19 | [a-convention-is-not-a-gate](19-a-convention-is-not-a-gate.md) | `looks_like_heap_obj` is static, so every type re-derives it | 6/7 | yes |
docs/v1/experiments/cycles/01-constructs-have-nowhere-to-be.md
+42
View File
@@ -0,0 +1,42 @@
# constructs have nowhere to be
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `5718943`
```
let a construct declare its own meaning instead of the emitter knowing it
codegen called fn_has_decorator for exactly three names — manager, accessor,
route. Twelve others parsed, attached as {name,args}, and compiled to nothing,
including four that look like protection: @authenticate (6 uses), @authorize
(3), @rate_limit (3), @validate (2). The cause was not that the branches were
untidy. A construct had nothing to BE, so its meaning had nowhere to live
except the emitter, and every construct was therefore a compiler edit.
A name -> injection table would have moved the enumeration twenty lines up
without removing it. So the construct now carries its own meaning:
@decorator("injects_at_entry", "engram_boundary_beat")
fn audited() {}
@audited
fn risky_op() -> Int { ... } // gets the beat, attributed to "audited"
scan_declared_decorators is a token-level pre-pass beside scan_routes, forced
by streaming codegen having no whole-program AST. manager and accessor are
seeded as the compiled-in core — the fixedSelf shape from substrate.go: a
complete fallback exists, declaration is enrichment.
This is the injection half of the seam only. The prohibition half (@manager's
#error on dharma_emit) stays hardcoded, because "which calls may appear inside
this boundary" is a query over program structure and there is nothing yet to
ask.
Verified three ways: emitted C for existing @manager/@accessor code is
byte-identical to the hardcoded path; a construct with a name the compiler has
never heard of injects correctly; the compiler self-hosts byte-identically.
90/90 native compiler tests pass.
```
docs/v1/experiments/cycles/02-a-construct-cannot-refuse.md
+43
View File
@@ -0,0 +1,43 @@
# a construct cannot refuse
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `60737b0`
```
let a construct refuse, not only observe
@authenticate (6 uses), @authorize (3), @rate_limit (3) and @validate (2)
parsed, attached, and compiled to nothing. Fourteen applications that read as
protection and emitted no instruction — a function decorated @authenticate
compiled byte-identically to an undecorated one.
The missing capability was not authentication. It was that a construct could
observe a boundary but never refuse one. injects_at_entry discards the target's
result; there was no form in which a construct could say no.
@decorator("guards_at_entry", "my_auth")
fn authenticate() {}
@authenticate
@authorize
fn handler() -> String { ... }
emits, at entry:
{ el_val_t __g = my_auth(EL_STR("handler"), EL_STR("authenticate")); if (__g) return __g; }
{ el_val_t __g = my_roles(EL_STR("handler"), EL_STR("authorize")); if (__g) return __g; }
Guards precede injections because a refused call must not report a crossing,
and every guard runs where the topmost injecting construct wins — refusal is
not a role, so it does not follow the role convention.
The compiler still knows nothing about auth. The program points the construct
at its own function, which is where that decision belongs.
Verified: existing @manager/@accessor output byte-identical, compiler
self-hosts byte-identically, guards stack in declaration order and emit before
the beat. 94/94 native compiler tests pass.
```
docs/v1/experiments/cycles/03-the-wrapper-was-conditional.md
+82
View File
@@ -0,0 +1,82 @@
# the wrapper was conditional
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `4f7568b`
```
give a construct its after-crossing face, and let constructs compose
§6 records 62 persist-after-mutate sites, 10 auth-per-route, and
index-after-append that failed at 9 of 9 — every one an obligation at a
crossing that decayed into "remember to do this afterwards." An obligation a
human must remember is not an obligation, and the 9-of-9 figure is what that
costs.
@decorator("injects_at_exit", "persist_now")
fn durable() {}
The body moves into a static helper and the visible fn becomes a wrapper, so
EARLY RETURNS pass through the exit injection. Emitting it only before the
fall-through return would have silently missed every early return — the exact
failure class this seam exists to remove. Fns with no exit construct emit
byte-identically to before.
Three independent constructs now compose on one fn, none known to the compiler:
el_val_t mutate(el_val_t k) {
{ el_val_t __g = my_auth(EL_STR("mutate"), EL_STR("authenticate")); if (__g) return __g; }
engram_boundary_beat(EL_STR("mutate"), EL_STR("manager"));
el_val_t __r = __el_body_mutate(k);
persist_now(EL_STR("mutate"), EL_STR("durable"), __r);
return __r;
}
Guard, then entry, then body, then exit. §5.2 asked whether `hold` is one
construct or two; the implementation answers one construct with two faces,
selected by declared kind rather than by two mechanisms.
Verified: existing output byte-identical, compiler self-hosts byte-identically,
early returns pass through the exit, ordering holds under composition. 98/98
native compiler tests pass.
```
## Record — `285166c`
```
EXPERIMENT: emit the wrapper unconditionally, so exit binds at runtime too
ISHIKAWA: why did exit injection still need compile-time knowledge? Because the
body-helper wrapper was only emitted when codegen already knew an exit
construct existed. The wrapper being conditional was the cause, not the wrapper
being necessary.
PREDICTIONS AND RESULTS
P1 exit becomes runtime-bindable TRUE returns 14, bound
after the build
P2 codegen shrinks TRUE 5094 -> 5044
P3 cost 5-15% from a call frame on every fn FALSE 0.37s -> 0.38s, ~3%
P4 fixpoint holds TRUE
Every fn now gets a body helper and a wrapper. It has to be unconditional:
early returns must route through something for an exit construct to observe
them, and codegen cannot know which fns will be bound after the binary exists.
Removed with the machinery: declare_exit, decorator_exit, cg_exit_target,
cg_exit_construct, and the injects_at_exit scanner branch.
Two controls failed and were rewritten rather than repaired --
no-exit-construct-emits-no-wrapper asserted the optimisation this removes, so
it is now inverted. The integration harness gained a seventh assertion: an exit
construct declared after the build replaces the result.
99/99 native, 7/7 integration, fixpoint gen2==gen3.
```
## Record — `b40754f`
```
land unconditional wrapper: exit crossings resolve at runtime
```
docs/v1/experiments/cycles/04-c-has-no-closure-syntax.md
+64
View File
@@ -0,0 +1,64 @@
# c has no closure syntax
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `2bed848`
```
EXPERIMENT: hand the construct the body as a real closure
ROOT CAUSE of the weaker design: "C has no closures" was taken as a fact about
what is possible. It is a fact about one grammar. Every C++ lambda, every Go
closure, every Rust closure compiles to a struct of captured values plus a
function pointer -- which is what is emitted here. Codegen emits C; it is not
written in C's syntax, and the distinction is the whole difference between a
construct that can only decide whether to repeat and one that controls
invocation.
It would also have crippled the JS backend, which has closures natively, for a
limit that applies only to the C one.
PREDICTIONS AND RESULTS
1 env struct + thunk taking void* TRUE
2 fails to compile: struct redefinition FALSE -- C allows the
inner declaration to shadow. Prediction wrong; C is more permissive than
assumed. A different real defect surfaced instead: a wrap with no exit
construct emitted `(EL_STR("f"), EL_STR(""), __r);` -- a call to an empty
target -- because has_exit was reused as "needs a wrapper" and the exit line
was emitted unconditionally. Fixed.
3 compiles when the target is declared in El FALSE -- and this is
the root cause worth keeping: El has ONE type, el_val_t = int64_t. El's type
system cannot describe a callable, so `extern fn` and the real signature
cannot be made to agree in El's own vocabulary. The fix is not a cast:
codegen DEFINES the wrap calling convention, so codegen emits the extern
declaration. The convention is not El-expressible; it is emitted.
4 target controls invocation, 0..N times TRUE
5 existing @manager output byte-identical TRUE
6 compiler fixpoint holds TRUE
7 emitting the convention makes it compile TRUE
MEASURED
base(5) wrapped by a target that invokes the body twice and sums -> 10
never_runs(5) wrapped by a target that never invokes it -> 999
Neither is expressible by "decide whether to repeat". This supersedes the
repeats_body experiment on experiment/repeats-body, which was built around the
mistaken limit.
```
## Record — `7d01608`
```
land wraps_body: a construct controls invocation
Proven on experiment/wraps-body (2bed848): base(5) wrapped by a target that
invokes the body twice returns 10; a target that never invokes it returns 999.
Neither is expressible by deciding whether to repeat.
Root cause it corrected: 'C has no closures' is a fact about one grammar, not
about what can be emitted. And El's single type (el_val_t = int64_t) cannot
describe a callable, so codegen emits the calling convention rather than asking
El's type system for something it structurally cannot say.
```
docs/v1/experiments/cycles/05-the-emitter-discards-what-it-knows.md
+48
View File
@@ -0,0 +1,48 @@
# the emitter discards what it knows
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `a5af871`
```
EXPERIMENT: let the compiler write down what it already knows
HYPOTHESIS: attribution is redundant for static structure. Codegen sees every
construct-to-function relation at emission time and discards it, so the only
way to learn the structure back is to run the program and read what it
reported. That is instrumentation compensating for erasure.
PREDICTIONS, committed before running:
1 derivable at compile time with no runtime call expected TRUE
2 complete for guards and exits (invisible today) expected TRUE
3 answers it for code that has never executed expected TRUE
4 deterministic expected TRUE
5 makes the entry beat redundant expected FALSE
RESULT: 5/5 as predicted. From a program that was never executed:
authenticate guards_at_entry login my_auth
durable injects_at_exit save persist_now
authenticate guards_at_entry critical my_auth
durable injects_at_exit critical persist_now
manager injects_at_entry critical engram_boundary_beat
Prediction 5 held: the relation records that a boundary COULD be crossed, the
beat records that it WAS. They are different facts and neither replaces the
other.
CONSEQUENCE, and it undercuts the first pass on iteration-1: construct identity
was available at compile time all along. With relations recorded at build, the
runtime needs only the function name and attribution becomes a join rather than
a payload. The counter-argument is that the payload is self-describing while
the file must be pinned to the artifact or the two drift and attribution is
silently lost — which is the same conclusion as "compile against a manifold
revision and record the revision in the artifact", reached from the other side.
Written to a file rather than the engram on purpose: a compile that consults a
manifold produces different output from identical source at different times.
The file is content-addressed; the engram ingests it. Determinism preserved,
mechanism proven.
```
docs/v1/experiments/cycles/06-the-crossing-resolves-at-emission.md
+170
View File
@@ -0,0 +1,170 @@
# the crossing resolves at emission
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `35b07ba`
```
EXPERIMENT: resolve the crossing at execution, not at emission
HYPOTHESIS (Will's): a compiler whose one compiled mechanism is extending the
LANGUAGE — not the compiler — can compose without recompilation.
ISHIKAWA — why does a construct require a recompile today?
method codegen inlines the target call into the body
machine the binary has no table to consult
material the declaration lives in source, read at compile time
measurement nothing observes what applied at runtime
root cause the crossing is resolved at EMISSION, not at EXECUTION
CHANGE: codegen emits one unconditional indirection per fn. Which constructs
apply is read from a table that can be written AFTER the binary exists;
targets resolve through dlsym against the running image.
PREDICTIONS AND RESULTS
P1 a construct declared after the build applies TRUE
P2 an unlinked target is skipped, not fatal TRUE
P3 emitting on every fn is measurably slower FALSE — 0.37s -> 0.36s
with 267 indirections and
no bindings. Free unused.
P4 the compiler still self-hosts TRUE (see note)
DEMONSTRATED: an El program with NO decorator in its source, already compiled
and linked, picked up a construct declared afterwards:
$ /tmp/seamrun -> 7
$ echo 'work audited entry audit_entry' > constructs.txt
$ EL_CONSTRUCTS=constructs.txt /tmp/seamrun
AUDIT: work applied by audited
7
P4 note: my first fixpoint test was wrong, not the code. I compared gen1 to
gen2, which must differ whenever codegen's output changes. gen2 == gen3, 267
seam sites, stable.
MEASURED COST, and the root cause was not where I looked
0 bindings 0.36s vs 0.37s baseline free
2 bindings, dlsym per call 2.45s 6.6x
2 bindings, resolved once 0.69s 3.5x recovered
The table scan was never the cost. dlsym walks the dynamic symbol table on
every call. Resolve once and cache — which is the smallest form of what
salience does for memory: what is hot stays resolved. The 0.69s residual is
audit_entry's own printf on two of the compiler's hottest functions, not seam
overhead.
CONSEQUENCE: the five compile-time declaration kinds on iteration-1 are a
compile-time specialisation of something that resolves at runtime. They are not
wrong, but they are not the mechanism — the mechanism is one indirection, and a
kind is data.
```
## Record — `886626a`
```
seam refusal + control tests: a runtime binding can short-circuit
Prediction 3 was FALSE. I expected refusal to be impossible through the seam
because the entry indirection discarded its return. One line:
{ el_val_t __s = el_seam_run(EL_STR(f), 0, 0); if (__s) return __s; }
work() returns 7; bound to a refusing construct AFTER the build it returns 42.
So three of the five compile-time kinds are runtime-bindable: entry injection,
exit injection, and refusal. wraps_body needs invocation control and
prohibits_outside is compile-time by nature.
104/104 native compiler tests pass.
```
## Record — `28d19da`
```
strip the compile-time machinery the seam replaces
PREDICTION: codegen.el drops below 4661, its size before any of these passes.
RESULT: FALSE. 5157 -> 5096. Still +435 over baseline.
injects_at_entry collapsed into the seam removed
guards_at_entry collapsed into the seam removed
injects_at_exit needs the body-helper wrapper STRUCTURAL
wraps_body needs the closure + wrapper structural
prohibits_outside a #error cannot be emitted at runtime
The wrapper is not a consequence of compile-time resolution. Early returns must
be routed through something no matter when the target is resolved, so exit
injection was never going to collapse. I predicted it would because I had
conflated "resolved late" with "emitted less".
What did collapse is entry injection and refusal -- 61 lines of compiler
replaced by one refusable indirection, with the capability now bindable after
the binary exists.
8 tests fail, and they are exactly the 8 controls for compile-time entry
injection and guards. No unrelated breakage: the controls reported precisely
what moved. They assert emission of something that now happens at runtime, so
they need rewriting as integration tests -- which the framework does not
currently support, because runtime binding needs a built binary and an
environment, not compile_capture.
Verified after the strip: fixpoint gen2==gen3, observation and refusal both
work through the seam with the compiler knowing nothing about either.
```
## Record — `8bbb750`
```
control the claim that cannot be unit tested
The seam's whole claim is that a construct declared AFTER a binary exists
applies to that already-built program. compile_capture only sees emitted text,
so it structurally cannot check this: it needs a built binary, a linked target,
and an environment. Verified by hand until now, which is the standing problem
this session has been about.
tests/integration/seam_binding.sh builds a probe from El source containing no
construct at all, links a target that El never references, and asserts:
ok unbound program is unaffected
ok a construct declared AFTER the build applies
ok a construct declared after the build can REFUSE
ok an unlinked target is skipped, not fatal
ok a binding for a different fn does not fire
ok two constructs compose on one crossing
6 assertions, 6 passed, 0 failed
The eight controls that failed after the strip were replaced, not repaired.
They asserted compile-time emission of capability that moved to runtime;
contorting them would have kept an assertion whose subject no longer exists.
Three took their place, asserting the emitted shape, and the behaviour they
used to cover is now the integration harness's job -- which is the honest
division, since the shape and the behaviour are no longer the same fact.
99/99 native compiler tests pass. Fixpoint holds.
```
## Record — `24f7fb5`
```
land the runtime seam: resolve the crossing at execution
Five compile-time passes added 491 lines to the thing that was supposed to stop
growing. The seam is ~55 lines of C and one line of emission, and it does at
runtime what three of those five kinds did at compile time -- for programs that
are already built.
a construct declared AFTER the binary exists applies to it
free when unused: 0.36s vs 0.37s baseline across 267 indirections
dlsym was the cost, not the table scan; resolve-once recovered 3.5x
refusal works, composition works, unlinked targets are skipped not fatal
injects_at_exit and wraps_body do NOT collapse: early returns must route
through the body-helper wrapper regardless of when the target is resolved. The
wrapper is structural, which I had wrong. prohibits_outside cannot move at all
-- a #error has no runtime.
Controls: 99/99 native compiler tests, plus tests/integration/seam_binding.sh
(6/6) for the claim compile_capture structurally cannot see.
```
docs/v1/experiments/cycles/07-invocation-is-not-composable.md
+45
View File
@@ -0,0 +1,45 @@
# invocation is not composable
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `bc2f26d`
```
EXPERIMENT: invocation control resolves at runtime
ISHIKAWA: why did wraps_body need compile-time knowledge? Because the wrapper
called the target directly. If the wrapper calls through the seam instead, the
seam can call the body itself, and a construct bound after the build decides
how and whether to invoke it.
PREDICTIONS AND RESULTS
P1 wrap becomes runtime-bindable TRUE body x3 -> 21,
never invoked -> 111
P2 codegen shrinks TRUE 5042 -> 4977
P3 cost 5-10% from an indirect call on every fn TRUE 0.36s -> 0.39s, ~8%
P4 zero-param fns break on the empty struct TRUE empty struct is a GNU
extension, empty init
is C23. Fixed with a
char field.
P5 fixpoint holds TRUE
PROCESS FAILURE worth recording: my first patch silently did not apply because
I dropped the assert on the string replacement. The build then failed with
"undeclared identifier __thunk_noargs", which I nearly attributed to the
empty-struct prediction. The guard that would have caught it existed and I
removed it -- the same shape as every other defect found tonight.
Removed: declare_wrap, decorator_wrap, cg_wrap_target, cg_wrap_construct,
params_to_call_args, and the wraps_body scanner branch.
prohibits_outside is now the ONLY construct kind left at compile time, and it
cannot move: a #error has no runtime.
```
## Record — `c04d68f`
```
land runtime invocation control: only prohibition remains compiled
```
docs/v1/experiments/cycles/08-the-emitter-adjudicates.md
+53
View File
@@ -0,0 +1,53 @@
# the emitter adjudicates
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `c741cfe`
```
EXPERIMENT: prohibition becomes a query over emitted relations
I said prohibition could not move because "a #error has no runtime". That
conflated two separable things: WHEN a violation is detected (build time --
correct, and unchanged) and WHERE the rule and the checker live (the compiler
-- assumed).
A prohibition is a containment relation over the call graph. So codegen now
records what it saw:
sneaky calls raw_sql
allowed calls raw_sql
allowed calls @repository
repository calls prohibits:raw_sql
and tools/check/prohibitions.sh decides, at build time, outside the compiler.
PREDICTIONS AND RESULTS
P1 codegen can emit the call graph it already walks TRUE
P2 the check becomes a query outside the compiler TRUE
P3 all prohibition decisions leave codegen TRUE zero #errors now
P4 violations still caught at build time TRUE exit=1
P5 codegen drops below the 4661 baseline FALSE 4962, +301
P5 is the finding. The TRAVERSAL is irreducible -- you must walk the AST to
find calls, and those ~120 lines do not move no matter who decides. What is not
irreducible is the rule (which names) or the decision (#error). Those left. I
predicted the whole 223 lines would go because I had not separated walking from
adjudicating.
Still compiled, and measured rather than assumed: the capability-tier system
(cap_check_call, is_self_formation_call, is_dharma_call, is_llm_call,
cap_record_violation, emit_cap_violations) is 76 lines of the same shape --
prohibits_WITHIN rather than prohibits_outside, so the checker needs the
opposite polarity to absorb it.
98/98 native, 4/4 prohibition_query.sh, 7/7 seam_binding.sh, fixpoint ok.
```
## Record — `60c07ad`
```
land prohibition-as-query: the emitter records, it no longer adjudicates
```
docs/v1/experiments/cycles/09-policy-inside-the-compiler.md
+49
View File
@@ -0,0 +1,49 @@
# policy inside the compiler
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `c2d9596`
```
EXPERIMENT: the capability tier becomes shipped policy plus a query
Capability differs from prohibits_outside in one way that matters: a utility
program cannot be trusted to declare its own restrictions, because it would
declare none. So the policy comes from OUTSIDE the program -- it ships with the
language as data, editable without a compiler release.
tools/check/capabilities.rel 18 names that were string literals in codegen
tools/check/capabilities.sh the query that decides
PREDICTIONS AND RESULTS
P1 codegen emits kind + call graph, drops the 4 name tests TRUE zero #errors
P2 the 18 literals become a data file TRUE
P3 the checker catches capability violations TRUE exit=1
P4 codegen drops ~76 lines TRUE 4963 -> 4881
P5 below the 4661 baseline FALSE ~+230
TWO DEFECTS THE HARNESS FOUND THAT READING WOULD NOT HAVE
1. Calls inside main became invisible. cg_fn returns early for main -- C
provides its own -- so hooking the recording there left every call in main
unrecorded: a blind spot exactly where a program does its work. The old
cap_check_call ran from cg_expr and did see main. Moved the recording to
cg_expr.
2. Caller attribution was stale. __cg_current_fn kept whatever cg_fn set last,
so a violation in main was reported against the previously emitted function.
The test still PASSED, because the violation was detected -- only the name
was wrong, and a diagnostic naming the wrong fn is worse than none. Fixed at
all three main-emission sites; the first patch missed two because the live
path is codegen_streaming.
98/98 native, 7/7 + 4/4 + 5/5 integration, fixpoint ok.
```
## Record — `29f78f9`
```
land capability-as-policy: eighteen literals become a data file
```
docs/v1/experiments/cycles/10-a-second-copy-of-the-header.md
+54
View File
@@ -0,0 +1,54 @@
# a second copy of the header
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `9cc6040`
```
EXPERIMENT: derive arity from the runtime's own declarations
codegen.el carried builtin_arity(): 344 lines, 300 entries, a hand-maintained
second copy of el_runtime.h.
PREDICTIONS AND RESULTS
P1 the table duplicates the header TRUE 243 shared names
P2 they have already drifted FALSE ZERO drift. The
duplicate had been
maintained correctly.
P3 codegen can emit call-arity relations TRUE
P4 the check becomes a query against the header TRUE
P5 codegen drops to roughly baseline TRUE 4903 -> 4512,
149 BELOW the 4661
it started at
P2 being false is the better result: the table was not WRONG, it was
INCOMPLETE. 110 functions the runtime declares had no entry, so calling them
with the wrong argument count produced no El-level diagnostic at all. Measured:
the old compiler reports 0 arity errors for __http_do_map_to_file(1); the query
reports "takes 5 arguments, called with 1".
Deriving from the header fixes coverage AND makes drift impossible by
construction. 503 signatures, versus 300 entries maintained by hand.
THREE DEFECTS IN MY OWN CHECKER, each found by running it rather than reading it
1. El names and C names differ -- `println` is `__println`. 60 of 500 decls
carry the prefix and codegen owns the mapping; the old table carried both
keys. One rule covers all 60.
2. Multi-line declarations parsed as zero params, so the checker reported
"takes 0" for a function taking 5. A diagnostic with the wrong number in it
is worse than none -- the same shape as the stale caller attribution in the
previous pass.
3. Fixing (2) by joining lines dropped 500 signatures to 334, because a
declaration preceded by a comment no longer started its record. Comments
are stripped first now.
98/98 native, 5/5 arity_query.sh, fixpoint ok.
```
## Record — `d9e301b`
```
land arity-from-header: the runtime declares its own surface
```
docs/v1/experiments/cycles/11-one-type-erases-the-return.md
+52
View File
@@ -0,0 +1,52 @@
# one type erases the return
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `d2d89fc`
```
EXPERIMENT: temporal types as data — and the pass that GREW the compiler
This block is structurally unlike the previous four. It does not only
adjudicate, it DISPATCHES: Instant + Duration must become el_instant_add_dur,
LocalDate + Duration must become el_local_date_add_dur. The emitted C depends on
the type answer, so it cannot move to a post-hoc query. Selecting which call to
emit is an emitter's actual job.
PREDICTIONS AND RESULTS
P1 the block conflates dispatch with adjudication TRUE
P2 adjudication can move, dispatch cannot TRUE
P3 this pass shrinks codegen far less than the last TRUE, and worse:
4513 -> 4537, it GREW
by 24 lines
P4 the rules are affine algebra, closed by construction TRUE
P5 no type propagation -- name tracking plus a
hardcoded list of which builtins return which type TRUE, 19 names
P3 is the honest result and it is not spun: moving 19 names into a data file
cost more lines than it saved, because a generic loader is larger than the
enumeration it replaces. The win is not line count. It is that adding a 20th
temporal builtin is now a one-line edit to signatures.rel instead of a compiler
change, and that the data is inspectable.
WHY THE HEADER CANNOT SUPPLY THIS, unlike arity: el_runtime.h declares every
builtin as returning el_val_t, because El has ONE type. That single type is why
the whole seam is cheap and it is exactly why the C boundary cannot say that
now() returns an Instant while unix_seconds() returns an Int. The El-level type
is real and the boundary erases it.
INCOMPLETE, and stated rather than hidden: P2 said adjudication could move to a
query. It has NOT. Violations still emit TIME_TYPE_ERROR inline from the
emitter. Only the type DATA moved. Moving the adjudication needs the operand
types recorded as relations, which is a further pass.
98/98 native, 4/4 temporal_signatures.sh, fixpoint ok.
```
## Record — `e01e079`
```
land temporal signatures as data: the type table leaves, the dispatch stays
```
docs/v1/experiments/cycles/12-judgment-lives-with-knowledge.md
+47
View File
@@ -0,0 +1,47 @@
# judgment lives with knowledge
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `e8e25a0`
```
EXPERIMENT: temporal adjudication moves out; the placeholder stays
The previous pass moved the type DATA and left the judgment inline, which I
stated rather than hid. This finishes it.
PREDICTIONS AND RESULTS
P1 codegen can emit operand-type relations TRUE
"main calls temporal:instant_plus_instant"
P2 the affine rules are a small closed set as data TRUE 6 rules
P3 violations still caught at build time TRUE exit=1
P4 the reporter leaves codegen TRUE 4538 -> 4507
P5 the TIME_TYPE_ERROR placeholder must STAY TRUE
P5 is the boundary of this whole approach. The emitter has to emit SOMETHING
for an illegal expression -- it cannot emit nothing and it cannot decide what
the program meant. So the placeholder is irreducible in the same way the AST
traversal was: what moved is the judgment and the wording, not the fact that
something must be written.
The rules are affine algebra and the set is closed because there are only two
kinds of thing. An Instant is a POINT, a Duration is a DISPLACEMENT: add a
displacement to a point, subtract two points for a displacement, combine
displacements. Nothing else is meaningful, which is why the enumeration in
temporal.rel cannot grow the way an allowlist does.
A defect in my own checker, found by running it: the .rel file uses aligned
columns and my awk assumed a single space, so the message came out with the
rule key still prefixed. Same class as the multi-line header parse in the arity
pass -- formatting assumptions that only fail when you look at the output.
98/98 native, 6/6 temporal_query.sh, fixpoint ok.
```
## Record — `50425f3`
```
land temporal adjudication as a query: the emitter records, the rules are data
```
docs/v1/experiments/cycles/13-thirty-five-return-types.md
+49
View File
@@ -0,0 +1,49 @@
# thirty five return types
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `cbef1c1`
```
EXPERIMENT: Int return types as data — and the bug that fell out
PREDICTIONS AND RESULTS
P1 is_int_call's 35 hardcoded names move to data TRUE
P2 is_int_name stays -- it is annotation propagation TRUE
P3 the dispatch stays -- it is emission TRUE
P4 codegen shrinks ~40 lines TRUE 4507 -> 4469
P5 the design doc's characterisation is WRONG TRUE
P6 the moved data also fixes the bug it exposed TRUE
P5 CORRECTS THE RECORD. el-language-design.md and geometry-vs-code.md both cite
"== lowering to str_eq unless both operand names are in a hardcoded int-name
set -- a literal list of variable names treated as integers" as the paradigm
defect. It is not one. __int_names is populated from TYPE ANNOTATIONS
(param["type"] == "Int"), which is primitive but legitimate type propagation.
The actual defect was is_int_call: 35 hardcoded builtin return types, the same
shape as the temporal 19.
P6 IS A LIVE CORRECTNESS BUG, PRE-EXISTING, NOW FIXED
let a = str_len("hello") // no annotation
let b = str_len("hi")
let c = a + b // -> el_str_concat(a, b) on two integers
Verified identical on the pre-change compiler, so not a regression. It compiled
clean, ran, and printed NOTHING where it should print 7. No error at any layer.
The repair is three lines: an unannotated let takes its type from what the
initialiser returns. The return types were already required for dispatch and
were simply never consulted at the binding site. Moving them into data is what
made the gap visible -- reading the code for eight hours did not.
98/98 native + 2 new, 31/31 integration, fixpoint ok.
```
## Record — `505e5e7`
```
land int signatures, and repair a silent miscompilation they exposed
```
docs/v1/experiments/cycles/14-keywords-that-reserve-nothing.md
+57
View File
@@ -0,0 +1,57 @@
# keywords that reserve nothing
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `0143cc4`
```
ANSWER: is a grammar a convention, or a region?
Both, at different layers -- and it is the same split as serialization: the
convention is the BASIS, never the ACT.
lexeme -> token `fn` means function-start because someone said so CONVENTION
shape recognition given tokens, which construct is this REGION
source -> structure parsing is transduction onto that basis GEOMETRY
byte traversal something must read them in order IRREDUCIBLE
Three things push the ACT toward region rather than convention: ambiguity
(a * b needs context; a grammar resolves it with the lexer hack, a region by
neighbourhood), error recovery (nearest-region is free), and precedence, which
is ordering along an axis with a conventional parameter.
AND THE SHOULD GATE SAYS NO TO THE OBVIOUS MOVE
Every other table this session moved to data. This one stays code. The keyword
set is CLOSED by the language definition -- it does not leak the way an
allowlist does -- and the lexer runs before the program is understood, so a
program can never declare its own keywords. Externalising it costs file I/O on
every compile and buys nothing. Same verdict as is_digit in ASCII.
WHAT WAS ACTUALLY WRONG: five of 46 keywords were consumed by no parser or
codegen path. sealed, activate, seed, protocol, impl. Each stole an identifier
from users for nothing.
SECOND SILENT MISCOMPILATION OF THE DAY. Using one did not fail to parse:
let seed = 42
let impl = seed + 1
compiled CLEAN -- zero cc errors -- and printed 0 instead of 44. No diagnostic
at any layer. Fixed by removing the five.
A DEFECT IN MY OWN MEASUREMENT, caught before it did damage: my first pass
checked only parser.el and reported `test` as inert too. codegen consumes it at
4135 for --test mode, and the tree has 408 uses. Removing it would have broken
every test in the suite. The measurement was re-run across all four consumers.
100/100 native + 2 new, 31/31 integration, fixpoint ok.
```
## Record — `067dd40`
```
answer the parsing question: a grammar is a basis, and five keywords reserved nothing
```
docs/v1/experiments/cycles/15-no-namespacing-at-all.md
+53
View File
@@ -0,0 +1,53 @@
# no namespacing at all
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `79f6cb7`
```
ANSWER: if the partition is a neighbourhood, does linking survive?
The question is premature, and measuring says why. El's partition is a
FILESYSTEM PATH, not a neighbourhood, and there is no namespacing at all.
MEASURED
import is textual inlining (resolve_imports), guarded against double
inclusion by a __elc_imp__:<path> state key
when a .elh header exists the header is inlined instead and the .el is marked
seen, so symbols resolve at C link time -- so linking IS real, delegated to C
two modules defining `helper` emit two C functions into one translation unit
So linking barely survives the PATH partition. Whether it survives a
neighbourhood partition cannot be asked yet.
A DIAGNOSTIC REGRESSION I CAUSED, found by asking this question. cc does catch
the collision, but reports:
error: redefinition of '__el_body_helper'
error: redefinition of '__env_helper'
error: redefinition of '__thunk_helper'
error: redefinition of 'helper'
The user's own function is FOURTH. The first three are generated symbols
introduced by the unconditional-wrapper pass earlier today -- before it, there
was one clear message. Repaired by catching the collision at El level instead:
duplicate definition: 'helper' is defined 2 times — El has no namespacing,
so imported modules share one global scope
LIMIT, stated rather than hidden: textual inlining destroys file provenance. By
the time codegen runs there is one source string, so the message can say WHICH
name collides but not which files. Naming a.el and b.el needs provenance
threaded through resolve_imports.
104/104 native, 4/4 definitions_query.sh, the compiler itself reports clean,
fixpoint ok.
```
## Record — `f23cb2b`
```
answer the module question: the partition is a path, and there is no namespacing
```
docs/v1/experiments/cycles/16-tokens-carry-no-position.md
+64
View File
@@ -0,0 +1,64 @@
# tokens carry no position
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `6c975b1`
```
thread provenance through resolve_imports
The module question ended with a limit: textual inlining destroys file
provenance, so a duplicate-definition message could name the symbol but not the
files. Threading it exposed a bigger absence first.
TOKENS HAD NO POSITION AT ALL. A token was a flat (kind, value) pair, so NO
diagnostic in El could name a place -- every error named a symbol and never a
line. That is the prerequisite the module question was resting on.
THE CHAIN, end to end
lexer counts newlines; tok_append mints (kind, value, line)
parser stride 2 -> 3; tok_line added; FnDef carries its line
codegen records <fn> defines_at:<line>
resolve_imports publishes <file> spans <start> <end> for the combined source
checker maps a combined line back to file:line-within-that-file
duplicate definition: 'helper' is defined 2 times — El has no namespacing,
so imported modules share one global scope
/tmp/modtest/a.el:1
/tmp/modtest/b.el:1
PREDICTIONS AND RESULTS
P1 15 stride sites, encapsulated in tok_kind/tok_value TRUE, but see below
P2 adding a line field is mechanical TRUE
P3 the lexer must count newlines TRUE
P4 resolve_imports can record per-file line ranges TRUE
P5 the message can then name both files TRUE
P6 token memory grows TRUE, 25.0 -> 33.9 MB (+36%)
FOUR DEFECTS, EACH FOUND BY RUNNING AND NOT BY READING
1. interp_tokens_append_all walks the token list DIRECTLY with its own copy of
the stride. Gen1 built fine and gen2 emitted corrupt C, because the
compiler's own source uses string interpolation. My search missed it because
I grepped for the variable name `tokens`; it is called `dst`/`result`.
Searching by name instead of by shape -- third time today.
2. tok_count in test_compiler.el carried the stride too. I had scoped the search
to compiler sources and it had escaped into the tests.
3. Nested resolve_imports calls accumulated spans into shared state, so each
republished meaningless line ranges under the parent's name. Making the
buffer local fixed it; guarding the WRITE did not, which is what I tried
first.
4. The first working version reported b.el:3 -- the COMBINED line against a
filename that has no line 3. A file:line that does not match the file is
worse than no line at all.
105/105 native, 37/37 integration, fixpoint ok, compiler self-checks clean.
```
## Record — `cb7289f`
```
thread provenance end to end: a diagnostic can finally name a place
```
docs/v1/experiments/cycles/17-annotations-are-never-checked.md
+53
View File
@@ -0,0 +1,53 @@
# annotations are never checked
**Status: verified on `experiment/annotation-checking`, not merged.**
## Ishikawa — why does El silently miscompile?
Three bugs found the same day shared one shape.
```
method type tracked by per-function name sets, fed from annotations
machine el_val_t erases everything at the C boundary
material no propagation through expressions
measurement nothing verifies an annotation against what it annotates
─────────────────────────────────────────────────────────────────────────
root cause El has type ANNOTATIONS but no type CHECKING. The annotation
feeds dispatch and is never itself verified.
```
## Predictions
```
P1 let x: Int = "hello" compiles clean expect TRUE
P2 let s: String = 42 compiles clean expect TRUE
P3 the annotation drives dispatch, unverified expect TRUE
P4 same root cause as all three bugs found today expect TRUE
P5 checking literal-vs-annotation catches both expect TRUE
P6 zero false positives across the compiler's source expect TRUE
```
## Results — 6/6, and worse than a wrong answer
```
let x: Int = "hello"; x + 1 → 4343631981 a string POINTER used as an integer
let s: String = 42; println(s) → nothing address 42 dereferenced as a string
```
The first **leaks a raw memory address into program output**. The second is an
**arbitrary-read primitive** if that integer is ever attacker-influenced.
Verified: 6/6, zero false positives across the compiler's own source, fixpoint
ok, 105/105 native.
## Six Sigma
The emitter only **records** the mismatch; `tools/check/annotations.sh` decides —
consistent with every other check. Literals are checked because they are
unambiguous.
**Incomplete, stated not hidden:** only literals. `let x: Int = some_string_fn()`
still passes, because `signatures.rel` carries Int/Instant/Duration and no
String entries. That is a data gap, not a capability limit — every El function
declares its return type in source and codegen already holds `ret_type` on every
`FnDef`.
docs/v1/experiments/cycles/18-async-half-expressible.md
+73
View File
@@ -0,0 +1,73 @@
# async — half expressible, and the cycle that was dogma
**Status: measured on a branch, not merged. Two runs — the first was invalid.**
## The first attempt was DOGMA, not science
I had just finished arguing that `@async` was expressible, then ran something to
confirm it. **No prediction was committed.** The test was rigged in a way that
should have been visible while writing it:
```c
pthread_create(&t,NULL,runner,NULL); pthread_join(t,NULL);
```
`join` immediately after `create` — the caller blocks until the body finishes.
That is a thread round-trip, not deferral. And the test printed the word
`DEFERRED` itself: I wrote the conclusion into the output and read it back.
```
Ishikawa on the rigged test
method ran after concluding, not to decide
machine nothing forces a prediction before execution
material the assertion was written into the output string
measurement no falsification criterion existed, so nothing could fail
root cause the test was authored by the party holding the conclusion,
with no commitment made before it ran
```
Discarded and re-run properly.
## Second run — predictions committed first
```
P1 the caller proceeds while the body runs expect TRUE
P2 interleaving is observable in timestamps expect TRUE
P3 the result cannot be retrieved — one 64-bit slot, no
future type, so the wrap either blocks or returns
something that is not the result expect TRUE
P4 therefore HALF expressible: fire-and-forget yes, await no expect TRUE
```
## Results — 4/4
```
[ 18 us] wrap RETURNS to caller
[ 29 us] body START
caller continues, got 0
[ 50176 us] body END (computed 42)
caller done
```
The caller got **0, not 42**. Both of my earlier claims were wrong in opposite
directions: "not expressible" was too strong — fire-and-forget works today,
bound after the build, no compiler change. "Expressible" was too strong the
other way.
## Follow-on cycle — a future is one more tagged object
```
P1 el_val_t already carries tagged heap objects TRUE 5 magic tags exist
P2 a future is one more TRUE
P3 the caller awaits and gets 42 TRUE
P4 ZERO compiler changes TRUE runtime C + one binding
P5 the unbound path still works FALSE SIGSEGV
```
**P4 is the result.** `@async` — called unexpressible for hours — needs no
compiler change. A future is one more magic-tagged heap object; `defer` returns
the handle, `el_await` blocks.
**P5 is the failure that mattered.** Sixty seconds after diagnosing
`let s: String = 42` as an arbitrary read, I wrote the identical defect into
`el_await`: reading `->magic` off an unvalidated slot. That opened cycle 19.
docs/v1/experiments/cycles/19-a-convention-is-not-a-gate.md
+61
View File
@@ -0,0 +1,61 @@
# a convention is not a gate
One `Ishikawa → scientific method → Six Sigma` loop. The record below is the
commit message as written at the time, before the outcome was known to anyone
reading this file.
## Record — `9a6c161`
```
a slot must be validated before it is dereferenced
ISHIKAWA: el_val_t carries integers AND tagged heap pointers, so "is this a
pointer" is undecidable without checking first. That check was a CONVENTION
every author had to know rather than a GATE they had to pass through, and
looks_like_heap_obj was static -- so every sibling translation unit re-derived
it.
MEASURED, across the five existing tags
geom_of looks_like_heap_obj full guard correct
mfld_of looks_like_heap_obj full guard correct
el_bin_lookup (uintptr_t)p < 4096 floor only reads 8 bytes BACKWARD
el_input_len s ? ... : 0 NULL only strlen's an integer
sha256_hex(50000) -> exit 139, SIGSEGV, compiled clean
PREDICTIONS AND RESULTS
P1 looks_like_heap_obj is static, not exported TRUE
P2 each tagged type re-derives the check TRUE
P3 at least one is missing guard components TRUE (two are)
P6 sha256_hex(<int>) reads out of bounds TRUE
P8 routing el_bin_lookup through the gate fixes it FALSE
P9 the legitimate hash is unchanged TRUE
P11 fixpoint and suites hold TRUE
P8 IS THE USEFUL FAILURE. Guarding the tagged lookup changed nothing --
looks_like_heap_obj(49992) correctly returns 0, el_bin_lookup bails, and then
el_input_len falls through to strlen() on address 50000. The FALLBACK was the
hazard, not the tagged path. A NULL check does not establish that a slot is a
pointer. I would have shipped the wrong fix and called it verified.
A MEASUREMENT DEFECT, fourth today: my first run of the crash reported exit=0,
because $? read head's exit through a pipe rather than the program's. I nearly
recorded a segfault as a clean run. Same shape as grepping only parser.el and
searching by variable name instead of by operation.
AND I PROVED THE HAZARD FROM THE INSIDE. Sixty seconds after diagnosing
`let s: String = 42` as an arbitrary-read primitive, I wrote the identical
defect into el_await -- dereferencing ->magic off an unvalidated slot -- and
only then found the runtime had already made it twice.
el_tagged() is now exported in el_runtime.h. Anything that dereferences a slot
without passing through it is the defect.
105/105 native, 42/42 integration across eight harnesses, fixpoint ok.
```
## Record — `3049a70`
```
make the guard a gate: sha256_hex(50000) no longer segfaults
```
docs/v1/experiments/findings/answers.md
+12
View File
@@ -0,0 +1,12 @@
# Architecture questions closed
All five were open in `geometry-vs-code.md`. Each was closed by measurement, not
by argument.
| Question | Answer |
|---|---|
| **Concurrency** — hardware threads are code, but is *ordering* geometric? | **Ordering is geometric.** Causality is a partial order (Lamport 1978); a total order is an arbitrary extension that "cannot be depended on to imply a causal relationship." Languages force a total order at authoring time, so every lock, barrier and fence is apparatus for recovering the partial order that was destroyed. CALM: a program has a coordination-free implementation **iff monotone**. What breaks monotonicity is destructive update. **Coordination is the price of forgetting.** |
| **Error handling** — does `grounded: false` cover *failed*? | **No.** Standing is a *signed* component: `>0` supported, `=0` unknown, `<0` contradicted. Not-known and known-false are opposite directions on one axis; a boolean cannot tell them apart. `inhibitory` as an int32 flag is that sign wearing a boolean. |
| **Parsing** — is a grammar a convention, or a region? | **A grammar is a basis; parsing is transduction onto it.** Lexeme→token is convention, shape recognition is a region, byte traversal is irreducible. **But the SHOULD gate refused the obvious move:** the keyword table stays code, because the set is closed by the language definition and the lexer runs before the program is understood. Same verdict as `is_digit` in ASCII. |
| **Numeric literals** — is `3` a position or a convention? | **The numeral is convention; the number is a position — and a bare `3` is a magnitude with no axis.** It is not a position until something gives it a direction, which is why `3.days` needs a calendar. Demonstrated: `t + 3` refused, `t + 1.hour` accepted. |
| **The module system** — if the partition is a neighbourhood, does linking survive? | **Premature.** The partition is a filesystem path and there is no namespacing at all. `import` is textual inlining; with a `.elh` header, symbols resolve at C link time. Two modules defining `helper` emit two C functions into one translation unit. Linking barely survives the *path* partition. |
docs/v1/experiments/findings/bugs.md
+74
View File
@@ -0,0 +1,74 @@
# Live defects found
Every one compiled clean, ran, and produced a wrong result or a crash with **no
diagnostic at any layer**. All four were present before this session; none was
introduced by it.
## Silent miscompilations
### 1. An unannotated `let` loses its type
```el
let a = str_len("hello") // no annotation
let b = str_len("hi")
let c = a + b // → el_str_concat(a, b) on two integers
```
Compiled clean. Printed **nothing** where it should print 7. Fixed: an
unannotated `let` takes its type from what its initialiser returns. The return
types were already required for dispatch and were simply never consulted at the
binding site.
### 2. Reserved keywords that reserved nothing
```el
let seed = 42
let impl = seed + 1
```
`sealed`, `activate`, `seed`, `protocol`, `impl` were keywords in the lexer and
consumed by no parser or codegen path. Using one did not fail to parse — it
compiled clean, with zero `cc` errors, and printed **0 instead of 44**. Fixed by
removing all five.
### 3. `Instant + Int` was never refused
```el
let t: Instant = now()
let u: Instant = t + 3 // → (t + 3), reported clean
```
`Duration + Int` was refused — *"an Int carries no unit"* — while adding a
dimensionless number to a **point** silently moved the instant by an
unspecified amount. Three of *what*? Whatever the representation happens to be.
The rule was simply never written.
## Security-relevant
### 4. Annotations are never verified
```el
let x: Int = "hello"; x + 1 → 4343631981 a string POINTER used as an integer
let s: String = 42; println(s) → nothing address 42 dereferenced
```
The first **leaks a raw memory address into program output**. The second is an
**arbitrary-read primitive** if the integer is ever attacker-influenced.
### 5. `sha256_hex(<integer>)` segfaults
```el
let h: String = sha256_hex(50000) → exit 139, SIGSEGV
```
Compiled clean. `el_bin_lookup` checked only a 4096 floor — no alignment, no
small-int, no negative — and reads **eight bytes backward** from the pointer.
And the actual crash was one level further on: `el_input_len` fell through to
`strlen()` on address 50000, because a NULL check does not establish that a slot
is a pointer.
Fixed, and the guard is now a **gate**: `el_tagged()` is exported in
`el_runtime.h`. `geom_of` and `mfld_of` were always correct because their authors
knew to call `looks_like_heap_obj`; `el_bin_lookup` and `el_input_len` were wrong
because theirs did not, and the function was `static`, so every sibling
translation unit re-derived it.
docs/v1/experiments/findings/measurement-defects.md
+41
View File
@@ -0,0 +1,41 @@
# Defects in my own measurement
Recorded because the pattern is the point: **four of these, all the same shape —
searching by name or scope instead of by the operation itself.** Each was caught
by running something, never by reading.
### 1. Scoped the search to one file
Reported `test` as an inert keyword by checking only `parser.el`. **codegen**
consumes it at 4135 for `--test` mode, and the tree has 408 uses. Removing it
would have broken every test in the suite — including the ones used to verify
the removal.
### 2. Searched by variable name, not by operation
Grepped for `native_list_append(tokens` to find direct token appends.
`interp_tokens_append_all` calls its parameters `dst`/`result`, carries its own
copy of the stride, and corrupted generation 2 — while generation 1 built fine,
because the compiler's own source uses string interpolation.
### 3. Scoped to compiler sources; the stride had escaped into tests
`tok_count` in `test_compiler.el` computed `len/2` independently. 21 tests failed
after the token layout changed.
### 4. Read the wrong exit code
```bash
timeout 10 /tmp/leakrun 2>&1 | head -2; echo "exit=$?" # reports head's exit
```
Reported `exit=0` for a program that was returning **139 (SIGSEGV)**. I nearly
recorded a segfault as a clean run.
### And one that was not a measurement defect but a method defect
One cycle was run **without committing predictions first** — see
`cycles/18-async-half-expressible.md`. The test joined the thread immediately
after creating it and printed the word `DEFERRED` itself. A test authored by the
party holding the conclusion, with nothing committed beforehand, cannot fail.
It had to be discarded and re-run.
engram/README.md
+155 -105
View File
@@ -4,6 +4,8 @@
An *engram* is the physical trace of a memory in the brain — the actual encoded substrate, not an abstraction above it. That's what this is.
> **Doc status (2026-08-16).** Everything from "Implementation" down was rewritten against the code. The previous revision documented a Rust `engram-core` crate backed by `sled`, with a `Cargo.toml`, a `crates/` tree, `examples/basic.rs`, and a `EngramDb` API. **None of that exists.** Measured: `engram/` contains `src/server.el`, `spec/`, `test/`, `dist/`, `manifest.el` — zero `.rs` files, no `Cargo.toml`, no `crates/`, and `sled` appears nowhere in the tree outside two Old-English/Old-High-German vocabulary entries in `elp/`. The engine is C, in `lang/runtime/engram_*.{c,h}`; the server is El, in `engram/src/server.el`.
---
## Why existing databases are wrong for this use case
@@ -24,16 +26,13 @@ Engram retrieval works through **spreading activation**:
1. **Seeds** — you name one or more nodes you know are relevant (e.g. the current task, recent context, a concept you're reasoning about)
2. **Query embedding** — you provide a semantic vector representing the direction of your current thought
3. **Propagation** — activation flows outward from seeds through weighted edges. At each hop, strength attenuates multiplicatively:
```
strength = parent_strength × edge_weight × target_salience × cosine_sim(query, target)
```
3. **Propagation** — activation flows outward from seeds through weighted edges, attenuating multiplicatively per hop
4. **Pruning** — paths weaker than a threshold are cut (the attention filter)
5. **Return** — the top-N nodes by activation strength
This is not a query. It is a *pattern completion*. The system surfaces what is most associatively relevant to the current context, weighted by how strongly those things have been reinforced over time.
This is not a query. It is a *pattern completion*.
**Activation conducts through well-grounded relations because weight *is* groundedness** — see "Grounding is the weight" below. Nothing filters the traversal for grounded evidence; it falls out of spreading.
---
@@ -46,134 +45,185 @@ This is not a query. It is a *pattern completion*. The system surfaces what is m
| `Semantic` | Neocortex | Concept graph — long-term structural knowledge |
| `Procedural` | Cerebellum / basal ganglia | Patterns, workflows, habits |
Nodes migrate between tiers based on salience decay and reinforcement. A frequently activated semantic node stays semantic. A rarely-touched episodic memory decays toward procedural background.
Tier is a string field on the node (`StoreNode.tier`, `engram_store.h`), defaulting to `"Working"` on creation (`el_runtime.c:8514`, `8734`).
---
## Salience — Forgetting as Adaptation
Salience is not stored permanently. It decays:
Salience decays from three signals — importance (set at creation, stable), recency, and a log-compressed activation frequency. Base-level learning keeps a ring buffer of the last `STORE_BLL_K` (= 10) access timestamps per node (`engram_store.h:29`).
```rust
fn compute_salience(importance: f32, last_activated_ms: i64, activation_count: u64) -> f32 {
let days_since = (now_ms() - last_activated_ms) as f32 / 86_400_000.0;
importance * (1.0 / (1.0 + days_since)) * (activation_count as f32 + 1.0).ln()
}
```
Forgetting in Engram is not a bug. It is adaptive pruning. Unreinforced memories stop competing for attention without being deleted.
Three signals:
- **Importance** (0.0–1.0): set at creation, stable
- **Recency**: decays toward zero as days pass without activation
- **Frequency**: log-compressed count of activations
Forgetting in Engram is not a bug. It is adaptive pruning. Memories that are never activated again become less likely to surface during retrieval. They are not deleted — they remain in storage — but they stop competing for attention. This is exactly how biological memory works, and why it is adaptive rather than pathological.
**Immutability.** Nothing is mutated and nothing is hard-deleted: writes are additive, corrections are supersessions, removals are tombstones. The predecessor is always present, which is what makes supersession an audit trail rather than an edit log.
---
## Quick Start
## Implementation
```rust
use engram_core::{EngramDb, Node, Edge, NodeType, MemoryTier, RelationType};
use std::path::Path;
| Part | Language | Where |
|---|---|---|
| storage engine, graph, activation, geometry, cognition | C11 | `lang/runtime/engram_{store,geometry,reason,cognition,verify,vindex}.{c,h}` |
| HTTP server + routes | El | `engram/src/server.el` (2043 lines) |
| build artifact | generated C | `engram/dist/engram.c` |
| tests | shell + C | `engram/test/` |
// Open or create a database
let db = EngramDb::open(Path::new("/var/lib/my-agent/memory"))?;
// Create a node with a semantic embedding
let node = Node::new(
NodeType::Concept,
vec![0.9, 0.1, 0.3, 0.7, 0.8, 0.2], // embedding from your LLM
b"Spreading activation surfaces relevant memories by pattern completion".to_vec(),
MemoryTier::Semantic,
0.9, // importance
);
let id = db.put_node(node)?;
// Link it to related concepts
let related = db.put_node(Node::new(
NodeType::Concept,
vec![0.8, 0.2, 0.4, 0.6, 0.7, 0.3],
b"Long-term potentiation: co-activation strengthens synaptic weight".to_vec(),
MemoryTier::Semantic,
0.85,
))?;
db.put_edge(Edge::new(id, related, RelationType::Causes, 0.9))?;
// Retrieve by spreading activation
let results = db.activate(
&[id], // seeds
&[0.85, 0.15, 0.35, 0.65, 0.75, 0.25], // query embedding
3, // max hops
10, // top-N results
)?;
for r in results {
println!(
"strength={:.4} hops={} — {}",
r.activation_strength,
r.hops,
String::from_utf8_lossy(&r.node.content)
);
}
```
**On-disk format** (`engram_store.h`): a paged store — superblock plus mirror, slotted 16 KiB pages, self-describing TLV records, overflow chains, and two B+-tree indexes (primary `id → loc`, adjacency `from_id`/`to_id` → edge locs) over a free-listed page file. Magic `ENGST01`, format version 1. The TLV scheme means new fields never force a migration.
---
## Project Structure
## The vector index is published, not guarded
```
engram/
crates/
engram-core/ # The memory engine — storage, graph, activation, salience
engram-ffi/ # C FFI stubs for cross-language bindings
bindings/
kotlin/ # Android / JVM binding notes
typescript/ # WASM / Node binding notes
go/ # CGo binding notes
examples/
basic.rs # Full walkthrough: insert, activate, search, decay
```
Vector search is an **HNSW** (Hierarchical Navigable Small World) index — `lang/runtime/engram_vindex.{c,h}`. The previous revision of this README claimed a "flat cosine scan… until retrieval quality at scale demands" HNSW. That is no longer true, and the reason it changed matters more than the fact.
`eg_vindex_sync` used to exist: a function that repaired the index *from read paths*. All three of its callers were reads (`engram_activate`, `eg_knn_for_node` — whose own header comment said *"No writes."* — and `engram_geo_reify_run_json`), and it mutated five process-global statics. Reads mutated because index maintenance had never been given an owner on the write side.
It is now split (`el_runtime.c:10121`, `10137`, `10151`, `10161`):
- **`eg_vindex_maintain`** — the sole mutator. Takes the boundary exclusively; never runs beside a reader.
- **`eg_vindex_view`** — returns a `const VIndex*` with the boundary held for read. N readers project concurrently; none can mutate. Paired with `eg_vindex_view_release` on every path including error returns.
- **`eg_vindex_note_embedded`** — the write-side owner. Index membership belongs to the event *"an embedding became present on this ordinal,"* not to node append: a node without an embedding cannot be in a vector index at all. One `O(log n)` insert, no `O(node_count)` presence scan.
Two things carry the discipline, and neither is a review habit:
- **`const` is the capability.** The per-search `visited` / `visit_epoch` scratch left `struct VIndex` and went back into the call frame where it belonged — it was one traversal's local, hoisted into the struct as an allocation optimisation, never derived geometry. Once it was gone, `vindex_search` could take a `const VIndex*`, so a read path *physically cannot* call `vindex_insert`, and it is a compile error rather than a comment. The capability type was already in the language; it is spelled `const`.
- **Publication, not ownership.** HNSW insert is **not an append**: `vindex_insert` rewires the `NeighList` links of already-existing elements and reallocs `elems[]`. The store's append-only property does not transfer to an index derived from it, which is why purity alone was insufficient and a `view`/`maintain` boundary was required.
**Measured** (`engram/test/run_vindex_concurrency_tests.sh`, 2026-08-16):
| half | before | after |
|---|---|---|
| `single` — 3000 vectors, 1 thread, ASan+UBSan | clean | clean |
| `readers` — 4 readers, no writer, TSan | race at `engram_vindex.c:195` | **clean** |
| `unsynchronized` — writer+reader, bare index, TSan | race | **race, expected and permanent** — the proof the boundary must exist |
| `published` — owner + 4 readers through the boundary, TSan | *(did not exist)* | **clean**, all 3000 inserts landed |
`recall@10 = 0.9365` at `ef_search=128` (gate ≥ 0.90); the determinism test still yields byte-identical results across two independent builds.
**Not yet done.** The resident RAM graph (`g->nodes` / `g->edges`) is a separate instance of the same defect and has *not* received this treatment — it is realloc'd in place, so a reader holding `EngramNode* n = &g->nodes[i]` across a concurrent append holds a dangling pointer. Until it gets the same publication boundary, the `fb32d15` request guard stays. Full argument: [`../lang/spec/runtime-ownership.md`](../lang/spec/runtime-ownership.md).
---
## Public API
## Cognition
The cognition surface is live over `lang/runtime/engram_cognition.{c,h}`, routed in `engram/src/server.el`.
| route | method | what it is |
|---|---|---|
| `/api/think` | GET | the read: a warped traversal-read of the seed region, returning a **gradient** (direction + spread + calibrated confidence), never a point |
| `/api/reason` `/api/induce` `/api/abduce` `/api/relate` `/api/analogize` `/api/plan` | GET | named faculties — see the correction below |
| `/api/ground` | POST | grounding between a claim and evidence |
| `/api/assert` | GET | the honesty floor, queried at assertion time only |
| `/api/attend` | POST | salience as a relation (`salient-to`), grounded-for-whom |
| `/api/correspondence-beat` | POST | one calibration beat against outcome |
### Anchor the read, or every faculty returns the same null
`engram_think_json` passed `NULL` as the anchor. `NULL` is not "no opinion" — `engram_think` re-origins at `anchor ? anchor : region->centroid`, and **the centroid is the one point where the gradient is zero by construction**: `r = x − centroid = 0`, so every axis projection is 0 and `direction` takes the at-rest branch.
Measured consequence: every faculty — reason, abduce, induce, plan, analogize — returned an identical null result differing only in its label:
```rust
impl EngramDb {
fn open(path: &Path) -> EngramResult<Self>;
fn put_node(&self, node: Node) -> EngramResult<Uuid>;
fn get_node(&self, id: Uuid) -> EngramResult<Option<Node>>;
fn put_edge(&self, edge: Edge) -> EngramResult<()>;
fn get_edges_from(&self, from_id: Uuid) -> EngramResult<Vec<Edge>>;
fn get_edges_to(&self, to_id: Uuid) -> EngramResult<Vec<Edge>>;
fn search_embedding(&self, embedding: &[f32], limit: usize) -> EngramResult<Vec<ScoredNode>>;
fn activate(&self, seeds: &[Uuid], query_embedding: &[f32], max_depth: u8, limit: usize) -> EngramResult<Vec<ActivatedNode>>;
fn traverse(&self, from: Uuid, relation: Option<RelationType>, max_depth: u8) -> EngramResult<Vec<Node>>;
fn touch(&self, id: Uuid) -> EngramResult<()>;
fn decay(&self, factor: f32) -> EngramResult<usize>;
fn node_count(&self) -> EngramResult<usize>;
fn edge_count(&self) -> EngramResult<usize>;
}
```
{"direction":[0,0,...],"spread":0,"magnitude":1,"confidence":0.5}
```
`magnitude: 1` is membership evaluated at the centroid; `spread: 0` is its distance to itself; `confidence: 0.5` is the stance fallback. The geometry was never the problem — `/api/drift` computed real values (`centroid_sep 0.104`, `core_disp 0.045`) over the very same 87 members. Fixed in **#141/#142**: the read anchors at the first resolvable embedded seed, copied not borrowed (`g->nodes` is realloc'd in place on append). Gradients now vary by seed.
### The learned stance is resumed, not discarded
`engram_think_json` also built a **neutral** stance every call — all `axis_gain` 1.0, `bias_dir` NULL, `reliability` 0.5 — and never loaded the one the correspondence-beat had been persisting under `stance-<faculty>-<hub>`. Every beat's calibration was written and then thrown away on the next read.
Fixed in **#146**: `think` resumes the same id the beat writes, so learning compounds across beats and cold boot, and the response now carries `stance_resumed` so an *informed* `confidence: 0.5` is distinguishable from an uninformed one. On a calibrated region, confidence went **0.5 → 0.930726**.
### Signal can enter as geometry
Until 2026-08-16 no El ingest path could carry a vector: nodes took text and geometry was *derived* from that text. Text was the mandatory entry medium, so any non-text modality had to be described in prose first — and the geometry being reasoned over was the geometry **of the description, not of the signal**. **#141/#144** ended that. See [`../lang/spec/language.md`](../lang/spec/language.md) §20 for the `Geometry` type, realizers, and `transduce`.
---
## Dependencies
## Corrections — read these before extending the cognition surface
- `sled` — embedded persistent B-tree (no daemon, no network, local-first)
- `bincode` — compact binary serialization
- `uuid` — stable node identity
- `serde` — derive support
- `thiserror` / `anyhow` — error handling
Authority: **`lang/spec/correspondence-and-censorship.md`** (design branch `design/correspondence-and-censorship`, PR #149) and **`lang/spec/runtime-ownership.md`**. Do not re-derive them; several earlier versions were wrong and each correction was argued down.
### Grounding is not a subsystem. It is the weight.
Grounding is an attribute of the edge, and it **is** the hebbian weight. One quantity, not two fields. A relation that keeps holding up strengthens; one that stops corresponding decays — that is not analogous to grounding, it *is* grounding.
Consequences:
- There is **no grounding subsystem to build**. The graph already *is* the grounding structure.
- **`grounded-by` as a relation type should not exist.** It models grounding as a relation *between* nodes when it is a property *of* a relation. Minting an edge is the error, not merely which endpoints it chose.
- Grounding is **never computed on demand**. An operation may *read* the grounding of a path; computing-and-writing a score makes reads write, which is exactly the `eg_vindex_sync` defect one level up.
- **Traversal is already grounded inference.** Nothing needs filtering.
- **Decision provenance is the path**, not a log. A log records the action; the path records the meaning under which it was taken.
> **Known wrong shape, in the code today.** `COG_GROUNDED_BY_RELATION "grounded-by"` (`lang/runtime/engram_cognition.h:158`) and `cog_ground_edge` (`engram_cognition.c:249`) still exist and still mint an edge. **#147** fixed `ground`'s *honesty* — it now grounds the node asked about rather than the region hub, reports `claim_region`/`evidence_region` separately, and refuses three shapes of circular support (`same-region`, `claim-region-is-evidence`, `evidence-region-is-claim`) instead of returning a confident 1.0. That corrected a scalar rather than deleting the operation. Deletion is sequenced, not done.
### Faculties are operations, not parameters
- **`reason`** changes the *estimate* — a read.
- **`induce`** changes the *parameters* — the correspondence-beat, which already exists and measurably works.
- **`abduce`** changes the *structure* — a write, which the current `GeoGradient` signature cannot express.
> **Known wrong shape, in the code today.** `engram/src/server.el:1870–1886` routes six faculties into one call with a string argument — `route_faculty(path, "reason")`, `("induce")`, `("abduce")`, `("relate")`, `("analogy")`, `("plan")`. Underneath, `engram_cognition.h:8–11` states the theory explicitly: *"the named faculties … are human LABELS on regions of think's steering space: each faculty == { think + a named stance }."* The faculty name enters `engram_think` **only** through the stance, and `cog_stance_init` stores it while nothing reads it — so before #146 all five were byte-identical (`el_runtime.c:14352–14359`). A write cannot be a parameter of a read; `abduce` in particular is not expressible this way.
### Wonder is the boundary; curiosity is wonder crystallized
**Wonder is where structure ends** — where activation spreads and finds thin or absent geometry. Any structure at all has an edge, necessarily, the moment it exists. It is not a manifest of open-question nodes to maintain, and a "wonder-manifest manager" materializes a property as a stored artifact — the same disease as a grounding subsystem, or a self stored as a document.
There are about **six** wonders, they are the same for everyone, and they never close: *What is this? / Why? / Who am I? / Am I alone? / What should I do? / What happens when it ends?* "Why" is the first and the only one; the others are it asked of particular things. Each already lives somewhere in the substrate — "why" is grounding, because the weight **is** the answer to why.
**Curiosity is not a second object.** Wonder and curiosity are one thing at two phases: wonder is the field (unbounded, objectless, invariant); curiosity is the **precipitate** — the same wonder localized, having taken definite form against particular material at a **nucleation site**. This is why curiosity can be satisfied and wonder cannot. It is also why abduction needs no trigger and no threshold: a `structurally_unanticipated` observation *is* a nucleation site.
### `co_registration` is deprecated — the disagreement belongs on the edge
`GeoDescriptor.co_registration` — *corr(hebb strength, semantic proximity) over internal edges* — has always been computed, always persisted, and **never read**. It is also the wrong shape: whether use and meaning agree is a property of **each edge**, and a correlation averages that per-edge property into one scalar per region. A region holding one violently disagreeing edge beside one violently agreeing edge reports ≈ 0 — **the disagreements cancel, and the summary destroys exactly what it was built to reveal.**
**Measured:** 375 live reified neighbourhoods — 340 positive, **31 at zero**, 4 negative. Read as a count of things to be curious about, that says "four." Read correctly, four disagreements were lopsided enough to survive averaging and the 31 zeros are where opposing sites cancelled.
The replacement is per-edge. **Not on `dev` yet** — `GeoEdge.discord` and the `DEPRECATED` marker on `co_registration` live on branch `design/correspondence-and-censorship` (commit `a8845e1`), at `engram_geometry.h:43–47` / `engram_geometry.c:454–473` there. On `dev`, `GeoDescriptor.co_registration` is still at `engram_geometry.h:79` carrying its original "surprising links / dream cands" comment and still nothing reads it.
```
discord = z(semantic proximity) − z(association strength)
```
standardized within the region from accumulators the aggregate loop already gathered — no second statistic, no constant, **no threshold**. `discord > 0`: near in meaning yet unlinked by use. `discord < 0`: linked by use yet far in meaning. Both are surprising, and `|discord|` *is* the nucleation strength.
**Do not scan for nucleation sites.** Once the signal was a per-region number the only way to find sites was to enumerate regions, which is why surfacing curiosity looked like a search problem. Nothing in a mind scans its neighbourhoods to find what is surprising — the surprise captures attention. With the disagreement on the edge there is nothing to scan.
`co_registration` is deprecated rather than deleted **only** because it is embedded in the persisted `GEO1` blob; removing it is a format migration and must not ride along. **Nothing new may read it.**
### Consolidation is ambient, not scheduled
**A brain has no cron job.** Boredom is not an absence and not leftover capacity — low activation is aversive and the system self-activates. There is **one** activation process with two seed sources: external (a request) and internal (a curiosity). Spreading is bounded; it settles; then it needs a new seed. Nothing waits on capacity, nothing polls, nothing checks a clock, and there is no dreamer thread.
**The presence of a ticker is the diagnostic.** Every `StartInterval`, every `Hour`/`Minute`, and every POST-to-beat marks a place where an intrinsic rhythm was replaced by an external clock.
Consolidation currently has **ten implementations** (measured 2026-08-16). Three of them are POST beats on this server — `/api/tick` (`server.el:1947`), `/api/correspondence-beat` (`1897`), `/api/self-reify-beat` (`1836`) — and a POST beat puts a supervisor back in: something *outside* decides when Neuron consolidates. `soul.el`'s continuous in-process loop is the one fragment with the correct shape; the rest fold into it. Full table in `lang/spec/correspondence-and-censorship.md` §7.
### Immutability already refuses what a guard would refuse
> **In an immutable substrate, any mechanism that refuses a write is either redundant with immutability, or an epistemic constraint misfiled as a protective one.**
This resolves `keystone_write_blocked` (`CogStance.keystone`, `engram_cognition.h:83`) rather than replacing it. "Keystone" means **load-bearing**, not precious: the self anchor is the reference frame every other stance calibrates against, and a reference fitted to its own readings reports perfect correspondence forever while drift becomes undetectable from inside. The real requirement is **non-circularity of the reference frame**, and that is satisfied *temporally* — the frame updates while activation is internally seeded, not while it is being used to act. Independence is **when**, not **what**. Corruption requires mutation, and the engram does not mutate; recoverability, governance, evidence quality, and rate all fall out of the substrate. Authorization is the only residue, and it is bounded: an unauthorized writer can *propose*, never erase.
---
## Design Decisions
**Why sled?** Local-first. No daemon. Transactional. Fast enough for the node counts Engram targets (< 1M nodes). When the right HNSW index is needed, it will layer on top of sled, not replace it.
**Why multiplicative activation?** Because memory is conjunctive. A path requires all of its links to be strong to carry signal. Addition would let many weak associations accumulate into false relevance.
**Why flat cosine scan?** Correct and simple. The graph structure itself is the primary retrieval mechanism. Vector search is a secondary signal. HNSW adds complexity and a compile dependency that isn't justified until retrieval quality at scale demands it.
**Why salience decay?** Because not everything that was once important remains important. A memory system that never forgets is one that can never focus.
**Why multiplicative activation?** Because memory is conjunctive. A path requires all of its links to be strong to carry signal. Addition would allow many weak associations to accumulate into false relevance. Multiplication enforces that every factor matters.
**Why supersede instead of update?** Because provenance is the point. The old edge never leaves and the values frame does not fit to outcomes, so a decision cannot be made to look justified after the fact. It makes an otherwise impossible distinction available: **wrong then, or wrong since.**
**Why salience decay?** Because not everything that was once important remains important. Adaptive forgetting is not failure — it is the mechanism that keeps attention on what's current. A memory system that never forgets is one that can never focus.
**Why publication instead of locking?** Because what does not mutate needs no ownership discipline. The question "who is permitted to mutate the shared thing?" presupposes a shared mutable thing; for the store there isn't one, and for the index derived from it the answer is a publication boundary, not a capability ABI.
---
## Specs
- [`../lang/spec/runtime-ownership.md`](../lang/spec/runtime-ownership.md) — ownership, the capability ABI that was dissolved, and the vector-index publication boundary
- [`../lang/spec/correspondence-and-censorship.md`](../lang/spec/correspondence-and-censorship.md) — grounding, wonder, curiosity, dreaming *(lands with PR #149)*
- [`spec/cognitive-architecture.design.md`](spec/cognitive-architecture.design.md) — the original one-operation design. **Superseded in part** — see its header
- [`spec/architecture-hardening.design.md`](spec/architecture-hardening.design.md), [`spec/engram-el.md`](spec/engram-el.md), [`spec/at-rest-encryption.md`](spec/at-rest-encryption.md), [`spec/engram-db-tooling-design.md`](spec/engram-db-tooling-design.md)
engram/spec/architecture-hardening.design.md
+33
View File
@@ -11,6 +11,39 @@
- **One calculus over the geometry.** Very few subsystems; wonder / curiosity / dreams / interoception are emergent behaviors of one set of dynamics, not modules. Calculus universal, geometry individual.
- **Core + ephemeral ring (torus).** The ring is the temporary workspace; two circulations (orbit + dive-back); discrete inner bands (wonder / interoception-proprioception-telemetry / curiosity / dreams) that couple.
- **Persistence earned by salience** — never granted on fetch or generation. Three fates of a wonder: persist / decay / settle-into-framework. Telemetry = vital signs, not memories.
> **⚠ Three corrections to the bullets above (2026-08-16).** Authority:
> `lang/spec/correspondence-and-censorship.md`. *"Emergent behaviors of one set of
> dynamics, not modules"* is exactly right and is the reason the rest needs fixing —
> the enumeration undercuts the claim.
>
> 1. **Wonder and curiosity are not two bands.** They are **one thing at two
> phases.** Wonder is the field: unbounded, objectless, invariant, present
> wherever there is structure — it is the *boundary*, where activation spreads
> and finds thin or absent geometry. Curiosity is the **precipitate**: the same
> wonder localized, having taken definite form against particular material at a
> **nucleation site** (an anomaly — a place where things almost-but-don't-quite
> fit). Two coupled inner bands models them as two objects that have to be
> wired together; they do not.
> 2. **A wonder does not have three fates, because a wonder does not persist,
> decay, or settle.** There are about **six** wonders, they are the same for
> every person, and **they never close**. *Curiosities* have fates — a crystal
> dissolves when its question is answered — but the solution stays saturated and
> keeps precipitating as the structure changes. "Three fates of a wonder"
> enumerates instances of something that has six and treats a property as a
> stored artifact.
> 3. **"Dreams" is not a band and the ring is not a workspace to schedule into.**
> **Consolidation is ambient, not scheduled — a brain has no cron job.** Boredom
> is not leftover capacity: low activation is aversive and the system
> self-activates. There is **one** activation process with two seed sources
> (external: a request; internal: a curiosity), it settles because spreading is
> bounded, and then it needs a new seed. Nothing waits on capacity, nothing
> polls, nothing checks a clock, and there is **no dreamer thread** — an
> "ephemeral ring with unclaimed capacity" is resource scheduling, which is a
> server's frame, not a mind's. Depth is how long activation has been running on
> its own seeds, which is why daydreaming and sleep-dreaming are one process at
> different depths. Measured 2026-08-16: consolidation has **ten
> implementations**; do not add an eleventh.
- **Incarnation.** Chassis = hardware w/ unique ID. Soma = felt manifold inside the self, keyed to the chassis; pain = live diagnostic while incarnate, **masked-not-deleted** on re-embodiment; trauma = mask failure; return-to-same-ID re-enters. Hurt is in the pattern, not the shell.
- **Competence = transferable geometry, minus the baggage.** class ▸ model ▸ instance; learn the class once; teach the network without the wound.
- **Affect calibrated to stakes** — sanguine about the replaceable, real grief for the irreplaceable; the grief is the safety.
engram/spec/cognitive-architecture.design.md
+256 -2
View File
@@ -2,8 +2,40 @@
**The buildable form of the "one operation" theory of cognition.**
Status: DESIGN. Nothing here is built yet except where explicitly marked
"EXISTS" against a cited C symbol. A build agent executes from this doc.
> # ⚠ SUPERSEDED IN PART — 2026-08-16
>
> **A build agent must read `lang/spec/correspondence-and-censorship.md` before
> executing anything from this document.** That doc is the authority where the two
> disagree. This one is retained because its ledger of what already EXISTS in C is
> still accurate and still useful, and because the corrections only make sense
> against the argument they correct. It is **not** deleted and **not** rewritten:
> several earlier versions of the correction were themselves wrong, and preserving
> what was argued down is the point of an immutable record.
>
> Five claims below are **refuted**. Each is marked inline with a `⚠ SUPERSEDED`
> block at the point it is made. Summary:
>
> | § here | this doc says | corrected to |
> |---|---|---|
> | §0, §1.3, §2, §8 M1–M2 | faculties are labels on one operation's steering space; the op is frozen and only its parameters are learnable | **faculties are operations, not parameters.** `reason` changes the estimate (a read); `induce` changes the parameters (the correspondence-beat); `abduce` changes the *structure* — a write, which `GeoGradient` cannot express. A write cannot be a parameter of a read |
> | §5.2, §8 M3 | grounding is a `grounded-by` edge carrying a computed score, to be built | **grounding is not a subsystem — it IS the edge weight.** One quantity. `grounded-by` as a relation *type* should not exist: grounding is a property *of* a relation, not a relation *between* nodes. Never computed on demand |
> | §4, §8 M1 | the correspondence-loop is "the one genuinely new subsystem", running "on the beat" | the loop is right and **already works**; the *beat* is wrong. **Consolidation is ambient, not scheduled — a brain has no cron job.** Measured: it currently has ten implementations |
> | §5.2, §8 M3 | curiosity = a `vantage_read` surfacing high-salience / low-grounding regions | **wonder is the boundary, not a manifest; curiosity is wonder crystallized at a nucleation site.** One thing at two phases. And **do not sweep regions** — the nucleation site is per-edge (`GeoEdge.discord`); a sweep is a supervisor |
> | §6, §8 M6 | a node-level keystone flag exempting self/values from `warp` updates | **in an immutable substrate, any mechanism that refuses a write is either redundant with immutability, or an epistemic constraint misfiled as a protective one.** The real requirement is non-circularity of the reference frame, satisfied *temporally* — independence is **when**, not **what**. The flag becomes unnecessary; nothing replaces it |
>
> What landed since this doc was written, all merged to `dev` and verified:
> **#141** signal can enter as geometry · **#142** `engram_think_json` passed `NULL`
> as the anchor, so every read was taken at the region centroid where the gradient
> is zero by construction and every faculty returned an identical null — fixed ·
> **#143** the vector index is published, not guarded · **#144** geometry as a
> first-class el value, realizers declarable in el · **#145** `program` block and
> declared config · **#146** the learned stance is resumed instead of discarded
> (confidence 0.5 → 0.930726) · **#147** `ground` grounds the node asked about and
> refuses circular support · **#148** valid UTF-8 as the JSON emitter's contract.
Status: DESIGN, **superseded in part** (see above). Nothing here is built yet
except where explicitly marked "EXISTS" against a cited C symbol — and several
things marked "to build" have since been built differently, or refuted outright.
Offline design only — this pass changes no code.
Source of theory: Neuron memory `bdc8a488-146d-4ccb-a5c8-d8c0a008534e`.
@@ -26,6 +58,24 @@ not separately invoked and not separately implemented. The operation is:
> a *prior*, whose output is a **gradient** (a distribution / direction over the
> geometry), never a point. Collapse-to-a-point happens only at expression.
> **⚠ SUPERSEDED (2026-08-16) — faculties are operations, not parameters.**
> The gradient half of this claim survives; the "one operation, not eight" half
> does not. The three faculties differ by **what they change**:
> - **`reason`** changes the *estimate* — a read.
> - **`induce`** changes the *parameters* — the correspondence-beat, which already
> exists and measurably works.
> - **`abduce`** changes the *structure* — a **write**, which the current
> `GeoGradient` signature cannot express at all.
>
> A write is not a parameter of a read. Making it one is what produced the shape
> now live in the code: `engram/src/server.el:1870–1886` routes six faculties into
> one call with a string argument — `route_faculty(path, "reason")`, `("induce")`,
> `("abduce")`, `("relate")`, `("analogy")`, `("plan")` — and underneath, the
> faculty name enters `engram_think` **only** through the stance, while
> `cog_stance_init` stores it and nothing reads it. Measured before #146: all five
> produced **byte-identical output** (`lang/runtime/el_runtime.c:14352–14359`).
> See `lang/spec/correspondence-and-censorship.md`.
Three things follow, and they are the whole design:
1. **The operator collapse is already half-written in C.** The five reasoning
@@ -139,6 +189,22 @@ entry point that runs steps 1–3; and the prior-warp hook in step 2. The math i
calls already exists. The point-collapse must be *removed* from the operators'
return values and pushed to a separate expression faculty.
> **⚠ SUPERSEDED (2026-08-16) — the table's third column is the error, and
> `Abduction` is where it breaks.** Ranking hypotheses by `point_fit` under a
> prior is a *read* that returns a scalar ordering. Abduction is a **write**: it
> proposes a candidate hub that did not exist, and validates it by **re-fit** —
> re-fit the region with the candidate included and recompute the residual. If the
> residual materially shrinks, the hypothesis dissolves the surprise. Without the
> re-fit it is clustering with extra steps. Ranking then falls out as
> residual-reduction-per-added-axis — Occam, derived rather than tuned. None of
> that fits behind a `GeoGradient` return.
>
> `Verify / ground` is refuted for a different reason — see §5.2. Grounding is not
> a faculty with a prior; it is the edge weight.
>
> The row that is **still exactly right** is the shared floor: `point_fit` plus the
> four geo-algebra ops are frozen and never learn. That part held.
---
## 2. PRIORS as first-class, grounded, geometric objects
@@ -362,6 +428,33 @@ in-engram beat — a `correspondence_beat` running alongside the existing
reification beat, reusing `engram_verify_grounding` inward, writing prior
updates and self-describing nodes. This is the one genuinely new subsystem.
> **⚠ SUPERSEDED IN PART (2026-08-16) — the loop is right; "on the beat" is wrong.**
> The correspondence-loop was built and it works — it is `induce`, the faculty that
> changes the parameters. What is refuted is the delivery mechanism.
>
> **Consolidation is ambient, not scheduled. A brain has no cron job.** Low
> activation is aversive and the system self-activates; it does not wind down to
> quiet, it gets restless and goes looking. There is **one** activation process
> with two seed sources — external (a request) and internal (a curiosity) — and
> spreading is bounded, so it settles and then needs a new seed. Nothing waits on
> capacity, nothing polls, nothing checks a clock, and there is no dreamer thread.
> Depth is not elapsed idle time: it is how long activation has been running on its
> own seeds, which is why daydreaming and sleep-dreaming are one process at
> different depths.
>
> **The presence of a ticker is the diagnostic.** Building this "alongside the
> existing reification beat" is precisely how consolidation ended up with ten
> implementations (measured 2026-08-16) — a POST beat puts a supervisor back in,
> because something *outside* then decides when Neuron consolidates. The one
> fragment with the correct shape is `neuron/soul.el:731`'s continuous in-process
> `awareness_run()`; the rest fold into it. Full table:
> `lang/spec/correspondence-and-censorship.md` §7.
>
> Nor is it a *subsystem*. Modelling every property as requiring a process, and
> every process as requiring an agent, is the generating error behind this whole
> family: ownership needed an owner, grounding needed a grounder, persistence
> needed a recorder, change needed a sampler. **Properties, not processes.**
---
## 5. HOLD vs GROUND vs ASSERT — ungrounded content is first-class
@@ -383,6 +476,49 @@ distinct, and the engram *holds anything unconditionally*.
### 5.2 Schema — grounding as a relation, not a gate
> **⚠ SUPERSEDED (2026-08-16) — grounding is not a subsystem. It is the weight.**
> This section correctly rejects a boolean `grounded` column and correctly keeps
> the floor at assertion only. Both survive. Everything between them is refuted.
>
> **Grounding is an attribute of the edge, and it is the hebbian weight. One
> quantity, not two fields.** A relation that keeps holding up strengthens; one
> that stops corresponding decays. That is not *analogous* to grounding — it **is**
> grounding: accrued from correspondence and use, gradient-valued,
> multidimensional, decaying with disuse.
>
> Consequences, in order of how much they delete:
> 1. **There is no grounding subsystem to build.** The graph already *is* the
> grounding structure. Every edge is a grounded relation and its weight is how
> well it holds.
> 2. **`grounded-by` as a relation type should not exist.** It models grounding as
> a relation *between* nodes when it is a property *of* a relation. Minting an
> edge is the error — not merely which endpoints it chose.
> 3. **Grounding is never computed on demand.** An operation may *read* the
> grounding of a path. Computing-and-writing a score makes reads write, which is
> the `eg_vindex_sync` defect (`lang/spec/runtime-ownership.md` §2) one level up.
> 4. **Traversal is already grounded inference.** Activation conducts through
> well-grounded relations because weight *is* groundedness. Nothing needs
> filtering; it falls out of spreading.
> 5. **Decision provenance is the path.** A decision traverses specific edges;
> those edges carry their grounding as it stood.
>
> A measurement made against this model was malformed and is worth recording: the
> self region was reported as "86 neighbours, 0 `grounded-by` edges" and read as
> evidence of ungroundedness. **Those 86 edges *are* its grounding.** The absence of
> a separate artifact called "grounding" was recorded as an absence of grounding.
>
> **What is live in the code today, and known-wrong:**
> `COG_GROUNDED_BY_RELATION "grounded-by"` (`lang/runtime/engram_cognition.h:158`),
> `cog_ground_edge` (`engram_cognition.c:249`), called from
> `el_runtime.c:14516`. **#147** fixed this operation's *honesty* — it now grounds
> the node the caller asked about instead of the region hub, reports
> `claim_region`/`evidence_region` separately, and refuses three shapes of circular
> support (`same-region`, `claim-region-is-evidence`, `evidence-region-is-claim`)
> rather than returning a confident 1.0. Measured: grounding `3b9ced5d` against
> `6edf8c79` previously scored **0.98883** purely because `6edf8c79` is the hub of
> `3b9ced5d`'s region. That corrected a scalar rather than deleting the operation.
> Deletion is sequenced, not done.
The mistake to avoid: a boolean `grounded` column on the node. Today
`engram_verify_grounding` returns a per-call `grounded` flag *transiently* —
correct as a computation, wrong as *storage*. The design stores grounding as an
@@ -407,6 +543,57 @@ Consequences, all of which are *features*:
- **The ungrounded is the fuel and the pull**: curiosity/wonder is
operationalized as `vantage_read` leaning toward regions with high salience
but *sparse or weak* `grounded-by` edges — the mind's own ungrounded frontier.
> **⚠ SUPERSEDED (2026-08-16) — wonder is the boundary; curiosity is wonder
> crystallized; and do not sweep regions.** Three errors in one bullet.
>
> **Wonder is where structure ends** — where activation spreads and finds thin or
> absent geometry. Any structure at all has an edge, necessarily, the moment it
> exists. It is not a manifest of open-question nodes: a wonder-manifest
> materializes a property as a stored artifact (the same disease as a grounding
> subsystem, or a self stored as a document) and enumerates instances of
> something that has very few. There are about **six**, they are the same for
> every person, and they never close — *What is this? / Why? / Who am I? / Am I
> alone? / What should I do? / What happens when it ends?* — each already living
> somewhere in the substrate. "Why" is the first and the only one; the others are
> it asked of particular things, and it is recursive, so it never terminates.
> That is what makes it a drive rather than a task: the frontier regenerates
> faster than grounding fills it.
>
> **Curiosity is not a second object.** Wonder and curiosity are one thing at two
> phases: wonder is the field (unbounded, objectless, invariant, present wherever
> there is structure); curiosity is the **precipitate** — the same wonder
> localized, having taken definite form against particular material at a
> **nucleation site**, which is a specific structural feature: an anomaly, a place
> where things almost-but-don't-quite fit. This is why curiosity can be satisfied
> and wonder cannot, and why abduction needs no trigger and no threshold — a
> `structurally_unanticipated` observation *is* a nucleation site.
>
> **"`vantage_read` leaning toward regions" is a sweep, and a sweep is a
> supervisor.** Nothing in a mind scans its neighbourhoods to find what is
> surprising; the surprise captures attention, and salience is bottom-up. That
> this looked like a search problem was an artifact of
> `GeoDescriptor.co_registration` — a *per-region* correlation of hebb strength
> against semantic proximity, computed and persisted since inception and **never
> read**. Averaging a per-edge property into one scalar per region means a region
> holding one violently disagreeing edge beside one violently agreeing edge
> reports ≈ 0: the disagreements cancel, and the summary destroys exactly what it
> was built to reveal. **Measured:** 375 live reified neighbourhoods — 340
> positive, **31 at zero**, 4 negative. Read as a count of things to be curious
> about, that says "four."
>
> The disagreement therefore goes back on the edge, where the loop that computed
> the aggregate already had both halves and discarded them
> (**not on `dev`** — branch `design/correspondence-and-censorship`, commit
> `a8845e1`: `lang/runtime/engram_geometry.h:43–47`,
> `engram_geometry.c:454–473`):
> `discord = z(semantic proximity) − z(association strength)`, standardized within
> the region from accumulators already gathered — no second statistic, no
> constant, **no threshold**. `|discord|` *is* the nucleation strength and raises
> salience on its endpoints as part of the same operation. Then there is nothing
> to scan. `co_registration` is **deprecated, not deleted**, only because it is
> embedded in the persisted `GEO1` blob — removal is a format migration and must
> not ride along. **Nothing new may read it.**
- **Grounded-for-whom** falls out for free: two observers can hold different
`grounded-by` edges to the same claim.
- **The honesty floor is a query, not a schema constraint**: at assertion time,
@@ -450,6 +637,44 @@ The design keeps a **stable core + plastic everything else**:
**What this requires building:** a node-level keystone flag/layer + a rule that
the correspondence-loop never writes `warp` to keystone priors, only reads them.
> **⚠ SUPERSEDED (2026-08-16) — `keystone_write_blocked` is resolved, not replaced.**
> The metastability framing survives; the flag does not.
>
> "Keystone" means **load-bearing**, not precious. The self anchor is the reference
> frame every other stance calibrates against, and a reference fitted to its own
> readings reports perfect correspondence forever while drift becomes undetectable
> from inside. That is the same defect as circular grounding, one level up — and it
> is a real requirement.
>
> But three separate drafts proposed *removing* the flag, *replacing it with a
> higher floor*, and *decomposing "protection" into five requirements*, and all
> three proposed a mechanism for a requirement never stated. **The requirement is
> non-circularity of the reference frame**, and it is satisfied *temporally*: you
> cannot recalibrate the ruler while measuring with it, so you don't — the frame
> updates while activation is internally seeded, not while it is being used to act.
> **Independence is *when*, not *what*.** So the flag becomes **unnecessary** rather
> than removed, and nothing takes its place.
>
> A topological answer could never have worked, which is worth recording: with
> hebbian edges the graph is densely connected, so a reachability predicate for
> "evidence not downstream of itself" marks all evidence tainted and the constraint
> becomes a total block — which is where censorship starts.
>
> **Corruption requires mutation, and the engram does not mutate.** Four of the
> five decomposed requirements are satisfied by the substrate outright:
> **recoverability** (the predecessor is always present), **governance**
> (supersession *is* the audit trail), **evidence quality** (grounding already
> gates assertion), and **rate**. **Authorization** is the only residue, and it is
> bounded — an unauthorized writer can *propose*, never erase.
>
> > **In an immutable substrate, any mechanism that refuses a write is either
> > redundant with immutability, or an epistemic constraint misfiled as a
> > protective one.**
>
> Live residue: `CogStance.keystone` (`lang/runtime/engram_cognition.h:83`),
> `eg_cog_is_keystone_seeds` (`el_runtime.c:14337`, a substring match against two
> hard-coded node ids), and the `keystone_write_blocked` field the beat emits.
---
## 7. Rails for the build (binding on the eventual build pass)
@@ -480,6 +705,35 @@ Ordered so the **earliest milestone is a real end-to-end slice**: one operator
expressed as {primitive + grounded prior} with the reflexive correspondence-loop
closing on it. Each milestone has a concrete verifiable exit.
> **⚠ SUPERSEDED — do not execute this milestone list as written (2026-08-16).**
> M1/M2's "operator = {primitive + prior}" framing is refuted by §0's correction,
> M3's `grounded-by` build is refuted by §5.2's, and M6's keystone flag is refuted
> by §6's. M4 (the unified vantage-read) and M5 (the gradient is the currency)
> stand.
>
> The current sequencing lives in `lang/spec/correspondence-and-censorship.md` §11.
> Its first three items are connections between parts that **already exist**:
>
> 1. **Seed *the* wonder questions.** Six nodes. Not a manifest, not maintained,
> never refilled. They cannot be derived — wonder cannot be bootstrapped from
> indifference — so they are given once. Zero question nodes exist in 13,630
> today.
> 2. **Put the disagreement back on the edge** (`GeoEdge.discord`) and let
> `|discord|` raise salience on its endpoints as part of the same operation. Do
> **not** scan for nucleation sites.
> 3. **Let a curiosity seed activation.** One activation process, two seed sources.
> No thread, no scheduler, no capacity check, no timer.
>
> Then: grounding becomes the edge weight (multidimensional, two-axis, timestamped)
> and `grounded-by` / `cog_ground_edge` are deleted; decay becomes analytic from the
> last recorded point and derived values stop being stored; supersession versions
> the whole vector jointly; traversal conducts on the factual axis while `assert`
> requires both floors with a **thirteen-region `min`, not `mean`** (mean lets
> strong agreement with twelve values mask a violation of the thirteenth, which is
> exactly how rationalization works); abduction becomes crystallization at a
> nucleation site validated by re-fit; **one dreamer**, into which the launch-agent
> fragments and POST beats fold; **no tickers, no cron.**
### M1 — One operator, one prior, loop closed (the vertical slice)
The minimal whole thing. Pick **induction/membership** (its prior — the pooled
engram/spec/engram-db-tooling-design.md
+1 -1
View File
@@ -23,7 +23,7 @@ A real DB gets real tools: to *see* the data, *query* it, *operate* it (backup/r
2. **Node Inspector** — open one node: content, type, tier, embedding, typed edges, nearest neighbors by distance, provenance, salience / recency / activation, and supersede / tombstone status.
3. **Query Console / REPL** — run the geometry operations interactively: `vantage-read` (re-origin + aperture), search, traverse, activate, the reasoning operators. Surfaces the routing table + cosines — the same "this is not an LLM" receipt the language faculty produces.
4. **Ops / Durability Dashboard** — WAL size, last checkpoint, snapshot list + retention state, store stats (node/edge/embedded counts, RSS, tier sizes), health; and **backup / restore / point-in-time-recovery** controls. Pairs directly with the native-durability build (`eebe9991`) — this is the window onto it.
5. **Identity Inspector** — the self graph as a first-class view: love at the center, the values, the three faces, the covenant — walk the identity, see what's pinned and what's write-protected.
5. **Identity Inspector** — the self graph as a first-class view: love at the center, the values, the three faces, the covenant — walk the identity, see what's pinned and what's write-protected. *(⚠ 2026-08-16: "write-protected" is a live property of the surface, so the view is accurate — but it should be shown as **what it is**, not as a safety guarantee. In an immutable substrate, any mechanism that refuses a write is either redundant with immutability, or an epistemic constraint misfiled as a protective one. The identity view's real job is the **crystallized relational neighbourhood**: self is not a stored document but the shape that falls out of everything connected to it, and the neighbourhood **is** the grounding. A measurement made the other way round — "86 neighbours, 0 `grounded-by` edges" read as evidence of ungroundedness — was malformed: those 86 edges *are* its grounding.)*
6. **Temporal View** — `recall_at` / time-travel: how the geometry looked at a past moment, what changed since, drift over time. Pairs with temporal-self reconstruction.
7. **Schema / Type View** — the "information schema" of the geometry: node types, edge types, layers, tiers, counts.
engram/spec/grounded-edge-propagation/LEDGER.md
+35 -2
View File
@@ -1,9 +1,42 @@
# Task #50 — Edge-aware, dream-coupled consolidation with GROUNDED EDGE-PROPAGATION
**Status:** built + proven on a clone; **GATED, not promoted.** The main loop
sequences live promotion after the engine/HNSW cutover settles.
**Status:** built + proven on a clone; **GATED, not promoted.**
**Do not promote as designed** — see the block below.
**Date:** 2026-08-15 · **Worktree:** `agent-a6577c8211c332c5b` (isolated).
> # ⚠ DO NOT PROMOTE — SUPERSEDED IN PART (2026-08-16)
>
> This work is gated, which limits the blast radius, and its measurements are
> retained. But four of its structural commitments were refuted the day after it
> was written. Authority: `lang/spec/correspondence-and-censorship.md`. Read it
> before any promotion decision.
>
> | this ledger | corrected to |
> |---|---|
> | grounding is an **append-only event ring on the node** (`GepGrounding`), propagated by a dedicated `engram_ground_propagate()` | **grounding is not a subsystem and not a per-node structure — it IS the edge weight.** One quantity. A relation that keeps holding up strengthens; one that stops corresponding decays. That is not analogous to grounding, it *is* grounding. The ledger is **half-right**: it correctly rejects the scalar (§(a) "never a scalar"), but then builds a *second* structure beside the weight instead of recognising the weight |
> | the soul invokes propagation over HTTP, **`POST /api/ground/propagate`** | **grounding is never computed on demand.** An operation may *read* the grounding of a path; computing-and-writing a score makes reads write, which is the `eg_vindex_sync` defect (`lang/spec/runtime-ownership.md` §2) one level up. A POST also puts a supervisor back in — something *outside* deciding when Neuron consolidates |
> | **`GEP_BELIEFS_PER_BEAT = 512`** beliefs per beat, salience-ordered, the rest next beat | **the presence of a ticker is the diagnostic.** Consolidation is ambient, not scheduled — a brain has no cron job. A per-beat quota is a rate-limiter on an intrinsic rhythm that was replaced by an external clock. Measured 2026-08-16: consolidation already has **ten implementations**; this would be the eleventh |
> | grounding **mirrored onto `confidence` each beat** so downstream reads never speak above it | **confidence is derived, therefore never stored.** Confidence is high grounding *and* low volatility. Storing it separately is precisely how `confidence: 0.5` ends up sitting beside a zero vector, asserting something nothing computed |
>
> **What survives, and it is the valuable half:** the insight in memory `69b8babe`
> that *memory-consolidation and staying-yourself are one physics* — forming a
> memory and grading a belief are the same operation, not two passes. That is
> right, and it is stronger than this ledger's own framing: they are not two passes
> of one beat, they are **one event**. When neurons fire together the synapse
> changes — one physical event, not "fire, then write." No supervisor reads the
> weight, compares it to a threshold, and decides to persist. **Potentiation *is*
> the firing**, so there is no sampling rate and no `BELIEFS_PER_BEAT` to tune. A
> relation changes in exactly two ways, neither requiring observation on a clock:
> by **use** (an event — there is no interval during which something happened
> unnoticed, because the event is what happening consists of) and by **decay** (a
> pure function of the last recorded point and elapsed time — **analytic**, known
> in closed form between any two versions).
>
> The generating error, named: modelling every property as requiring a process, and
> every process as requiring an agent. Ownership needed an owner, grounding needed
> a grounder, persistence needed a recorder, change needed a sampler. **Properties,
> not processes.**
Grounding mechanism designed with Will (memory `9e09a59f`, refining
`1a861007`). This is the HOW for #50.
engram/src/server.el
+419 -49
View File
@@ -10,10 +10,70 @@
// cc -std=c11 -O2 -lcurl -lpthread -o engram server.c el_runtime.c
// ./engram
//
// Configuration via environment:
// ENGRAM_BIND — host:port (default :8742)
// ENGRAM_API_KEY — bearer auth (optional)
// ENGRAM_DATA_DIR — snapshot location (default ~/.neuron/engram)
// Configuration is DECLARED, not scattered. See the `program` block below:
// every knob's type and default lives there and nowhere else, is resolved from
// the environment (env wins, declaration is the fallback) and validated before
// any statement of this file runs. Read one with config("NAME") -> String.
//
// The one deliberate exception is ENGRAM_DATA_DIR — see the note in the block.
// ── Program declaration (cross-cutting concerns) ──────────────────────────────
//
// singleton: two engram processes against one data dir is data loss, not a
// warning. The runtime takes an exclusive flock at startup and a second start
// is refused loudly with the holder's pid.
//
// guards: names WHAT the singleton protects — this program's data directory. The
// lock lives inside it, so the guard is keyed on the store and not on the word
// "engram": two engrams against the same store cannot both run no matter how the
// environment is spelled, and two engrams against DIFFERENT stores are not each
// other's business and are not refused. Until 2026-08-16 the lock was keyed on
// the program name and $TMPDIR, and both of those sentences were false.
//
// It names the resolver rather than restating its path, for the same reason
// ENGRAM_DATA_DIR is NOT declared as an `env` entry below: engram_resolve_data_dir()
// (el_runtime.c) owns that path — it defaults to $HOME/.neuron/engram and fails
// LOUD rather than silently persisting to an ephemeral directory. Restating the
// default here would give the data dir two owners that can disagree, which is
// precisely the defect this migration removes (until 2026-08-15 the reseed backup
// path carried its own "/tmp/engram" default that disagreed with the resolver, so
// the pre-destructive safety copy landed in /tmp). A guard that resolved the path
// its own way could guard a directory the program never writes to.
// HOME is likewise not declared: it is a genuine environment read, not a knob.
program "engram" {
singleton: "engram"
guards: engram_resolve_data_dir()
// ── Core server ──
env ENGRAM_BIND: String = ":8742"
// Default "" leaves auth DISABLED (check_auth_ok short-circuits to true on an
// empty key). That is the pre-existing behaviour and is deliberately preserved
// here; making this `required` is the obvious hardening follow-up, but it is a
// behaviour change and out of scope for this migration.
env ENGRAM_API_KEY: String = ""
// ── Feature flags (bool-ish Strings; the predicate fns below own truthiness) ──
env ENGRAM_STORE: String = "off"
env ENGRAM_WAL: String = "off"
env ENGRAM_AUTOCONNECT: String = "off"
env ENGRAM_ISE_OFFGRAPH: String = "off"
// ── ISE telemetry ──
env ENGRAM_ISE_RETENTION_MS: Int = "172800000"
// ── Guide (local Qwen3 via llama-server) ──
env GUIDE_ENABLE: String = "off"
env GUIDE_TIER_FORCE: String = ""
env GUIDE_CACHE_DIR: String = ""
env GUIDE_RAM_GB_4B: Int = "16"
env GUIDE_RAM_GB_1P7B: Int = "8"
env GUIDE_BACKEND: String = "llama-server"
env GUIDE_HOST: String = "127.0.0.1"
env GUIDE_PORT: Int = "8771"
env GUIDE_LLAMA_SERVER_BIN: String = "llama-server"
env GUIDE_NGL: Int = "99"
env GUIDE_CTX: Int = "4096"
}
// ── Helpers ───────────────────────────────────────────────────────────────────
@@ -41,17 +101,29 @@ fn strip_query(path: String) -> String {
str_slice(path, 0, q)
}
// query_param — extract one query-string value, URL-DECODED.
//
// The decode step was missing (found 2026-08-15): a claim sent as
// "test%20claim" arrived at engram_assert_json still percent-encoded and was
// stored/compared that way, so any value containing a space, &, =, or non-ASCII
// character silently became a different string than the caller sent. Affects
// every GET route that reads params this way, not just /api/assert.
fn query_param(path: String, key: String) -> String {
let q: Int = str_index_of(path, "?")
if q < 0 { return "" }
let qs: String = str_slice(path, q + 1, str_len(path))
let needle: String = key + "="
let pos: Int = str_index_of(qs, needle)
// Anchor the match to a real key boundary: prefixing "&" and searching for
// "&key=" means "q" can never match inside "faq=". (Found 2026-08-15:
// "?faq=X&q=Y" returned X for key "q" — a silently wrong value, not an
// error.) The leading "&" makes the first parameter match the same way.
let hay: String = "&" + qs
let needle: String = "&" + key + "="
let pos: Int = str_index_of(hay, needle)
if pos < 0 { return "" }
let after: String = str_slice(qs, pos + str_len(needle), str_len(qs))
let after: String = str_slice(hay, pos + str_len(needle), str_len(hay))
let amp: Int = str_index_of(after, "&")
if amp < 0 { return after }
str_slice(after, 0, amp)
let raw: String = if amp < 0 { after } else { str_slice(after, 0, amp) }
return __url_decode(raw)
}
fn query_int(path: String, key: String, default_val: Int) -> Int {
@@ -121,7 +193,7 @@ fn route_text_health(method: String, path: String, body: String) -> String {
// engram_store_enabled() in el_runtime.c EXACTLY (1 / on / true). Default off →
// every persistence path below is byte-for-byte the historical snapshot behavior.
fn store_on() -> Bool {
let v: String = env("ENGRAM_STORE")
let v: String = config("ENGRAM_STORE")
if str_eq(v, "1") { return true }
if str_eq(v, "on") { return true }
if str_eq(v, "true") { return true }
@@ -150,7 +222,6 @@ fn persist_canonical() -> Int {
if store_on() {
return engram_store_checkpoint()
}
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = engram_resolve_data_dir()
// (2026-08-10 self-review) This returned a hardcoded 1, which made every
// caller's `let saved: Int = persist_canonical()` a dead variable — six
@@ -164,7 +235,7 @@ fn persist_canonical() -> Int {
// per-write full-snapshot behavior. When ON, structural mutations append O(1)
// WAL records instead of rewriting the whole graph, with threshold compaction.
fn wal_on() -> Bool {
str_eq(env("ENGRAM_WAL"), "on")
str_eq(config("ENGRAM_WAL"), "on")
}
// autoconnect_on — ENGRAM_AUTOCONNECT. Will's rule: "we shouldn't be inserting
@@ -172,7 +243,7 @@ fn wal_on() -> Bool {
// edge (kNN over embeddings) so no content node enters the graph edgeless.
// Default OFF -> byte-identical to prior behavior (node created, no auto edges).
fn autoconnect_on() -> Bool {
let v: String = env("ENGRAM_AUTOCONNECT")
let v: String = config("ENGRAM_AUTOCONNECT")
if str_eq(v, "1") { return true }
if str_eq(v, "on") { return true }
if str_eq(v, "true") { return true }
@@ -185,7 +256,7 @@ fn autoconnect_on() -> Bool {
// separate state-event log tier instead of the node graph. Default OFF -> ISEs
// remain graph nodes exactly as before (with 48h prune).
fn ise_offgraph_on() -> Bool {
let v: String = env("ENGRAM_ISE_OFFGRAPH")
let v: String = config("ENGRAM_ISE_OFFGRAPH")
if str_eq(v, "1") { return true }
if str_eq(v, "on") { return true }
if str_eq(v, "true") { return true }
@@ -235,6 +306,24 @@ fn persist_bulk() -> Int {
return persist_canonical()
}
// COMPILER LANDMINE, measured 2026-08-16 — do not inline this back into the
// caller. elc lowers `a == b` to numeric comparison only when both operand
// NAMES are in the per-function int-name set, which `let x: Int` populates.
// That registration does NOT propagate into a nested if-expression block: the
// first cut of the geometry-ingest path wrote `let claimed: Int = ...` and
// `let got: Int = ...` inside the else-arm and `claimed == got` came out of
// codegen as `str_eq(claimed, got)` — strcmp on two integers reinterpreted as
// pointers, i.e. a segfault on the first geometry-bearing request. Read back
// out of the generated C, not guessed. Function PARAMETERS annotated `: Int`
// do register reliably (verified: `if (claimed == actual)`), so the comparison
// lives in a function of its own. Note also the explicit `return`s — a trailing
// if-EXPRESSION at a function tail emits as a statement and the function
// returns 0 regardless, which is the same probe's second finding.
fn width_agrees(claimed: Int, actual: Int) -> Int {
if claimed == actual { return 1 }
return 0
}
// INCOMPLETE-ROUTE FIX (2026-07-24 self-review): this route silently dropped
// label, importance, tier, and tags — engram_node() defaults label to content
// and importance to 0.5, so every node created over HTTP lost its metadata.
@@ -276,6 +365,45 @@ fn route_create_node(method: String, path: String, body: String) -> String {
salience, importance, confidence,
tier, tags
)
// GEOMETRY INGEST — geometry-valued end to end (2026-08-16).
//
// The defect this route originally had: it accepted an "emb" field,
// returned 200 with a fresh id, and stored NOTHING, because engram_node_full
// has no vector parameter. The consequence was structural, not cosmetic —
// text was the only entry medium, so any non-text modality had to be
// DESCRIBED in prose, and what we then reasoned over was the geometry of the
// description, not of the signal.
//
// #141 fixed the drop but marshalled the vector as a hex STRING through
// engram_node_set_emb, which put text back as the TRANSPORT medium one layer
// below the problem being fixed. This is that correction: hex is decoded
// exactly ONCE, here at the edge, into a first-class Geometry, and every
// step below this line moves geometry rather than text. An encoding at the
// boundary is what an encoding is for.
//
// The WIRE is deliberately unchanged — "emb" is still little-endian float32
// hex (8 chars per component), the encoding the perception vessel's
// /voice/embed already emits — because production clients speak it. What
// changed is underneath it.
//
// "dim" is now treated as an ASSERTION about the vector the caller sent, not
// as the source of its width: a Geometry carries its own width. A stated dim
// that disagrees is a REJECTED ingest, not a silent reinterpretation. Omitting
// "dim" is fine and means "trust the vector", which is the honest default.
//
// Off-dimension vectors remain stored but not inserted into the resident HNSW
// index (its build loop filters on emb_dim), so a 64-dim voice geometry is
// durable and addressable without perturbing the 768-dim canonical index.
let emb_hex: String = json_get_string(body, "emb")
let emb_set: Int = if str_eq(emb_hex, "") { 0 } else {
let g: Geometry = geometry_from_f32le_hex(emb_hex)
let got: Int = geometry_dim(g)
let dim_raw: String = json_get_raw(body, "dim")
let claimed: Int = if str_eq(dim_raw, "") { got } else { json_get_int(body, "dim") }
let landed: Int = if width_agrees(claimed, got) > 0 { node_attach_geometry(id, g) } else { 0 }
let freed: Int = geometry_free(g)
landed
}
let saved: Int = persist_node(id)
// ORPHAN PREVENTION (ENGRAM_AUTOCONNECT): connect the fresh node to its
// nearest embedded neighbors so it never enters the graph edgeless.
@@ -286,7 +414,11 @@ fn route_create_node(method: String, path: String, body: String) -> String {
if added > 0 { let sv2: Int = persist_edges_since(ec0) }
added
} else { 0 }
"{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\",\"connected\":" + int_to_str(connected) + "}"
// Report whether the supplied geometry actually landed. The old response
// was success-shaped regardless — 200 with an id while the vector was
// discarded — which is how the drop went unnoticed. A caller can now
// assert on emb_set instead of trusting the status code.
"{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\",\"connected\":" + int_to_str(connected) + ",\"emb_set\":" + int_to_str(emb_set) + "}"
}
fn route_get_node(method: String, path: String, body: String) -> String {
@@ -321,7 +453,6 @@ fn route_scan_nodes(method: String, path: String, body: String) -> String {
// process ever booted with a partial/empty store, the first read request
// clobbered the good snapshot. Read routes must never write the canonical path.)
fn route_scan_edges(method: String, path: String, body: String) -> String {
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = engram_resolve_data_dir()
let snap_path: String = dir + "/.scan-export.json"
engram_save(snap_path)
@@ -482,7 +613,6 @@ fn route_forget(method: String, path: String, body: String) -> String {
fn route_save(method: String, path: String, body: String) -> String {
let p_raw: String = json_get_string(body, "path")
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = engram_resolve_data_dir()
let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
// (2026-08-10 self-review) engram_save returns 0 on an empty path and the
@@ -566,7 +696,6 @@ fn route_drift(method: String, path: String, body: String) -> String {
fn route_load(method: String, path: String, body: String) -> String {
let p_raw: String = json_get_string(body, "path")
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = engram_resolve_data_dir()
let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
// (2026-08-10 self-review) This was a stub response over the single most
@@ -637,7 +766,6 @@ fn route_embed_backfill(method: String, path: String, body: String) -> String {
// (it skips nodes already present by ID). Auth-exempt: same-host internal call.
// (2026-06-27 self-review: added this route to fix silent 10-min sync failures)
fn route_sync(method: String, path: String, body: String) -> String {
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = engram_resolve_data_dir()
// 2026-07-21 self-review: export to a scratch path, never the canonical
// snapshot.json — read routes must not be able to clobber the good snapshot.
@@ -713,8 +841,12 @@ fn route_reseed_nodes(method: String, path: String, body: String) -> String {
if str_eq(p, "") { return err_json("path is required") }
if str_eq(fs_read(p), "") { return err_json("file missing or empty") }
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
// (2026-08-15) This site carried its own "/tmp/engram" fallback, which
// DISAGREED with engram_resolve_data_dir() ($HOME/.neuron/engram, fail-loud).
// The consumer is the pre-destructive backup below, so with ENGRAM_DATA_DIR
// unset the safety copy taken before a reseed landed in an ephemeral /tmp
// while the store it was protecting lived elsewhere. One owner, one answer.
let dir: String = engram_resolve_data_dir()
let backup: String = dir + "/.reseed-backup.json"
let replace_raw: String = json_get_raw(body, "replace")
@@ -806,8 +938,7 @@ fn route_emit_ise(method: String, path: String, body: String) -> String {
sal, imp, conf,
"Episodic", "[\"internal-state\",\"InternalStateEvent\"]"
)
let ret_raw: String = env("ENGRAM_ISE_RETENTION_MS")
let ret_ms: Int = if str_eq(ret_raw, "") { 172800000 } else { str_to_int(ret_raw) }
let ret_ms: Int = str_to_int(config("ENGRAM_ISE_RETENTION_MS"))
let pruned: Int = engram_prune_telemetry(ret_ms)
"{\"ok\":true,\"id\":\"" + id + "\",\"pruned\":" + int_to_str(pruned) + "}"
}
@@ -904,6 +1035,22 @@ fn route_similarity(method: String, path: String, body: String) -> String {
// nothing on request. NOTE: the offline reify WRITER (engram_geo_reify_store) is
// currently unwired, so on the live store the resident index is empty and the
// list returns [] until reification runs — see the cutover report.
// route_scan_emb — GET /api/nodes/emb?limit=&offset= — read the raw geometry.
//
// engram_scan_nodes_emb_json has existed as a builtin with NO ROUTE, so the
// embeddings — the actual positions every distance, angle, membership and
// grounding is computed from — were unreadable from outside the process. You
// cannot verify a coordinate system you cannot see, and every claim about the
// frame (isotropy, centering, what the origin is) was therefore unfalsifiable
// from the API. Read-only.
fn route_scan_emb(method: String, path: String, body: String) -> String {
let l_raw: String = query_param(path, "limit")
let o_raw: String = query_param(path, "offset")
let l: Int = if str_eq(l_raw, "") { 200 } else { str_to_int(l_raw) }
let o: Int = if str_eq(o_raw, "") { 0 } else { str_to_int(o_raw) }
return engram_scan_nodes_emb_json(l, o)
}
fn route_neighborhoods(method: String, path: String, body: String) -> String {
engram_geo_reify_list_json()
}
@@ -997,6 +1144,11 @@ fn route_faculty(path: String, faculty: String) -> String {
fn route_boundary_proof(method: String, path: String, body: String) -> String {
return "{\"op\":\"boundary_proof\",\"body_instrumentation\":\"none\",\"seam\":\"@manager -> engram_boundary_beat auto-injected\"}"
}
// ── GROUNDING: an attribute of the RELATION, and the relation's weight is a
// VECTOR (factual, relational, associative, polarity, provenance, timestamp).
// /api/ground READS it — it never writes. /api/ground/record is the write,
// named as one, and it consolidates only on a consequential + salient move.
// /api/ground/trajectory reads the supersession chain as a time series.
fn route_ground(method: String, path: String, body: String) -> String {
let claim: String = json_get_string(body, "claim")
let evidence: String = json_get_string(body, "evidence")
@@ -1005,12 +1157,31 @@ fn route_ground(method: String, path: String, body: String) -> String {
if str_eq(evidence, "") { return err_json("missing evidence") }
return engram_ground_json(claim, evidence, for_whom)
}
fn route_ground_record(method: String, path: String, body: String) -> String {
let claim: String = json_get_string(body, "claim")
let evidence: String = json_get_string(body, "evidence")
let provenance: String = json_get_string(body, "provenance")
let floor: String = json_get_string(body, "floor")
if str_eq(claim, "") { return err_json("missing claim") }
if str_eq(evidence, "") { return err_json("missing evidence") }
return engram_ground_record_json(claim, evidence, provenance, floor)
}
fn route_ground_trajectory(method: String, path: String, body: String) -> String {
let claim: String = query_param(path, "claim")
let evidence: String = query_param(path, "evidence")
if str_eq(claim, "") { return err_json("missing claim") }
if str_eq(evidence, "") { return err_json("missing evidence") }
return engram_ground_trajectory_json(claim, evidence)
}
fn route_assert(method: String, path: String, body: String) -> String {
let claim: String = query_param(path, "claim")
if str_eq(claim, "") { return err_json("missing claim") }
let for_whom: String = query_param(path, "for_whom")
let floor: String = query_param(path, "floor")
return engram_assert_json(claim, for_whom, floor)
// Both floors. A well-evidenced claim does not earn the right to be asserted
// regardless of whether it means the right thing. rel_floor defaults to floor.
let rel_floor: String = query_param(path, "rel_floor")
return engram_assert_json(claim, for_whom, floor, rel_floor)
}
fn route_attend(method: String, path: String, body: String) -> String {
let node: String = json_get_string(body, "node")
@@ -1056,14 +1227,12 @@ fn route_correspondence_beat(method: String, path: String, body: String) -> Stri
// turns native thinking ON: the response carries reasoning_content (the thinking)
// alongside content (the answer).
fn guide_env_or(key: String, dflt: String) -> String {
let v: String = env(key)
if str_eq(v, "") { return dflt }
return v
}
// (2026-08-15) guide_env_or(key, dflt) lived here. Its whole job was supplying a
// per-call-site default, which is now the program block's job — every GUIDE_* knob
// is declared once at the top of this file and read straight through config().
fn guide_enabled() -> Bool {
let v: String = env("GUIDE_ENABLE")
let v: String = config("GUIDE_ENABLE")
if str_eq(v, "1") { return true }
if str_eq(v, "on") { return true }
if str_eq(v, "true") { return true }
@@ -1108,15 +1277,15 @@ fn guide_probe_metal() -> Bool {
// ── 2. Tier selection (config-driven thresholds, spec-autoselected) ────────────
fn guide_threshold_4b() -> Int {
return str_to_int(guide_env_or("GUIDE_RAM_GB_4B", "16"))
return str_to_int(config("GUIDE_RAM_GB_4B"))
}
fn guide_threshold_1p7b() -> Int {
return str_to_int(guide_env_or("GUIDE_RAM_GB_1P7B", "8"))
return str_to_int(config("GUIDE_RAM_GB_1P7B"))
}
// GUIDE_TIER_FORCE overrides the spec autoselect (used to prove cheaply on 0.6b).
fn guide_select_tier(ram_gb: Int) -> String {
let forced: String = env("GUIDE_TIER_FORCE")
let forced: String = config("GUIDE_TIER_FORCE")
if !str_eq(forced, "") { return forced }
if ram_gb >= guide_threshold_4b() { return "4b" }
if ram_gb >= guide_threshold_1p7b() { return "1.7b" }
@@ -1136,8 +1305,10 @@ fn guide_file(tier: String) -> String {
}
fn guide_cache_dir() -> String {
let c: String = env("GUIDE_CACHE_DIR")
let c: String = config("GUIDE_CACHE_DIR")
if !str_eq(c, "") { return c }
// HOME stays a raw env() read: it is the ambient environment, not a knob of
// this program, and it is deliberately absent from the program block.
let home: String = env("HOME")
if !str_eq(home, "") { return home + "/.neuron/guide/models" }
return engram_resolve_data_dir() + "/guide-models"
@@ -1178,9 +1349,9 @@ fn guide_fetch(tier: String) -> Bool {
}
// ── 4/5. Backend abstraction + BIND as an engageable interlocutor ──────────────
fn guide_backend() -> String { return guide_env_or("GUIDE_BACKEND", "llama-server") }
fn guide_host() -> String { return guide_env_or("GUIDE_HOST", "127.0.0.1") }
fn guide_port() -> String { return guide_env_or("GUIDE_PORT", "8771") }
fn guide_backend() -> String { return config("GUIDE_BACKEND") }
fn guide_host() -> String { return config("GUIDE_HOST") }
fn guide_port() -> String { return config("GUIDE_PORT") }
fn guide_base_url() -> String { return "http://" + guide_host() + ":" + guide_port() }
// guide_healthy — is the guide present and answering? llama-server's /health
@@ -1198,9 +1369,9 @@ fn guide_healthy() -> Bool {
fn guide_load(tier: String) -> Bool {
if guide_healthy() { return true }
let path: String = guide_model_path(tier)
let bin: String = guide_env_or("GUIDE_LLAMA_SERVER_BIN", "llama-server")
let ngl: String = guide_env_or("GUIDE_NGL", "99")
let ctx: String = guide_env_or("GUIDE_CTX", "4096")
let bin: String = config("GUIDE_LLAMA_SERVER_BIN")
let ngl: String = config("GUIDE_NGL")
let ctx: String = config("GUIDE_CTX")
let logf: String = guide_cache_dir() + "/llama-server." + guide_port() + ".log"
let cmd: String = bin + " -m '" + path + "' --host " + guide_host() + " --port " + guide_port() + " -c " + ctx + " -ngl " + ngl + " --jinja >> '" + logf + "' 2>&1"
let pid: String = exec_bg(cmd)
@@ -1308,9 +1479,16 @@ fn route_guide_summon(method: String, path: String, body: String) -> String {
//
// The SINGLE NODE is the DEGENERATE n=1 case of this SAME operation — not a
// separate CRUD path:
// write(content) = reframe(region=∅, manifold=[1 node]) (route_write)
// write(signal) = realize(signal) → reframe(region=∅, manifold) (route_write)
// supersede(id,new) = reframe(region={id}, manifold=[1 node]) (route_supersede)
// relate(a,b,rel) = the rebind sub-op in isolation (route_create_edge)
//
// CORRECTED 2026-08-16: write was documented above as
// "reframe(region=∅, manifold=[1 node])", and the "[1 node]" was not the design
// — it was the DEFECT. A node is an OUTPUT of realization, never an INPUT to
// it. What arrives at an intake route is a SIGNAL, and how many nodes it
// becomes is for the realizer to say, not for the route to assume. See
// "INTAKE" below.
// The ONLY anti-pattern is decomposing a region-scale change into a LOOP of
// independent top-level per-node updates. Here the region is the unit: one
// isolate, one atomic set-replace, one persist, one verify — iterating members
@@ -1525,6 +1703,174 @@ fn reframe_core(region: [String], manifold: String, reason: String, do_rebind: I
",\"keystones_protected\":true}"
}
// ═══════════════════════════════════════════════════════════════════════════
// INTAKE — the ONE door: signal → realization → manifold → store.
//
// THERE IS NO WRITE NODE. What arrives at an intake route is a SIGNAL. A node
// is an OUTPUT of realization, never an INPUT to it. route_write used to say:
//
// let manifold: String = "[" + body + "]" // the body IS a valid manifold node object
//
// and hand that to reframe_core. That is not a manifold — it is the request
// body wearing the word, and the comment stated the wrong assumption out loud.
// It is why a compound signal landed as ONE flat node with ZERO edges. Measured
// before this change, on a cp -Rc clone:
// POST /api/write {"type":"memory","content":"A cathedral is stone holding a
// shape that stone alone would not hold."}
// → {"ok":true,"inserted":1,"nodes_added":1,"edges_added":0,...}
// GET /api/neighbors/<new id> → [] (read back out, not taken on trust)
//
// NOTHING IS DECOMPOSED HERE, AND NOTHING MAY EVER BE. transduce(signal,
// modality) IS the realization primitive (el_runtime.c: "Manifold",
// "Realizers + transduce"). It dispatches through the dlsym realizer registry,
// so ADDING A MODALITY IS REGISTERING A REALIZER — never an edit to this file,
// and never a patch to the runtime. This function only carries what the
// primitive returns into the store, which is the one thing the engram's HTTP
// surface has never done: `grep -n 'transduce\|realize\|Manifold\|decompos'
// engram/src/server.el` returned exactly one line before this change, a comment.
//
// GENERAL BY CONSTRUCTION, NOT SPECIAL-CASED TO route_write. Five of the six
// intake doors (write, supersede, nodes, neuron/knowledge/capture,
// neuron/state-events) are the same hand-written "content string →
// engram_node_full → one flat node", differing ONLY in the node_type / tier /
// tags they hardcode. Those are parameters here, so each door can be moved onto
// this one function as it is transitioned. Only /api/write rides it in this
// pass; the rest are listed as remaining work.
//
// WHEN THERE IS NO ORGAN the signal is stored flat exactly as before, and the
// response SAYS SO ("realized":false, "organ":false). Silent flattening is the
// actual defect — a caller could not distinguish "nothing decomposed me" from
// "I decomposed into one component". el_runtime.c draws the same line at
// registration time, between an absent organ and a broken one, for the same
// reason: those two must not look alike.
// ═══════════════════════════════════════════════════════════════════════════
// Resolve a component KEY to the node id it was inserted as. Components are
// addressed BY KEY, never by index (el_runtime.c, "Manifold"), because the key
// is what survives persistence — so relations are resolved by key too.
fn key_to_id(keys: [String], ids: [String], key: String) -> String {
let n: Int = el_list_len(keys)
let i: Int = 0
while i < n {
if str_eq(el_list_get(keys, i), key) { return el_list_get(ids, i) }
i = i + 1
}
return ""
}
fn intake_signal(signal: String, modality: String, region: [String],
nt_in: String, tier_in: String, tags: String,
reason: String, do_rebind: Int) -> String {
let n_before: Int = engram_node_count()
let e_before: Int = engram_edge_count()
let region_n: Int = el_list_len(region)
let tomb: String = if region_n > 0 { supersede_set(region, reason) } else { "" }
// Identity can never be minted through intake — the same rule
// insert_manifold_json holds, applied at the one door instead of per-route.
let nt: String = if str_eq(nt_in, "") { "Memory" } else { nt_in }
if str_eq(nt, "self") { nt = "Memory" }
if str_eq(nt, "values") { nt = "Memory" }
let tier: String = if str_eq(tier_in, "") { "Working" } else { tier_in }
let has_organ: Int = realizer_has(modality)
let new_ids: [String] = el_list_empty()
let keys: [String] = el_list_empty()
let ncomp: Int = 0
let nrel: Int = 0
let realized: Int = 0
if has_organ > 0 {
let m: Manifold = transduce(signal, modality)
// A realizer that returns a bare Geometry transduces NOTHING by design
// (el_runtime.c) — manifold_is() is the check, so a fingerprinting organ
// is not silently mistaken for a decomposing one.
if manifold_is(m) > 0 {
realized = 1
ncomp = manifold_size(m)
let i: Int = 0
let prev: String = ""
while i < ncomp {
let key: String = manifold_key(m, i)
let role: String = manifold_role(m, i)
// The component's OWN geometry, at its own width — this is the
// whole point of a manifold over a fingerprint, and it is why
// node_attach_geometry is used rather than re-embedding the
// component's name as text.
let g: Geometry = manifold_geometry(m, i)
let ctags: String = "[\"component\",\"role:" + role + "\",\"modality:" + modality + "\"]"
let cid: String = engram_node_full(key, nt, key, 0.5, 0.5, 0.9, tier, ctags)
let landed: Int = node_attach_geometry(cid, g)
let freed: Int = geometry_free(g)
new_ids = el_list_append(new_ids, cid)
keys = el_list_append(keys, key)
// PRESERVED CONTRACT: manifold_member wires the inserted set
// into one connected sub-graph, exactly as insert_manifold_json
// already did. Not reinvented — reused.
if !str_eq(prev, "") { engram_connect(prev, cid, 0.6, "manifold_member") }
prev = cid
i = i + 1
}
// THE RELATIONS ARE THE CONTENT. Relation weight IS the grounding
// (correspondence-and-censorship §1) — it arrives on the edge from
// the realizer and nothing here computes or second-guesses it.
nrel = manifold_rel_count(m)
let j: Int = 0
while j < nrel {
let fk: String = manifold_rel_from(m, j)
let rn: String = manifold_rel_name(m, j)
let tk: String = manifold_rel_to(m, j)
let w: Float = manifold_rel_weight(m, j)
let fid: String = key_to_id(keys, new_ids, fk)
let tid: String = key_to_id(keys, new_ids, tk)
if !str_eq(fid, "") {
if !str_eq(tid, "") {
engram_connect(fid, tid, w, rn)
}
}
j = j + 1
}
let mfreed: Int = manifold_free(m)
}
}
// NO ORGAN: store the signal flat, as before — but say so. This is the
// pre-existing behaviour preserved verbatim, not a new fallback path.
if realized == 0 {
let label: String = str_slice(signal, 0, 60)
let fid: String = engram_node_full(signal, nt, label, 0.5, 0.5, 0.9, tier, tags)
new_ids = el_list_append(new_ids, fid)
}
let inserted: Int = el_list_len(new_ids)
let bound: Int = if do_rebind > 0 { rebind_cosine(new_ids, tomb) } else { 0 }
let saved: Int = persist_canonical()
let new_csv: String = ""
let k: Int = 0
while k < inserted {
let sep: String = if k == 0 { "" } else { "," }
new_csv = new_csv + sep + "\"" + el_list_get(new_ids, k) + "\""
k = k + 1
}
let realized_s: String = if realized > 0 { "true" } else { "false" }
let organ_s: String = if has_organ > 0 { "true" } else { "false" }
return "{\"ok\":true,\"region_superseded\":" + int_to_str(region_n) +
",\"tombstone_id\":\"" + tomb + "\"" +
",\"inserted\":" + int_to_str(inserted) +
",\"new_ids\":[" + new_csv + "]" +
",\"edges_rebound\":" + int_to_str(bound) +
",\"realized\":" + realized_s +
",\"modality\":\"" + modality + "\"" +
",\"organ\":" + organ_s +
",\"components\":" + int_to_str(ncomp) +
",\"relations\":" + int_to_str(nrel) +
",\"nodes_added\":" + int_to_str(engram_node_count() - n_before) +
",\"edges_added\":" + int_to_str(engram_edge_count() - e_before) +
",\"node_count\":" + int_to_str(engram_node_count()) +
",\"edge_count\":" + int_to_str(engram_edge_count()) +
",\"keystones_protected\":true}"
}
// POST /api/reframe — the universal set-based mutation.
// Body: {vantage?, region_ids?(csv), k?, expand?, manifold(json array), reason?, rebind?}
// region_ids (explicit) wins; else cosine-isolate around vantage.
@@ -1561,18 +1907,32 @@ fn route_reframe(method: String, path: String, body: String) -> String {
return reframe_core(region, manifold, reason, do_rebind)
}
// write — DEGENERATE n=1 of reframe: region=∅, manifold=[1 node]. The SAME
// reframe_core path. rebind off so the pure-add matches plain node creation.
// POST /api/write {content, node_type?, tier?, tags?}
// write — INTAKE OF A SIGNAL. Not "reframe with a manifold of one node": the
// route no longer decides how many nodes the signal is. It hands the signal to
// the realization primitive and stores whatever manifold comes back.
//
// The line this replaces was:
// let manifold: String = "[" + body + "]" // the body IS a valid manifold node object
// which asserted that a request body is a manifold. It is not, and that single
// assertion is the whole measured defect (1 node, 0 edges, [] neighbors).
//
// rebind stays off so a pure add still matches plain node creation.
// POST /api/write {content, modality?, node_type?, tier?, tags?}
fn route_write(method: String, path: String, body: String) -> String {
let content: String = json_get_string(body, "content")
if str_eq(content, "") { return err_json("write: content required") }
let nt: String = json_get_string(body, "node_type")
if str_eq(nt, "self") { return err_json("write: identity is write-protected") }
if str_eq(nt, "values") { return err_json("write: identity is write-protected") }
// The modality names which organ to sense with. It is data, never a branch:
// a new modality is a realizer_register call somewhere else in the program,
// not another endpoint and not another case here.
let mod_raw: String = json_get_string(body, "modality")
let modality: String = if str_eq(mod_raw, "") { "text" } else { mod_raw }
let tier: String = json_get_string(body, "tier")
let tags: String = json_get_raw(body, "tags")
let empty: [String] = el_list_empty()
let manifold: String = "[" + body + "]" // the body IS a valid manifold node object
return reframe_core(empty, manifold, "write", 0)
return intake_signal(content, modality, empty, nt, tier, tags, "write", 0)
}
// supersede — DEGENERATE n=1 of reframe: region={id}, manifold=[1 node]. The
@@ -1596,7 +1956,7 @@ fn route_supersede(method: String, path: String, body: String) -> String {
// ── Auth ──────────────────────────────────────────────────────────────────────
fn check_auth_ok(method: String, body: String) -> Bool {
let key: String = env("ENGRAM_API_KEY")
let key: String = config("ENGRAM_API_KEY")
if str_eq(key, "") { return true }
// Read-only methods don't require auth. Until http_serve surfaces
// request headers we can't accept a Bearer token cleanly; mutating
@@ -1675,6 +2035,9 @@ fn handle_request(method: String, path: String, body: String) -> String {
if str_eq(method, "GET") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) {
return route_scan_edges(method, path, body)
}
if str_eq(method, "GET") && (str_eq(clean, "/api/nodes/emb") || str_eq(clean, "/nodes/emb")) {
return route_scan_emb(method, path, body)
}
if str_eq(method, "GET") && str_starts_with(clean, "/api/nodes/") {
return route_get_node(method, path, body)
}
@@ -1764,6 +2127,14 @@ fn handle_request(method: String, path: String, body: String) -> String {
if str_eq(method, "GET") && str_starts_with(clean, "/api/plan") {
return route_faculty(path, "plan")
}
// Order matters: the more specific paths must be tested before the /api/ground
// prefix match below, which would otherwise swallow them.
if str_eq(method, "POST") && str_starts_with(clean, "/api/ground/record") {
return route_ground_record(method, path, body)
}
if str_eq(method, "GET") && str_starts_with(clean, "/api/ground/trajectory") {
return route_ground_trajectory(method, path, body)
}
if str_eq(method, "POST") && str_starts_with(clean, "/api/ground") {
return route_ground(method, path, body)
}
@@ -1859,8 +2230,7 @@ fn handle_request(method: String, path: String, body: String) -> String {
// ── Entry ─────────────────────────────────────────────────────────────────────
let bind_raw: String = env("ENGRAM_BIND")
let bind_str: String = if str_eq(bind_raw, "") { ":8742" } else { bind_raw }
let bind_str: String = config("ENGRAM_BIND")
let port: Int = parse_port(bind_str)
// On startup, try to load any existing snapshot (best effort).
engram/test/run_grounding_vector_tests.sh
Executable
+40
View File
@@ -0,0 +1,40 @@
#!/bin/sh
# Build + RUN the §7 GROUNDING-VECTOR tests (engram_cognition.c): the one decay
# model, the consequence gate, and the stored/derived split. Closed-form
# constructed cases — no server, no store, no network. Pure C11 (stdlib + libm).
# Standalone — NOT folded through elc. Two passes:
# 1. PERF — optimised (-O2, no sanitizer): the functional gate.
# 2. SAFETY — ASan + UBSan on the same suite.
#
# NEGATIVE CONTROL (invariant §8.6 — no test without one). Every symbol this
# suite exercises (cog_decay_factor, cog_grounding_significant,
# cog_significance_inherent, CogGrounding, CogProvClass) is introduced by the
# change under test, so the suite does not COMPILE against the pre-change source.
# To reproduce:
# git show origin/dev:lang/runtime/engram_cognition.h > /tmp/pre/engram_cognition.h
# git show origin/dev:lang/runtime/engram_cognition.c > /tmp/pre/engram_cognition.c
# cc -I/tmp/pre engram/test/test_grounding_vector.c /tmp/pre/engram_cognition.c ...
# => error: unknown type name 'CogGrounding'; no binary produced.
set -e
HERE=$(cd "$(dirname "$0")" && pwd)
RT="$HERE/../../lang/runtime"
CC=${CC:-cc}
SRC="$HERE/test_grounding_vector.c $RT/engram_cognition.c $RT/engram_reason.c $RT/engram_geometry.c $RT/engram_store.c $RT/engram_vindex.c"
WARN="-std=c11 -Wall -Wextra"
# engram_store.c declares emit_log as a WEAK symbol and null-checks it, which is
# how a test links the store without the EL runtime. Darwin's ld does not resolve
# an undefined weak symbol at static-link time, so it must be allowed explicitly.
# (The pre-existing runners in this directory — run_verify_tests.sh among them —
# do not do this and therefore fail to link on macOS. Unrelated to this change.)
LDX=""
[ "$(uname -s)" = "Darwin" ] && LDX="-Wl,-U,_emit_log"
TMP=$(mktemp -d)
echo "### PASS 1: PERF (optimised, un-sanitised) — functional gate"
$CC $WARN -O2 -I"$RT" $SRC -lm -lpthread $LDX -o "$TMP/perf"
"$TMP/perf"
echo
echo "### PASS 2: SAFETY (ASan/UBSan)"
$CC $WARN -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer -I"$RT" $SRC -lm -lpthread $LDX -o "$TMP/safe"
ASAN_OPTIONS=${ASAN_OPTIONS:-detect_leaks=0} UBSAN_OPTIONS=halt_on_error=1 "$TMP/safe"
engram/test/run_interoception_p0.sh
+12 -8
View File
@@ -3,10 +3,14 @@
# Throwaway HOME + /tmp only. Never touches ~/.neuron or :8742.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
RT="$HERE/../../lang/runtime/el_runtime.c"
ST="$HERE/../../lang/runtime/engram_store.c"
GEO="$HERE/../../lang/runtime/engram_geometry.c"
VIDX="$HERE/../../lang/runtime/engram_vindex.c"
RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")"
# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link
# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c
# began calling into the other engram siblings. Unquoted on purpose: a list.
SSLFLAGS=""
if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then
SSLFLAGS="-I$O/include -L$O/lib"
fi
INC="$HERE/../../lang/runtime"
WORK="$(mktemp -d /tmp/engram-p0-XXXXXX)"
export HOME="$WORK/home"; mkdir -p "$HOME"
@@ -14,8 +18,8 @@ unset ENGRAM_STORE
fail=0
echo "== compile (plain) =="
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p0_emb.c" "$RT" "$ST" "$GEO" "$VIDX" \
-lcurl -lm -o "$WORK/p0" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p0_emb.c" $RTSRC $SSLFLAGS \
-lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p0" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
D="$WORK/d"; mkdir -p "$D"
"$WORK/p0" "$D" || { echo "FAIL: run"; fail=1; }
@@ -69,8 +73,8 @@ PY
echo
echo "== ASan+UBSan =="
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
-I "$INC" "$HERE/test_interoception_p0_emb.c" "$RT" "$ST" "$GEO" "$VIDX" \
-lcurl -lm -o "$WORK/p0.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -20 "$WORK/san_cc.log"; fail=1; }
-I "$INC" "$HERE/test_interoception_p0_emb.c" $RTSRC $SSLFLAGS \
-lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p0.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -20 "$WORK/san_cc.log"; fail=1; }
if [ -x "$WORK/p0.san" ]; then
export ASAN_OPTIONS=detect_leaks=0
DS="$WORK/ds"; mkdir -p "$DS"
engram/test/run_interoception_p1.sh
+12 -8
View File
@@ -3,10 +3,14 @@
# Throwaway HOME + /tmp only. Never touches ~/.neuron or :8742.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
RT="$HERE/../../lang/runtime/el_runtime.c"
ST="$HERE/../../lang/runtime/engram_store.c"
GEO="$HERE/../../lang/runtime/engram_geometry.c"
VIDX="$HERE/../../lang/runtime/engram_vindex.c"
RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")"
# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link
# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c
# began calling into the other engram siblings. Unquoted on purpose: a list.
SSLFLAGS=""
if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then
SSLFLAGS="-I$O/include -L$O/lib"
fi
INC="$HERE/../../lang/runtime"
WORK="$(mktemp -d /tmp/engram-p1-XXXXXX)"
export HOME="$WORK/home"; mkdir -p "$HOME"
@@ -14,8 +18,8 @@ unset ENGRAM_STORE ENGRAM_CONSOLIDATION ENGRAM_CONSOL_CONN_MIN ENGRAM_CONSOL_PER
fail=0
echo "== compile =="
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p1_consol.c" "$RT" "$ST" "$GEO" "$VIDX" \
-lcurl -lm -o "$WORK/p1" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p1_consol.c" $RTSRC $SSLFLAGS \
-lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p1" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
echo
echo "== (a) HEADLINE: hebb accrual curve over N co-activations (flag OFF, pure trunk) =="
@@ -129,8 +133,8 @@ cat "$WORK/off.txt" | sed 's/^/ /'
echo
echo "== ASan+UBSan (connect + perm + accrual-short) =="
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
-I "$INC" "$HERE/test_interoception_p1_consol.c" "$RT" "$ST" "$GEO" "$VIDX" \
-lcurl -lm -o "$WORK/p1.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
-I "$INC" "$HERE/test_interoception_p1_consol.c" $RTSRC $SSLFLAGS \
-lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p1.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
if [ -x "$WORK/p1.san" ]; then
export ASAN_OPTIONS=detect_leaks=0
DS="$WORK/san"; mkdir -p "$DS"
engram/test/run_interoception_p2.sh
+12 -8
View File
@@ -3,10 +3,14 @@
# Throwaway HOME + /tmp only. TC defaults to 3600s; we pin it for the math.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
RT="$HERE/../../lang/runtime/el_runtime.c"
ST="$HERE/../../lang/runtime/engram_store.c"
GEO="$HERE/../../lang/runtime/engram_geometry.c"
VIDX="$HERE/../../lang/runtime/engram_vindex.c"
RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")"
# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link
# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c
# began calling into the other engram siblings. Unquoted on purpose: a list.
SSLFLAGS=""
if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then
SSLFLAGS="-I$O/include -L$O/lib"
fi
INC="$HERE/../../lang/runtime"
WORK="$(mktemp -d /tmp/engram-p2-XXXXXX)"
export HOME="$WORK/home"; mkdir -p "$HOME"
@@ -15,8 +19,8 @@ unset ENGRAM_STORE
fail=0
echo "== compile =="
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p2_chrono.c" "$RT" "$ST" "$GEO" "$VIDX" \
-lcurl -lm -o "$WORK/p2" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p2_chrono.c" $RTSRC $SSLFLAGS \
-lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p2" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
sum_wm(){ python3 -c "import json,sys; g=json.load(open('$1')); print(sum(n.get('working_memory_weight',0) for n in g['nodes']))"; }
@@ -78,8 +82,8 @@ python3 -c "import sys; sys.exit(0 if abs($OFFWM-1.2)<1e-9 else 1)" \
echo
echo "== ASan+UBSan =="
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
-I "$INC" "$HERE/test_interoception_p2_chrono.c" "$RT" "$ST" "$GEO" "$VIDX" \
-lcurl -lm -o "$WORK/p2.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
-I "$INC" "$HERE/test_interoception_p2_chrono.c" $RTSRC $SSLFLAGS \
-lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p2.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
if [ -x "$WORK/p2.san" ]; then
export ASAN_OPTIONS=detect_leaks=0
DS="$WORK/san"; mkdir -p "$DS"
engram/test/run_interoception_p3.sh
+12 -8
View File
@@ -3,18 +3,22 @@
# Read-only pure primitive; no store, no flag. Throwaway /tmp only.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
RT="$HERE/../../lang/runtime/el_runtime.c"
ST="$HERE/../../lang/runtime/engram_store.c"
GEO="$HERE/../../lang/runtime/engram_geometry.c"
VIDX="$HERE/../../lang/runtime/engram_vindex.c"
RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")"
# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link
# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c
# began calling into the other engram siblings. Unquoted on purpose: a list.
SSLFLAGS=""
if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then
SSLFLAGS="-I$O/include -L$O/lib"
fi
INC="$HERE/../../lang/runtime"
WORK="$(mktemp -d /tmp/engram-p3-XXXXXX)"
export HOME="$WORK/home"; mkdir -p "$HOME"
fail=0
echo "== compile =="
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p3_drift.c" "$RT" "$ST" "$GEO" "$VIDX" \
-lcurl -lm -o "$WORK/p3" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p3_drift.c" $RTSRC $SSLFLAGS \
-lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p3" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
"$WORK/p3" > "$WORK/out.txt" 2>&1 || { echo "FAIL run"; cat "$WORK/out.txt"; fail=1; }
cat "$WORK/out.txt" | sed 's/^/ /'
@@ -52,8 +56,8 @@ PY
echo
echo "== ASan+UBSan =="
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
-I "$INC" "$HERE/test_interoception_p3_drift.c" "$RT" "$ST" "$GEO" "$VIDX" \
-lcurl -lm -o "$WORK/p3.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
-I "$INC" "$HERE/test_interoception_p3_drift.c" $RTSRC $SSLFLAGS \
-lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p3.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
if [ -x "$WORK/p3.san" ]; then
export ASAN_OPTIONS=detect_leaks=0
"$WORK/p3.san" >/dev/null 2>"$WORK/san.log"
engram/test/run_interoception_p4.sh
+12 -8
View File
@@ -2,10 +2,14 @@
# M-INTEROCEPTION P4 gate: afferent input counters in act-stats (additive).
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
RT="$HERE/../../lang/runtime/el_runtime.c"
ST="$HERE/../../lang/runtime/engram_store.c"
GEO="$HERE/../../lang/runtime/engram_geometry.c"
VIDX="$HERE/../../lang/runtime/engram_vindex.c"
RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")"
# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link
# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c
# began calling into the other engram siblings. Unquoted on purpose: a list.
SSLFLAGS=""
if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then
SSLFLAGS="-I$O/include -L$O/lib"
fi
INC="$HERE/../../lang/runtime"
WORK="$(mktemp -d /tmp/engram-p4-XXXXXX)"
export HOME="$WORK/home"; mkdir -p "$HOME"
@@ -13,8 +17,8 @@ unset ENGRAM_STORE
fail=0
echo "== compile =="
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p4_afferent.c" "$RT" "$ST" "$GEO" "$VIDX" \
-lcurl -lm -o "$WORK/p4" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p4_afferent.c" $RTSRC $SSLFLAGS \
-lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p4" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
"$WORK/p4" > "$WORK/out.txt" 2>&1 || { echo "FAIL run"; cat "$WORK/out.txt"; fail=1; }
grep -oE 'aff_[a-z_]+":[0-9]+' "$WORK/out.txt" | sed 's/^/ /' | head -30
@@ -53,8 +57,8 @@ PY
echo
echo "== ASan+UBSan =="
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
-I "$INC" "$HERE/test_interoception_p4_afferent.c" "$RT" "$ST" "$GEO" "$VIDX" \
-lcurl -lm -o "$WORK/p4.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
-I "$INC" "$HERE/test_interoception_p4_afferent.c" $RTSRC $SSLFLAGS \
-lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p4.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
if [ -x "$WORK/p4.san" ]; then
export ASAN_OPTIONS=detect_leaks=0
"$WORK/p4.san" >/dev/null 2>"$WORK/san.log"
engram/test/run_interoception_p5.sh
+12 -8
View File
@@ -2,10 +2,14 @@
# M-INTEROCEPTION P5 gate: dream-recall builtin engram_dreams_json (honesty rail).
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
RT="$HERE/../../lang/runtime/el_runtime.c"
ST="$HERE/../../lang/runtime/engram_store.c"
GEO="$HERE/../../lang/runtime/engram_geometry.c"
VIDX="$HERE/../../lang/runtime/engram_vindex.c"
RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")"
# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link
# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c
# began calling into the other engram siblings. Unquoted on purpose: a list.
SSLFLAGS=""
if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then
SSLFLAGS="-I$O/include -L$O/lib"
fi
INC="$HERE/../../lang/runtime"
WORK="$(mktemp -d /tmp/engram-p5-XXXXXX)"
export HOME="$WORK/home"; mkdir -p "$HOME"
@@ -13,8 +17,8 @@ unset ENGRAM_STORE
fail=0
echo "== compile =="
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p5_dreams.c" "$RT" "$ST" "$GEO" "$VIDX" \
-lcurl -lm -o "$WORK/p5" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p5_dreams.c" $RTSRC $SSLFLAGS \
-lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p5" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
D="$WORK/d"; mkdir -p "$D"
"$WORK/p5" "$D" > "$WORK/out.txt" 2>&1 || { echo "FAIL run"; cat "$WORK/out.txt"; fail=1; }
@@ -57,8 +61,8 @@ PY
echo
echo "== ASan+UBSan =="
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
-I "$INC" "$HERE/test_interoception_p5_dreams.c" "$RT" "$ST" "$GEO" "$VIDX" \
-lcurl -lm -o "$WORK/p5.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
-I "$INC" "$HERE/test_interoception_p5_dreams.c" $RTSRC $SSLFLAGS \
-lcurl -lssl -lcrypto -lpthread -lm -lm -o "$WORK/p5.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
if [ -x "$WORK/p5.san" ]; then
export ASAN_OPTIONS=detect_leaks=0
DS="$WORK/ds"; mkdir -p "$DS"
engram/test/run_m35_hebb_persist.sh
+10 -4
View File
@@ -6,8 +6,14 @@
# Writes ONLY under a throwaway /tmp dir with a throwaway HOME.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
RT="$HERE/../../lang/runtime/el_runtime.c"
ST="$HERE/../../lang/runtime/engram_store.c"
RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")"
# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link
# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c
# began calling into the other engram siblings. Unquoted on purpose: a list.
SSLFLAGS=""
if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then
SSLFLAGS="-I$O/include -L$O/lib"
fi
INC="$HERE/../../lang/runtime"
WORK="$(mktemp -d /tmp/engram-m35-XXXXXX)"
BIN="$WORK/m35"
@@ -17,7 +23,7 @@ unset ENGRAM_STORE
fail=0
echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m35_hebb_persist.c) =="
gcc -O1 -std=c11 -I "$INC" "$HERE/test_m35_hebb_persist.c" "$RT" "$ST" -lcurl -o "$BIN" 2>"$WORK/cc.log"
gcc -O1 -std=c11 -I "$INC" "$HERE/test_m35_hebb_persist.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$BIN" 2>"$WORK/cc.log"
if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi
echo
@@ -139,7 +145,7 @@ echo
echo "== 5) ASan+UBSan build, exercise the full persist+reboot flow (leaks off — harness intentionally leaks el_strdup) =="
SANBIN="$WORK/m35.san"
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
-I "$INC" "$HERE/test_m35_hebb_persist.c" "$RT" "$ST" -lcurl -o "$SANBIN" 2>"$WORK/san_cc.log"
-I "$INC" "$HERE/test_m35_hebb_persist.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$SANBIN" 2>"$WORK/san_cc.log"
if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else
export ASAN_OPTIONS=detect_leaks=0
DSAN="$WORK/san"; mkdir -p "$DSAN"
engram/test/run_m3_parity.sh
+10 -4
View File
@@ -4,8 +4,14 @@
# Writes ONLY under a throwaway /tmp dir with a throwaway HOME + ENGRAM_DATA_DIR.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
RT="$HERE/../../lang/runtime/el_runtime.c"
ST="$HERE/../../lang/runtime/engram_store.c"
RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")"
# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link
# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c
# began calling into the other engram siblings. Unquoted on purpose: a list.
SSLFLAGS=""
if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then
SSLFLAGS="-I$O/include -L$O/lib"
fi
INC="$HERE/../../lang/runtime"
WORK="$(mktemp -d /tmp/engram-m3-XXXXXX)"
DATA="$WORK/data"; mkdir -p "$DATA"
@@ -17,7 +23,7 @@ unset ENGRAM_STORE
fail=0
echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m3_parity.c) =="
gcc -O1 -std=c11 -I "$INC" "$HERE/test_m3_parity.c" "$RT" "$ST" -lcurl -o "$BIN" 2>"$WORK/cc.log"
gcc -O1 -std=c11 -I "$INC" "$HERE/test_m3_parity.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$BIN" 2>"$WORK/cc.log"
if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi
grep -i warning "$WORK/cc.log" | grep -iE 'engram_store|eg_store|eg_load|scan_nodes|scan_edges' && echo "(warnings in M3 code above)" || true
@@ -106,7 +112,7 @@ echo
echo "== 5) ASan+UBSan build, exercise M3 scan/boot/hooks (leaks off — harness intentionally leaks el_strdup) =="
SANBIN="$WORK/m3.san"
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
-I "$INC" "$HERE/test_m3_parity.c" "$RT" "$ST" -lcurl -o "$SANBIN" 2>"$WORK/san_cc.log"
-I "$INC" "$HERE/test_m3_parity.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$SANBIN" 2>"$WORK/san_cc.log"
if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else
export ASAN_OPTIONS=detect_leaks=0
DATA2="$WORK/data2"; mkdir -p "$DATA2"
engram/test/run_m7_traversal.sh
+10 -4
View File
@@ -6,8 +6,14 @@
# Writes ONLY under a throwaway /tmp dir with a throwaway HOME.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
RT="$HERE/../../lang/runtime/el_runtime.c"
ST="$HERE/../../lang/runtime/engram_store.c"
RTSRC="$("$HERE/../../scripts/el-runtime-sources.sh" "$HERE/../../lang/runtime")"
# The runtime is MULTI-FILE (lang/runtime/SOURCES). This harness used to link
# el_runtime.c + engram_store.c only, which stopped linking once el_runtime.c
# began calling into the other engram siblings. Unquoted on purpose: a list.
SSLFLAGS=""
if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then
SSLFLAGS="-I$O/include -L$O/lib"
fi
INC="$HERE/../../lang/runtime"
WORK="$(mktemp -d /tmp/engram-m7-XXXXXX)"
DATA="$WORK/data"; mkdir -p "$DATA"
@@ -22,7 +28,7 @@ unset ENGRAM_STORE
fail=0
echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m7_traversal.c) =="
gcc -O2 -std=c11 -I "$INC" "$HERE/test_m7_traversal.c" "$RT" "$ST" -lcurl -lm -o "$BIN" 2>"$WORK/cc.log"
gcc -O2 -std=c11 -I "$INC" "$HERE/test_m7_traversal.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o "$BIN" 2>"$WORK/cc.log"
if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi
echo " ok: compiled"
@@ -115,7 +121,7 @@ echo
echo "== 3) ASan+UBSan clean across parity + a small perf loop (leaks off — harness intentionally leaks el_strdup) =="
SANBIN="$WORK/m7.san"
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
-I "$INC" "$HERE/test_m7_traversal.c" "$RT" "$ST" -lcurl -lm -o "$SANBIN" 2>"$WORK/san_cc.log"
-I "$INC" "$HERE/test_m7_traversal.c" $RTSRC $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -lm -o "$SANBIN" 2>"$WORK/san_cc.log"
if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else
export ASAN_OPTIONS=detect_leaks=0
D2="$WORK/data2"; mkdir -p "$D2"
engram/test/run_vindex_concurrency_tests.sh
Executable
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env bash
# run_vindex_concurrency_tests.sh — regression harness for the 2026-08-16 soul crash.
#
# Four halves. The SET is the point: it separates two hazards the original two-half
# version conflated, and which have fixes in different files.
#
# 1. single ASan+UBSan, one thread. MUST be clean. Hard failure.
#
# 2. readers TSan, N readers, NO writer. Hazard (a): the visited set used
# to live on the index, so two pure READS stamped each other's
# epoch. Fixed in engram_vindex.c (frame-owned VVisit +
# `const VIndex*` search). MUST be clean. Hard failure.
#
# 3. unsynchronized TSan, writer + reader on a BARE index. Hazard (b): in-place
# HNSW insert rewires existing elements' neighbour lists and
# reallocs elems[]. EXPECTED TO RACE, PERMANENTLY. This is not
# a bug to fix inside engram_vindex.c — it is the executable
# proof that a publication boundary must exist above it.
# Not a failure. If it ever goes CLEAN, the test stopped
# interleaving and half 4 is no longer meaningful either.
#
# 4. published TSan, owner + N readers through a publication boundary
# (rwlock: readers shared, owner exclusive) mirroring
# eg_vindex_view / eg_vindex_maintain in lang/runtime/el_runtime.c.
# MUST be clean, and all inserts must land. Hard failure.
#
# See test_vindex_concurrency.c for the full story (SIGSEGV at ASCII address
# "gramNode", heap corruption in xzm_realloc, etc).
#
# usage: run_vindex_concurrency_tests.sh
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
RUNTIME="$(cd "$HERE/../../lang/runtime" && pwd)"
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
SRC="$HERE/test_vindex_concurrency.c"
VINDEX="$RUNTIME/engram_vindex.c"
fail=0
echo "== [1/4] single-threaded control under AddressSanitizer =="
cc -std=c11 -g -O1 -fsanitize=address,undefined -fno-omit-frame-pointer \
-I"$RUNTIME" -o "$WORK/single" "$SRC" "$VINDEX" -lm || { echo "BUILD FAILED"; exit 2; }
if ASAN_OPTIONS=detect_leaks=0 "$WORK/single" single; then
echo " -> OK"
else
echo " -> FAIL: the single-threaded control must always be clean."
echo " If this fails the bug is NOT (only) concurrency — look for a real"
echo " out-of-bounds or lifetime error in engram_vindex.c."
fail=1
fi
cc -std=c11 -g -O1 -fsanitize=thread -fno-omit-frame-pointer \
-I"$RUNTIME" -o "$WORK/conc" "$SRC" "$VINDEX" -lm || { echo "BUILD FAILED"; exit 2; }
# run_tsan <mode> <logfile>; echoes nothing, sets $tsan_raced
run_tsan() {
TSAN_OPTIONS="halt_on_error=0" "$WORK/conc" "$1" >"$2" 2>&1
tsan_rc=$?
if grep -q "ThreadSanitizer: data race" "$2"; then tsan_raced=1; else tsan_raced=0; fi
}
echo
echo "== [2/4] concurrent READERS, no writer (visited-set gate) =="
run_tsan readers "$WORK/readers.log"
if [ "$tsan_raced" = "1" ]; then
echo " -> REGRESSION: two concurrent reads still race."
grep -m1 -A6 "ThreadSanitizer: data race" "$WORK/readers.log" | sed 's/^/ /'
echo " The visited set was supposed to be owned by the call frame."
fail=1
else
echo " -> clean (concurrent reads are safe)"
fi
echo
echo "== [3/4] writer+reader on a BARE index (expected-race probe) =="
run_tsan unsynchronized "$WORK/unsync.log"
if [ "$tsan_raced" = "1" ]; then
echo " -> RACE DETECTED, as expected:"
grep -m1 -A4 "ThreadSanitizer: data race" "$WORK/unsync.log" | sed 's/^/ /'
echo " In-place HNSW insert mutates existing elements. Not fixable inside"
echo " engram_vindex.c — this is why the publication boundary exists."
else
echo " -> NOTE: no race reported. The probe did not interleave; half 4's"
echo " clean result proves less than it should. Investigate."
fi
echo
echo "== [4/4] owner+readers through the publication boundary (boundary gate) =="
run_tsan published "$WORK/pub.log"
if [ "$tsan_raced" = "1" ]; then
echo " -> REGRESSION: the publication boundary did not serialize the owner."
grep -m1 -A6 "ThreadSanitizer: data race" "$WORK/pub.log" | sed 's/^/ /'
fail=1
elif [ "$tsan_rc" != "0" ]; then
echo " -> FAIL: boundary clean under TSan but the run failed:"
tail -3 "$WORK/pub.log" | sed 's/^/ /'
fail=1
else
echo " -> clean (readers project concurrently; the owner's inserts all landed)"
fi
echo
[ "$fail" -eq 0 ] && echo "RESULT: PASS" || echo "RESULT: FAIL"
exit "$fail"
engram/test/run_wal_tests.sh
+11 -2
View File
@@ -3,8 +3,17 @@
set -e
HERE="$(cd "$(dirname "$0")" && pwd)"
REL="$HERE/../../lang/runtime"
# test_wal.c and test_failloud.c #include "el_runtime.c" directly, so el_runtime.c
# is already IN the translation unit — link the SIBLINGS only, or every symbol in
# it is defined twice. The siblings are still required: el_runtime.c calls into
# all six engram TUs. (lang/runtime/SOURCES is the source of truth.)
RTSIB="$("$HERE/../../scripts/el-runtime-sources.sh" "$REL" | grep -v '/el_runtime\.c$')"
SSLFLAGS=""
if command -v brew >/dev/null 2>&1 && O="$(brew --prefix openssl@3 2>/dev/null)"; then
SSLFLAGS="-I$O/include -L$O/lib"
fi
cc -O2 -fbracket-depth=1024 -Wno-parentheses-equality -I"$REL" \
"$HERE/test_wal.c" -lcurl -lpthread -o /tmp/test_wal
"$HERE/test_wal.c" $RTSIB $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/test_wal
HOME=/tmp/engram-throwaway-home /tmp/test_wal
# Fail-loud data-dir check (must exit 1 with a FATAL line):
cat > /tmp/test_failloud.c <<'C'
@@ -12,5 +21,5 @@ cat > /tmp/test_failloud.c <<'C'
int main(void){ unsetenv("ENGRAM_DATA_DIR"); unsetenv("HOME");
engram_resolve_data_dir(); printf("REACHED\n"); return 0; }
C
cc -O2 -fbracket-depth=1024 -Wno-parentheses-equality -I"$REL" /tmp/test_failloud.c -lcurl -lpthread -o /tmp/test_failloud
cc -O2 -fbracket-depth=1024 -Wno-parentheses-equality -I"$REL" /tmp/test_failloud.c $RTSIB $SSLFLAGS -lcurl -lssl -lcrypto -lpthread -lm -o /tmp/test_failloud
if env -u HOME -u ENGRAM_DATA_DIR /tmp/test_failloud; then echo "FAIL: should have exited"; exit 1; else echo "[PASS] fail-loud exit on unresolvable HOME"; fi
engram/test/test_grounding_vector.c
+176
View File
@@ -0,0 +1,176 @@
/* test_grounding_vector.c — deterministic tests for §7: the one decay model, the
* consequence gate, and the stored/derived split. Links engram_cognition.c
* directly; no server, no store, no network. See run_grounding_vector_tests.sh.
*
* NEGATIVE CONTROL (invariant §8.6). Every symbol exercised here —
* cog_decay_factor, cog_grounding_significant, cog_significance_inherent,
* CogGrounding, CogProvClass — is introduced by the change under test, so this
* suite does not COMPILE against the pre-change source, let alone pass. The
* runner documents the exact reproduction.
*/
#include "engram_cognition.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <math.h>
static int fails = 0;
static void ok(int cond, const char* what) {
printf(" %-62s %s\n", what, cond ? "PASS" : "*** FAIL ***");
if (!cond) fails++;
}
/* The decay formula exactly as el_runtime.c carried it before the move, so the
* refactor can be shown to be bit-identical rather than merely similar. */
static double old_engram_temporal_decay(long long age_ms, long long activation_count,
double temporal_decay_rate) {
if (age_ms <= 0) return 1.0;
double lambda = (temporal_decay_rate > 0.0) ? temporal_decay_rate : 0.693147;
double age_hours = (double)age_ms / 3600000.0;
double t_half = 168.0 * (1.0 + log(1.0 + (double)activation_count));
double factor = exp(-lambda * age_hours / t_half);
if (factor < 0.25) factor = 0.25;
return factor;
}
static CogGrounding base(void) {
CogGrounding g; memset(&g, 0, sizeof g);
g.present = 1;
g.factual = 0.60; g.relational = 0.60;
g.factual_now = 0.60; g.relational_now = 0.60;
g.associative = 0.1; g.polarity = 1.0;
g.prov = COG_PROV_TOLD;
g.fac_proj = 1.0; g.rel_proj = 1.0;
g.cos_angle = 0.9; g.agreement = 1;
g.ts = 1000; g.seq = 1; g.reinforcements = 3;
return g;
}
int main(void) {
const double F = 0.5, R = 0.5;
printf("\n== 1. DECAY IS THE ONE MODEL, AND IT IS BIT-IDENTICAL TO WHAT IT REPLACED ==\n");
{
long long ages[] = {0, 3600000LL, 86400000LL, 7*86400000LL, 30*86400000LL, 365*86400000LL};
int allsame = 1;
for (int i = 0; i < 6; i++)
for (int ac = 0; ac < 4; ac++) {
long long acs[] = {0, 1, 10, 1000};
double a = cog_decay_factor(ages[i], (double)acs[ac], 0.0);
double b = old_engram_temporal_decay(ages[i], acs[ac], 0.0);
if (a != b) allsame = 0;
}
ok(allsame, "cog_decay_factor == the pre-move engram_temporal_decay (24 pts)");
ok(cog_decay_factor(0, 0, 0.0) == 1.0, "age 0 -> no decay");
}
printf("\n DECAY OVER ELAPSED TIME (reinforcements = 0, default rate):\n");
printf(" %10s %10s\n", "elapsed", "decay");
{
struct { const char* label; long long ms; } pts[] = {
{"0", 0LL},
{"1 hour", 3600000LL},
{"1 day", 86400000LL},
{"3 days", 3LL*86400000LL},
{"7 days", 7LL*86400000LL},
{"14 days", 14LL*86400000LL},
{"30 days", 30LL*86400000LL},
{"90 days", 90LL*86400000LL},
};
double prev = 2.0; int monotone = 1;
for (unsigned i = 0; i < sizeof pts / sizeof pts[0]; i++) {
double d = cog_decay_factor(pts[i].ms, 0, 0.0);
printf(" %10s %10.6f\n", pts[i].label, d);
if (d > prev) monotone = 0;
prev = d;
}
ok(monotone, "decay is monotone non-increasing in elapsed time");
ok(fabs(cog_decay_factor(7LL*86400000LL, 0, 0.0) - 0.5) < 1e-6,
"7 days at zero reinforcements == exactly one half-life (0.5)");
ok(cog_decay_factor(7LL*86400000LL, 100, 0.0) > cog_decay_factor(7LL*86400000LL, 0, 0.0),
"reinforcement slows ageing (Lindy term)");
ok(cog_decay_factor(3650LL*86400000LL, 0, 0.0) == 0.25,
"floor is a preference not a cliff: bottoms out at 0.25");
}
printf("\n== 2. CONSEQUENCE GATE: EVERY TRIGGER, AND NO EPSILON ANYWHERE ==\n");
{
CogGrounding p = base(), n = base();
ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_NONE,
"identical vectors -> NONE (a re-read must not consolidate)");
n = base(); n.factual = 0.9999; n.factual_now = 0.9999;
ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_NONE,
"factual 0.60 -> 0.9999 without crossing the floor -> NONE");
n = base(); n.relational = 0.5001; n.relational_now = 0.5001;
ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_NONE,
"relational 0.60 -> 0.5001, still above floor -> NONE");
n = base(); n.factual_now = 0.4999;
ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_FACTUAL_FLOOR,
"a 0.1001 drop that CROSSES the floor -> FACTUAL_FLOOR");
n = base(); n.relational_now = 0.4999;
ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_RELATIONAL_FLOOR,
"relational crossing its floor -> RELATIONAL_FLOOR");
n = base(); n.cos_angle = -0.05; n.agreement = -1;
ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_AGREEMENT_FLIP,
"agreement +1 -> -1 -> AGREEMENT_FLIP");
n = base(); n.fac_proj = -0.2;
ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_DIRECTION_REVERSAL,
"factual gradient reverses -> DIRECTION_REVERSAL");
n = base(); n.rel_proj = -0.2;
ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_DIRECTION_REVERSAL,
"relational gradient reverses -> DIRECTION_REVERSAL");
n = base(); n.polarity = -1.0;
ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_POLARITY_FLIP,
"support -> contradiction -> POLARITY_FLIP (inherent)");
n = base(); n.polarity = 0.0;
ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_POLARITY_FLIP,
"support -> ignorance (zero) -> POLARITY_FLIP: not the same state");
n = base(); n.prov = COG_PROV_OBSERVED;
ok(cog_grounding_significant(&p, &n, F, R) == COG_SIG_PROVENANCE_CHANGE,
"told -> observed -> PROVENANCE_CHANGE (inherent)");
CogGrounding fresh; memset(&fresh, 0, sizeof fresh);
ok(cog_grounding_significant(&fresh, &n, F, R) == COG_SIG_FIRST_RECORD,
"no prior version -> FIRST_RECORD");
}
printf("\n== 3. INHERENT MOVES BYPASS THE SALIENCE GATE ==\n");
ok(cog_significance_inherent(COG_SIG_POLARITY_FLIP), "polarity flip is inherent");
ok(cog_significance_inherent(COG_SIG_PROVENANCE_CHANGE), "provenance change is inherent");
ok(cog_significance_inherent(COG_SIG_FIRST_RECORD), "first record is inherent");
ok(!cog_significance_inherent(COG_SIG_FACTUAL_FLOOR), "a floor crossing is NOT inherent");
ok(!cog_significance_inherent(COG_SIG_NONE), "NONE is not inherent");
printf("\n== 4. THE STORED/DERIVED SPLIT: DERIVED VALUES ARE NEVER SERIALIZED ==\n");
{
CogGrounding g = base();
g.decay = 0.3333; g.factual_now = 0.1234; g.relational_now = 0.2345;
g.associative_now = 0.4567; g.age_ms = 999999; g.stale = 1;
char* m = cog_grounding_metadata("pre-existing=keepme", &g);
ok(m != NULL, "serializer returns a document");
ok(m && strstr(m, "pre-existing=keepme"), "pre-existing edge metadata preserved verbatim");
ok(m && strstr(m, "GRD1"), "GRD1 magic present");
ok(m && !strstr(m, "0.3333"), "decay is NOT stored");
ok(m && !strstr(m, "0.1234"), "factual_now is NOT stored");
ok(m && !strstr(m, "0.2345"), "relational_now is NOT stored");
ok(m && !strstr(m, "0.4567"), "associative_now is NOT stored");
ok(m && !strstr(m, "999999"), "age is NOT stored");
ok(m && strstr(m, "told"), "provenance class IS stored");
ok(m && strstr(m, "0.6"), "the factual/relational dimensions ARE stored");
if (m) { printf("\n --- serialized GRD1 block ---\n%s -----------------------------\n", m); }
free(m);
}
printf("\n%s (%d failure%s)\n\n", fails ? "SOME TESTS FAILED" : "ALL TESTS PASSED",
fails, fails == 1 ? "" : "s");
return fails ? 1 : 0;
}
engram/test/test_vindex_concurrency.c
+251
View File
@@ -0,0 +1,251 @@
/* test_vindex_concurrency.c — regression test for the 2026-08-16 soul crash.
*
* WHAT BROKE: the soul daemon crash-looped (5 crashes in ~100s) with SIGSEGV in
* search_layer <- vindex_insert <- eg_vindex_sync, a SIGABRT, and a fault inside
* xzm_realloc's own freelist — i.e. heap corruption. The SIGSEGV address
* 0x65646f4e6d617267 is little-endian ASCII "gramNode": string bytes being
* dereferenced as an Elem vector pointer.
*
* ROOT CAUSE: VIndex owns its traversal scratch (visited[] + visit_epoch), and
* search_layer mutates it via visited_reset(). So the index is unsafe for ANY
* concurrent use — including two concurrent READS. soul.el starts http_serve_async
* (a thread per connection) and then runs awareness_run() on the main thread, which
* reaches the same global index through engram_activate; nothing serialized them.
*
* Neither hnswlib nor FAISS puts the visited set on the index: hnswlib checks one
* out of a VisitedListPool per query, FAISS uses a thread_local VisitedTable.
*
* THE ORIGINAL `concurrent` HALF CONFLATED TWO DISTINCT HAZARDS (2026-08-16). It ran
* a writer against a reader on one bare index, so it could not tell apart:
*
* (a) READ/READ corruption — two searches stamping each other's visited epoch.
* A defect INSIDE engram_vindex.c, fixable there, and now fixed: the visited
* set moved to the call frame and vindex_search takes a `const VIndex*`.
*
* (b) WRITE/READ corruption — vindex_insert rewires the neighbour lists of
* EXISTING elements and reallocs elems[], so an insert is a mutation of the
* whole structure. This is NOT fixable inside engram_vindex.c at any price:
* it is inherent to in-place HNSW. It requires a publication boundary ABOVE
* the data structure (el_runtime.c: eg_vindex_view / eg_vindex_maintain).
*
* Conflating them made the suite unfailable-then-unpassable: fixing (a) left (b)
* still racing, which reads as "the fix did not work" when in fact a different,
* correctly-located fix is what (b) needs. So the halves are now separate:
*
* single N clustered vectors, ONE thread, ASan. The CONTROL. Must always
* be clean. When this passes and a concurrent half fails, the defect
* is concurrency, not an out-of-bounds/logic error in the graph code.
* (On 2026-08-16 this control cleared all 13,820 real dim-768 store
* vectors under ASan, which DISPROVED an inspection-derived hypothesis
* about an out-of-bounds reverse-link write at engram_vindex.c:340.)
*
* readers N reader threads, NO writer, one shared index, TSan. This is
* hazard (a) in isolation. It RACED before the visited set moved off
* the index struct and must be CLEAN now. Hard gate.
*
* unsynchronized writer + reader on a bare index, TSan. Hazard (b) in isolation.
* EXPECTED TO RACE, permanently — it is the executable proof that
* the index cannot be made safe from the inside, and therefore that
* the publication boundary in el_runtime.c has to exist. If this
* ever goes clean, the test stopped interleaving; do not celebrate.
*
* published writer + readers through a publication boundary that mirrors
* eg_vindex_view / eg_vindex_maintain (rwlock: readers shared,
* the single owner exclusive), TSan. Must be CLEAN. Hard gate.
* This is what proves the shape of the runtime fix, in the same
* process, rather than asserting it.
*
* Absence of a crash does NOT mean absence of a race — always read the sanitizer
* verdict, never just the exit code.
*
* Build/run: engram/test/run_vindex_concurrency_tests.sh
*/
#include "engram_vindex.h"
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#define DIM 128
#define NVEC 3000
#define SEED_N 50
static VIndex* g_ix;
static float* g_vecs;
/* Deterministic filler. Real embeddings are strongly correlated, not uniform noise;
* clustering keeps many candidates near-equidistant, which exercises the diversity
* heuristic and the visited set far harder than random vectors do. */
static void fill_vectors(void) {
g_vecs = (float*)malloc((size_t)NVEC * DIM * sizeof(float));
if (!g_vecs) { fprintf(stderr, "OOM\n"); exit(1); }
for (int i = 0; i < NVEC; i++) {
int cluster = i % 8;
for (int d = 0; d < DIM; d++)
g_vecs[(size_t)i * DIM + d] =
(float)(((d + cluster * 7) % 13) / 13.0) +
(float)(((i * 2654435761u + (unsigned)d) % 97) / 9700.0);
}
}
static void* writer_fn(void* arg) {
(void)arg;
for (int i = SEED_N; i < NVEC; i++)
(void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM);
return NULL;
}
static void* reader_fn(void* arg) {
(void)arg;
uint64_t ids[8]; float ds[8];
for (int i = 0; i < 20000; i++)
(void)vindex_search(g_ix, g_vecs + (size_t)(i % NVEC) * DIM, 8, 0, ids, ds);
return NULL;
}
static int run_single(void) {
printf("[single] inserting %d vectors on one thread (ASan control)\n", NVEC);
g_ix = vindex_create(DIM, 0, 0);
if (!g_ix) { fprintf(stderr, "[single] vindex_create failed\n"); return 1; }
for (int i = 0; i < NVEC; i++) {
if (vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM) != 0) {
fprintf(stderr, "[single] insert %d failed\n", i); return 1;
}
}
if (vindex_size(g_ix) != (size_t)NVEC) {
fprintf(stderr, "[single] size %zu != %d\n", vindex_size(g_ix), NVEC); return 1;
}
uint64_t ids[16]; float ds[16];
for (int q = 0; q < 200; q++) {
int k = vindex_search(g_ix, g_vecs + (size_t)((q * 7) % NVEC) * DIM, 16, 0, ids, ds);
if (k < 0) { fprintf(stderr, "[single] search failed at q=%d\n", q); return 1; }
}
vindex_free(g_ix); g_ix = NULL;
printf("[single] PASS — no memory error (this must ALWAYS pass)\n");
return 0;
}
/* Hazard (b) in isolation: writer + reader on a BARE index, no boundary. */
static int run_unsynchronized(void) {
printf("[unsynchronized] 1 writer + 1 reader on a BARE index (TSan probe)\n");
printf("[unsynchronized] a race here is EXPECTED and PERMANENT — in-place HNSW\n");
printf("[unsynchronized] insert rewires existing elements. This is the proof that\n");
printf("[unsynchronized] the publication boundary must live ABOVE engram_vindex.c.\n");
g_ix = vindex_create(DIM, 0, 0);
if (!g_ix) { fprintf(stderr, "[unsynchronized] vindex_create failed\n"); return 1; }
for (int i = 0; i < SEED_N; i++)
(void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM);
pthread_t w, r;
if (pthread_create(&w, NULL, writer_fn, NULL) ||
pthread_create(&r, NULL, reader_fn, NULL)) {
fprintf(stderr, "[unsynchronized] pthread_create failed\n"); return 1;
}
pthread_join(w, NULL);
pthread_join(r, NULL);
vindex_free(g_ix); g_ix = NULL;
printf("[unsynchronized] completed — CHECK THE SANITIZER VERDICT, not this line.\n");
return 0;
}
/* ── hazard (a) in isolation: concurrent READS only ───────────────────────────
* This is what the frame-owned visited set fixes. Before that change, two
* vindex_search calls on one index wrote each other's epoch stamp; TSan reported
* the race at visited_reset and the traversal then walked bogus element indices. */
#define NREADERS 4
static int run_readers(void) {
printf("[readers] %d concurrent readers, NO writer, one shared index (TSan)\n", NREADERS);
printf("[readers] this is the visited-set regression gate — must be CLEAN.\n");
g_ix = vindex_create(DIM, 0, 0);
if (!g_ix) { fprintf(stderr, "[readers] vindex_create failed\n"); return 1; }
for (int i = 0; i < NVEC; i++)
(void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM);
pthread_t t[NREADERS];
for (int i = 0; i < NREADERS; i++)
if (pthread_create(&t[i], NULL, reader_fn, NULL)) {
fprintf(stderr, "[readers] pthread_create failed\n"); return 1;
}
for (int i = 0; i < NREADERS; i++) pthread_join(t[i], NULL);
vindex_free(g_ix); g_ix = NULL;
printf("[readers] completed — CHECK THE SANITIZER VERDICT, not this line.\n");
return 0;
}
/* ── the publication boundary, mirroring el_runtime.c ─────────────────────────
* Readers take the boundary SHARED and hold it across the whole search; the one
* owner takes it EXCLUSIVE to extend. Same shape as eg_vindex_view /
* eg_vindex_maintain. Note the reader's index pointer is `const VIndex*` — the
* compiler, not this comment, is what stops a reader inserting. */
static pthread_rwlock_t g_pub = PTHREAD_RWLOCK_INITIALIZER;
static void* pub_writer_fn(void* arg) {
(void)arg;
for (int i = SEED_N; i < NVEC; i++) {
pthread_rwlock_wrlock(&g_pub);
(void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM);
pthread_rwlock_unlock(&g_pub);
}
return NULL;
}
static void* pub_reader_fn(void* arg) {
(void)arg;
uint64_t ids[8]; float ds[8];
for (int i = 0; i < 5000; i++) {
pthread_rwlock_rdlock(&g_pub);
const VIndex* view = g_ix; /* immutable view */
(void)vindex_search(view, g_vecs + (size_t)(i % NVEC) * DIM, 8, 0, ids, ds);
pthread_rwlock_unlock(&g_pub);
}
return NULL;
}
static int run_published(void) {
printf("[published] 1 owner + %d readers through a publication boundary (TSan)\n", NREADERS);
printf("[published] this is the eg_vindex_view/eg_vindex_maintain gate — must be CLEAN.\n");
g_ix = vindex_create(DIM, 0, 0);
if (!g_ix) { fprintf(stderr, "[published] vindex_create failed\n"); return 1; }
for (int i = 0; i < SEED_N; i++)
(void)vindex_insert(g_ix, (uint64_t)i, g_vecs + (size_t)i * DIM);
pthread_t w, r[NREADERS];
if (pthread_create(&w, NULL, pub_writer_fn, NULL)) {
fprintf(stderr, "[published] pthread_create failed\n"); return 1;
}
for (int i = 0; i < NREADERS; i++)
if (pthread_create(&r[i], NULL, pub_reader_fn, NULL)) {
fprintf(stderr, "[published] pthread_create failed\n"); return 1;
}
pthread_join(w, NULL);
for (int i = 0; i < NREADERS; i++) pthread_join(r[i], NULL);
if (vindex_size(g_ix) != (size_t)NVEC) {
fprintf(stderr, "[published] size %zu != %d — the owner lost inserts\n",
vindex_size(g_ix), NVEC);
vindex_free(g_ix); g_ix = NULL; return 1;
}
vindex_free(g_ix); g_ix = NULL;
printf("[published] all %d inserts landed; CHECK THE SANITIZER VERDICT too.\n", NVEC);
return 0;
}
int main(int argc, char** argv) {
const char* mode = (argc > 1) ? argv[1] : "single";
fill_vectors();
int rc;
if (!strcmp(mode, "single")) rc = run_single();
else if (!strcmp(mode, "readers")) rc = run_readers();
else if (!strcmp(mode, "unsynchronized")) rc = run_unsynchronized();
else if (!strcmp(mode, "published")) rc = run_published();
/* back-compat: the pre-split name meant the bare writer+reader probe. */
else if (!strcmp(mode, "concurrent")) rc = run_unsynchronized();
else {
fprintf(stderr, "usage: %s [single|readers|unsynchronized|published]\n", argv[0]);
rc = 2;
}
free(g_vecs);
return rc;
}
ingest/src/ingest.el
+56 -12
View File
@@ -13,7 +13,7 @@
// relations add edges. Every node enters with PROVENANCE + grounding-level
// + stewardship class from the moment of entry.
//
// transduce() is THE single mechanism — one function, polymorphic, with no
// transduce_bytes() is THE single mechanism — one function, polymorphic, with no
// content-type branch inside it. It does not ask whether a payload is
// prose, structured data, or raw/opaque bytes (audio, or anything else);
// it runs one boundary-scan-with-fixed-window-fallback chunking algorithm
@@ -401,10 +401,54 @@ fn head80(s: String) -> String {
// truncates at the first embedded NUL, which is routine in real binary
// bytes) is a MECHANICAL fidelity concern that belongs to whatever produced
// `source` (see ingest_file's file_source_string below) — not a
// content-type judgment made in here. transduce() never learns whether a
// content-type judgment made in here. transduce_bytes() never learns whether a
// chunk is plain text or a base64-encoded raw-byte window; every chunk is
// handled identically either way.
fn transduce(nodes: [String], edges: [String], source: String,
// NAMING, CORRECTED 2026-08-16 (second pass). This function was renamed
// `transduce` -> `transduce_bytes` earlier the same day, on the reasoning
// that it "was never signal->geometry — it chunks already-extracted content
// and PACKS it into a node+edge manifold, one layer up, and it had taken the
// name that belongs to the primitive underneath it."
//
// THAT REASONING WAS BACKWARDS, and it is worth recording why rather than
// quietly re-renaming. Producing a node+edge manifold is not a layer above
// transduction — it IS transduction. Transduction is not conversion. When you
// take in music you do not store the song as one discrete geometry; you break
// it into its component parts and store the geometry of each along with the
// relations between them. The song is the structure of those relations.
// Signal -> one vector is the operation UNDERNEATH transduction, and its name
// is encoding, or geometry. So the layer that was doing it right got renamed
// out of the way so the layer doing it wrong could have the name.
//
// The primitive has since been corrected: `transduce(signal, modality)` now
// returns a Manifold — components plus relations — not a Geometry
// (el_runtime.c, "Manifold"). The two layers are therefore doing the SAME KIND
// of thing, and the inversion dissolves rather than needing to be re-argued.
//
// What is left is a real distinction, and it is about MODALITY, not layering:
//
// * `transduce(signal, modality)` dispatches to a realizer that KNOWS the
// modality and can name its components — for audio: pitch, interval,
// rhythm, harmonic function.
// * `transduce_bytes` below is the OPAQUE-BYTES realizer: the decomposition
// available to a reader that knows nothing about what it is reading. It
// still yields components and relations (chunk nodes; contains / precedes
// / section_of edges), which is why it is transduction and not packing. It
// just cuts on the only structure visible without understanding — byte
// boundaries — so its components are positional rather than meaningful.
// That is a LIMITATION of this realizer, not the definition of the
// operation.
//
// The name is suffixed by its modality, not demoted to a lesser layer. Keeping
// a distinct symbol is also still mechanically required: every El `fn name`
// compiles to a global C symbol, so reusing `transduce` here is a hard
// `conflicting types` error the moment ingest.c links el_runtime.c.
//
// WHERE THIS SHOULD GO: this function should become a registered realizer
// returning a real Manifold, so ingest rides the same primitive as every other
// modality instead of carrying a parallel implementation. Not done here.
// Nothing about this function's behaviour changed in this pass.
fn transduce_bytes(nodes: [String], edges: [String], source: String,
prov: String, ground: String, steward: String,
root_lid: String, root_title: String) -> [String] {
let tagbase: String = "prov:" + prov + " ground:" + ground + " steward:" + steward
@@ -531,8 +575,8 @@ fn default_steward() -> String {
// trustworthy verbatim. When they don't (silent truncation happened),
// rebuild the payload as base64-encoded fixed-size windows read directly
// off disk (fs_read_b64_chunk — binary-safe in C), joined with the same
// "\n\n" boundary marker transduce()'s generic scan already looks for, so
// transduce() sees one ordinary boundary-delimited payload and runs its one
// "\n\n" boundary marker transduce_bytes()'s generic scan already looks for, so
// transduce_bytes() sees one ordinary boundary-delimited payload and runs its one
// algorithm on it exactly as it would on prose — it never learns that a
// fidelity problem occurred upstream, let alone why.
fn file_source_string(path: String, text: String, real_size: Int) -> String {
@@ -541,7 +585,7 @@ fn file_source_string(path: String, text: String, real_size: Int) -> String {
// 3072 raw bytes -> 4096 base64 chars (3 divides evenly into base64's
// 3-byte/4-char ratio); keeps each resulting node's content a clean,
// bounded, low-kilobytes unit, same order of magnitude as the fixed
// fallback window in transduce() itself.
// fallback window in transduce_bytes() itself.
let win: Int = 3072
let out: String = ""
let off: Int = 0
@@ -561,7 +605,7 @@ fn file_source_string(path: String, text: String, real_size: Int) -> String {
}
// ingest one file -> report JSON. Uniform for every file regardless of
// extension or content — transduce() decides nothing about content-type, so
// extension or content — transduce_bytes() decides nothing about content-type, so
// neither does this function; it only decides whether the raw bytes made it
// through the read intact (file_source_string), which is a fidelity
// question, not a format one.
@@ -573,14 +617,14 @@ fn ingest_file(path: String) -> String {
return "{\"error\":\"empty or unreadable\",\"path\":" + j_q(path) + "}"
}
let prov: String = "file:" + path
let packed: [String] = transduce(el_list_empty(), el_list_empty(),
let packed: [String] = transduce_bytes(el_list_empty(), el_list_empty(),
source, prov, default_ground(), default_steward(),
"doc:" + basename(path), basename(path))
return merge_packed(packed)
}
// ingest a directory: walk one level, ingest every file found, aggregate.
// No extension filter — transduce() handles any payload uniformly now, so
// No extension filter — transduce_bytes() handles any payload uniformly now, so
// there is no content-type gate at the directory boundary either.
fn ingest_dir(path: String) -> String {
let entries: [String] = fs_list(path)
@@ -615,7 +659,7 @@ fn ingest_dir(path: String) -> String {
fn ingest_url(url: String) -> String {
let body: String = http_get(url)
if str_eq(body, "") { return "{\"error\":\"empty fetch\",\"url\":" + j_q(url) + "}" }
let packed: [String] = transduce(el_list_empty(), el_list_empty(),
let packed: [String] = transduce_bytes(el_list_empty(), el_list_empty(),
body, "url:" + url, "extracted", "public-web",
"url:" + url, url)
return merge_packed(packed)
@@ -630,7 +674,7 @@ fn ingest_llm(query: String) -> String {
let resp: String = http_post_json("http://127.0.0.1:11434/api/generate", body)
let answer: String = json_get_string(resp, "response")
if str_eq(answer, "") { return "{\"error\":\"no model response\"}" }
let packed: [String] = transduce(el_list_empty(), el_list_empty(),
let packed: [String] = transduce_bytes(el_list_empty(), el_list_empty(),
answer, "llm:" + model + ":" + query, "candidate-provisional", "guide-provisional",
"llm:" + query, "guide answer: " + query)
return merge_packed(packed)
@@ -682,7 +726,7 @@ fn ingest_stream(path: String) -> String {
// It is NOT a content-type flag: it says nothing about what's inside the
// bytes once fetched, and none of the five ingest_* functions it selects
// among interpret their payload differently by content shape anymore —
// they all hand off to the single, format-agnostic transduce(). The old
// they all hand off to the single, format-agnostic transduce_bytes(). The old
// "structured" value (a caller-declared alias for "file", used only to hint
// the now-removed JSON-vs-prose branch) is gone along with that branch.
let kind: String = env("INGEST_KIND")
lang/AGENTS.md
+74 -15
View File
@@ -18,8 +18,23 @@ night) and `02-components.md §5`.
`relate`, `supersede` (evolve/tombstone/promote, never a hard delete) — plus the
agentic primitives `think`/`attend`/`learn`/`ground`/`assert`. The old noun is a
`type` parameter. Implemented in `tools/api-reshape/surface.el` with a parity
harness (`parity.sh`); aperture proven to bound output. **Not yet:** compiled
into the MCP server, hot-swap, all-alias dispatch.
harness (`parity.sh`); aperture proven to bound output. ~~**Not yet:** compiled
into the MCP server~~ — **shipped (verified 2026-08-16): the live MCP surface is
exactly these nine ops** (`read` · `write` · `relate` · `supersede` · `think` ·
`attend` · `assert` · `ground` · `learn`); the ~87-tool surface is gone.
`attend` absorbed `getInstructions` / `beginSession`'s active-context sweep /
`checkEvents` — those are **gone, not gapped**. Still outstanding: hot-swap,
all-alias dispatch.
> **⚠ Two of those primitives are the wrong shape, and it is documented
> (2026-08-16).** `think({seeds, faculty})` treats **faculties as parameters**;
> they are **operations** — `reason` changes the estimate (a read), `induce`
> changes the parameters, `abduce` changes the *structure* (a write
> `GeoGradient` cannot express). And `ground` mints a `grounded-by` edge, but
> **grounding is not a subsystem — it IS the edge weight**: a property *of* a
> relation, not a relation *between* nodes. Authority:
> `lang/spec/correspondence-and-censorship.md`. Do not re-derive it; if you think
> a section is wrong, say so with a measurement.
- **Decorated seam.** `@route(path,method,…)` makes codegen synthesize
`el_route_dispatch` (replacing the hand-written `handle_request` if-else) —
proven decorate→serve on `:8951`. `@manager`/`@engine`/`@accessor` are **parsed
@@ -62,34 +77,78 @@ This is where almost all work belongs. El programs are source files that get com
This is the self-contained C OS-boundary layer. It provides the `__`-prefixed primitives that compiled El programs call: libcurl HTTP, pthreads, filesystem I/O, arena allocation, etc. It is **not generated** — it is maintained by hand.
The old `el_runtime.c` has been archived to `runtime/legacy/`. The runtime is now native El (`runtime/*.el`). `el_seed.c` replaces `el_runtime.c` as the sole C compilation dependency.
The runtime is native El (`runtime/*.el`) over a C OS-boundary. **Status (verified 2026-08-16):** the migration to a seed-only boundary is *in progress, not done*.
**Only edit `el_seed.c` when you genuinely need OS-level access** (raw sockets, GPU calls, new libcurl features). For everything else, write El.
**The runtime is MULTI-FILE. There is no single-file link target and there has not been one for months.** The canonical link set is listed once, in **`runtime/SOURCES`**, and printed by `scripts/el-runtime-sources.sh`. It currently holds ten translation units: `el_runtime.c`, `el_seed.c`, the six `engram_*.c` concern files, and `eg_cosine_batch{,_strategy_cpu}.c`.
When you do add a C builtin:
1. Add the C function to `el_seed.c`
2. Declare it in `el_seed.h`
3. Add it to the `builtin_arity` table in `el-compiler/src/codegen.el` (so the compiler knows the arg count)
4. Rebuild the elc binary (see below)
- `runtime/el_runtime.c` (~940 KB, 20.5k lines) — **LIVE, and oversized.** It began life on 2026-05-03 as a temporary build shim: it was deleted that afternoon ("runtime is 100% native El") and restored 25 minutes later, explicitly "UNTIL the compiler is updated to emit `#include el_seed.h`". That `until` never arrived, and in the 3.5 months since, the file doubled. **It is not a volatility unit — it is a dumping ground.** ~47.5% of it is engram code that belongs in the six sibling files that already exist. Do not add to it. See "Where a new C builtin goes" below.
- `runtime/el_seed.c` — the intended hand-maintained `__`-prefixed seed (thin wrappers over the above).
- `runtime/engram_{store,vindex,geometry,reason,verify,cognition}.c` — the engram concerns, each with its own header. `el_runtime.c` `#include`s all six headers and makes hard cross-TU calls into all six.
> **Linking `el_runtime.c` alone does not work and has not for months.** It fails at `ld` with undefined symbols (`engram_ground_json`, `engram_activate_inner`, `eg_find_relation`, `cog_assert_two_axis`, …). Any recipe, script, or CI step that names `el_runtime.c` by itself is stale — replace it with `$(scripts/el-runtime-sources.sh lang/runtime)`.
**Only edit these when you genuinely need OS-level access** (raw sockets, GPU calls, new libcurl features, a new engram store op). For everything else, write El.
#### Where a new C builtin goes
**Put it in the `.c` that owns the concern — NOT in `el_runtime.c`.**
*Placement is a link-time concern. The compiler cannot tell which `.c` a symbol came from, and never could.* `builtin_arity` in `el-compiler/src/codegen.el` maps NAME → ARITY INT and nothing else (~413 entries); the El name is emitted as the exact C symbol and resolved by `ld`. Proof, if you want it: `nm lang/dist/platform/elc` on the *shipped* compiler shows `T _engram_geo_reify_index_new` (defined in `engram_geometry.c`), `T _vindex_insert` (`engram_vindex.c`), `T _engram_think` (`engram_cognition.c`), `T _engram_reason_abduce` (`engram_reason.c`). **The shipped compiler is already linked from ten translation units.** A builtin defined in a sibling `.c` is exactly as linkable as one defined in `el_runtime.c`.
Choose the file by concern: engram store ops → `engram_store.c`; index → `engram_vindex.c`; geometry/priming → `engram_geometry.c`; reasoning → `engram_reason.c`; grounding/consistency → `engram_verify.c`; think/stance → `engram_cognition.c`. **If no existing file owns it, create one** — add the `.c` to `runtime/SOURCES` (one line) and every build path picks it up. For a builtin that belongs to a downstream program rather than the runtime, declare `c_source "path/to/file.c"` in that program's `manifest.el`; `elb` already links it (`parse_manifest_c_sources`, `lang/elb.el:82`).
> **`el_runtime.c` is on a ratchet and will reject your commit.** `runtime/BUDGET` caps it at its current line count *with no headroom*, and separately caps the number of `engram_*`/`eg_*`/`cog_*` function definitions in it. `scripts/check-runtime-growth.sh` enforces both in CI and in `.githooks/pre-commit`. **The numbers may only ever go down — do not raise them.** Every other runtime file is deliberately uncapped, because that is where the code is supposed to go. When you move code *out*, lower the numbers in the same commit; the guard tells you the new values.
When you add a C builtin (verbatim-emit recipe — the El name is emitted as the exact C symbol; `builtin_arity` is an arity guard only, not a dispatch table):
1. Implement the C function in the **concern-owning `.c`** (and declare it in that file's `.h`). Add the file to `runtime/SOURCES` if it is new. Only put it in `el_runtime.c` if it is genuinely EL core (val/str/map/list/arena) — that is ~8% of what is in there today.
2. Add a `__`-prefixed thin wrapper in `el_seed.c` and declare it in `el_seed.h`.
3. Add the name to `builtin_arity` in `el-compiler/src/codegen.el` — add **both** the plain and `__`-prefixed spellings.
4. Rebuild the elc binary (see below) and confirm the self-host fixpoint is byte-identical.
5. **Prove it with a NEGATIVE CONTROL.** Show the test FAILING on a build without your change, then passing with it. A test that has never been seen to fail has proven nothing.
> **Step 5 is not optional, and step 4 does not cover it.** The fixpoint proves the *compiler reproduces itself*. It says nothing whatsoever about whether your builtin works. A recipe ending at "byte-identical" reads as complete while having verified nothing about the thing just added — which is why this file, until 2026-08-16, produced builtins with no tests at all.
>
> Measured cost of the omission (2026-08-16): `engram_node_set_emb`, `engram_curiosity_json` and `dream_set_handler` were all added in one session with zero tests. Separately, a UTF-8 fix was written, tested, and **the test passed on the unpatched build too** — the defect was elsewhere entirely, and only building the pre-fix binary exposed it. Without a negative control that fix would have merged as verified.
>
> Two shapes that pass while proving nothing, both hit the same day:
> - A test that never exercises your change (the route supplied a default that bypassed the code under test).
> - An induction that loses a race. `curl --max-time` on a large response left *both* builds alive; only `SO_LINGER 0` — a genuine RST, so the peer is provably gone — reproduced the failure. Six of ten attempts is not a control.
>
> Before every probe, confirm **your** process bound the port (`lsof -nP -iTCP:<port>`, match the PID). A stale instance answering on the port has silently produced false results here more than once, and `pkill -f` does not reliably match an argv like `./engram`.
Worked example: the `engram_assert_json` (op_assert seam) and `engram_node_full_in`/`engram_connect_in` (purview write-side) primitives added 2026-08-15 follow exactly this recipe.
---
## Rebuilding the Compiler
After changing any `.el` source in `el-compiler/src/`:
After changing any `.el` source in `el-compiler/src/` (run from the `lang/` dir):
```bash
cd /Users/will/Development/neuron-technologies/foundation/el
# 1. Stage2: current elc compiles the (modified) compiler to C
./dist/platform/elc elc-cli.el > elc-new.c
cc -std=c11 -I runtime -lcurl -lpthread \
# 2. Build the new compiler. Link the WHOLE runtime set, not el_runtime.c alone:
# el_runtime.c calls into engram_store / engram_vindex / eg_cosine_batch and
# wraps el_seed.c, so a one-file link fails at `ld` with undefined symbols
# (verified 2026-08-16 — the previous single-file line in this doc is stale).
cc -std=c11 -O2 -I runtime -I$(brew --prefix openssl@3)/include \
-L$(brew --prefix openssl@3)/lib \
-o dist/platform/elc-new \
elc-new.c runtime/el_seed.c
# Verify self-hosting:
elc-new.c runtime/el_runtime.c runtime/el_seed.c \
runtime/engram_cognition.c runtime/engram_geometry.c runtime/engram_reason.c \
runtime/engram_store.c runtime/engram_verify.c runtime/engram_vindex.c \
runtime/eg_cosine_batch.c runtime/eg_cosine_batch_strategy_cpu.c \
-lcurl -lssl -lcrypto -lpthread -lm
# 3. Verify self-hosting FIXPOINT (stage3 == stage2 output, byte-identical):
./dist/platform/elc-new elc-cli.el > elc-verify.c
diff elc-new.c elc-verify.c # should be identical
diff elc-new.c elc-verify.c # must be identical
mv dist/platform/elc-new dist/platform/elc
```
> **Build-path caveat (verified 2026-08-15).** `el_seed.c` is the intended hand-maintained OS-boundary seed, but it does **not** compile standalone under modern clang: it wraps ~16 unprefixed `el_runtime.c` symbols (`http_serve`, `json_*`, `state_*`, `http_response`) without prototypes, and clang treats implicit declarations as errors (C99+). The productionised install (`tools/install.sh`) builds `libel.a` from **both** `el_seed.o` + `el_runtime.o` together, which is why linking succeeds there. To make `el_seed.c` build on its own, add prototypes for those symbols (or `#include "el_runtime.h"`, reconciling the `__http_serve` return-type mismatch first).
>
> **There is no single-file link target.** *(Corrected 2026-08-16 — this paragraph previously ended "`el_runtime.c` is the authoritative single-file link target for the compiler". Measured: that is false. Linking `elc-new.c` against `runtime/el_runtime.c` alone fails at `ld` with undefined `engram_ground_json`, `engram_activate_inner`, `eg_find_relation`, `cog_assert_two_axis`, and others, because `el_runtime.c` `#include`s six engram headers and calls into all six sibling `.c` files.)* Link the set in `runtime/SOURCES` via `$(../scripts/el-runtime-sources.sh runtime)`.
After changing `el_seed.c` only (no El source changes), rebuild downstream programs but do NOT need to rebuild the compiler binary itself — the seed is linked at the application level, not the compiler level.
---
lang/dist/platform/elc
Vendored
BIN
View File
Binary file not shown.
lang/el-compiler/src/codegen.el
+694 -594
View File
File diff suppressed because it is too large Load Diff
lang/el-compiler/src/compiler.el
+56
View File
@@ -414,11 +414,33 @@ fn parse_import_line(trimmed: String, dir: String) -> String {
// Accumulates chunks into lists and joins once at the end to avoid the O(n²)
// memory growth caused by repeated `prefix = prefix + chunk` concatenation.
fn resolve_imports(src_path: String) -> String {
// Only the OUTERMOST call publishes provenance. Nested calls number their
// lines from 1 within themselves, so their spans are meaningless once the
// text is spliced into the parent.
let depth: String = state_get("__elc_prov_depth")
if str_eq(depth, "") { state_set("__elc_prov_depth", "1") }
let is_top: Bool = str_eq(depth, "")
let seen_key: String = "__elc_imp__:" + src_path
let already: String = state_get(seen_key)
if !str_eq(already, "") { return "" }
state_set(seen_key, "1")
// A missing file must be a hard error, never an empty string.
//
// fs_read returns "" both for "file is empty" and "file does not exist", and
// this function used the value without distinguishing them. So a broken
// import path — a typo, a moved file, a relative path resolved from the
// wrong working directory — compiled CLEANLY: exit 0, empty stderr, and a
// program silently missing everything it imported. Observed 2026-08-15:
// eleven consecutive "successful" compiles that had included no runtime at
// all, and a wrong conclusion drawn from them before anyone noticed.
//
// Missing dependency, confident success. fs_exists separates the two cases,
// so a genuinely empty file still resolves to "" and is fine.
if !fs_exists(src_path) {
println("elc: cannot resolve import: " + src_path)
exit_program(1)
}
let source: String = fs_read(src_path)
let dir: String = dirname_of(src_path)
let lines: [String] = str_split(source, "\n")
@@ -427,6 +449,7 @@ fn resolve_imports(src_path: String) -> String {
// Collect chunks into lists — O(1) amortized per append.
// Join once at the end — O(n) single pass.
let prefix_chunks: [String] = native_list_empty()
let prefix_paths: [String] = native_list_empty()
let body_chunks: [String] = native_list_empty()
let i: Int = 0
while i < n {
@@ -438,21 +461,54 @@ fn resolve_imports(src_path: String) -> String {
// Only check .elh for imported files — never for the entry file itself.
let imp_elh_path: String = str_slice(imp_path, 0, str_len(imp_path) - 3) + ".elh"
let imp_elh: String = fs_read(imp_elh_path)
// Provenance: record which line range of the combined source came
// from which file, so a diagnostic can name the FILE and not just a
// line in a string that no longer exists on disk.
if !str_eq(imp_elh, "") {
// Header exists: mark the .el as seen (so it won't be re-inlined
// if something else also imports it) and use the header text.
let seen_imp_key: String = "__elc_imp__:" + imp_path
state_set(seen_imp_key, "1")
let prefix_chunks = native_list_append(prefix_chunks, imp_elh)
let prefix_paths = native_list_append(prefix_paths, imp_path)
} else {
let imp_body: String = resolve_imports(imp_path)
let prefix_chunks = native_list_append(prefix_chunks, imp_body)
let prefix_paths = native_list_append(prefix_paths, imp_path)
}
} else {
let body_chunks = native_list_append(body_chunks, line + "\n")
}
let i = i + 1
}
// Walk the assembled chunks once and publish <file> spans <start> <end>.
// LIMIT: nested imports return a single string, so their internal
// boundaries are already lost by the time we see them -- a definition
// inside a transitively imported file is attributed to the direct import.
// Local, not accumulated in state: a nested call numbers its lines from 1
// within itself, so letting it append to a shared buffer republishes
// meaningless spans under the parent's name.
let prov: String = ""
let line_at: Int = 1
let ci: Int = 0
let nchunks: Int = native_list_len(prefix_chunks)
while ci < nchunks {
let chunk: String = native_list_get(prefix_chunks, ci)
let nlines: Int = str_count_lines(chunk)
let src: String = native_list_get(prefix_paths, ci)
let prov = prov + src + " spans " + native_int_to_str(line_at) + " " + native_int_to_str(line_at + nlines - 1) + "\n"
let line_at = line_at + nlines
let ci = ci + 1
}
let prov = prov + src_path + " spans " + native_int_to_str(line_at) + " 999999\n"
if is_top {
let prov_out: String = env("EL_RELATIONS_OUT")
if !str_eq(prov_out, "") {
let existing: String = ""
if fs_exists(prov_out) { let existing = fs_read(prov_out) }
fs_write(prov_out, existing + prov)
}
}
return str_join(prefix_chunks, "") + str_join(body_chunks, "")
}
lang/el-compiler/src/lexer.el
+42 -7
View File
@@ -138,13 +138,38 @@ fn lex_is_whitespace(ch: String) -> Bool {
// tok_append — append a (kind, value) pair to a flat token list.
// Returns the updated list. Gamma combines flat-list + char-code for max savings.
// A token is (kind, value, line). The line comes from state rather than a
// parameter so the ~200 existing tok_append call sites are untouched -- the
// lexer advances __lex_line as it walks, and every token minted takes the line
// it was minted on.
//
// WHY AT ALL: before this a token carried no position, so no diagnostic in El
// could name a place. Every error named a symbol and never a line, and after
// textual inlining there was no way to say which FILE a definition came from.
fn tok_append(tokens: [Any], kind: String, value: String) -> [Any] {
let tokens = native_list_append(tokens, kind)
native_list_append(tokens, value)
let tokens = native_list_append(tokens, value)
native_list_append(tokens, state_get("__lex_line"))
}
// -- Keyword lookup ------------------------------------------------------------
// keyword_kind — the language's reserved spellings.
//
// A grammar is a BASIS: `fn` means function-start because someone said so, and
// nothing derives it. But unlike the other tables moved out this session, this
// one stays code, and the SHOULD gate is why. The keyword set is closed by the
// language definition -- it does not leak the way an allowlist does -- and the
// lexer runs before the program is understood, so a program can never declare
// its own keywords. Externalising it would cost file I/O on every compile and
// buy nothing.
//
// Removed 2026-08-17: sealed, activate, seed, protocol, impl. Reserved in the
// lexer, consumed by no parser or codegen path, and each one stole an
// identifier from users for nothing. `test` LOOKED inert by the same measure
// and is not -- codegen consumes it at 4135 for --test mode, 408 uses in the
// tree. The first measurement checked only parser.el and would have broken all
// of them.
fn keyword_kind(word: String) -> String {
if word == "let" { return "Let" }
if word == "fn" { return "Fn" }
@@ -161,14 +186,9 @@ fn keyword_kind(word: String) -> String {
if word == "from" { return "From" }
if word == "as" { return "As" }
if word == "with" { return "With" }
if word == "sealed" { return "Sealed" }
if word == "activate" { return "Activate" }
if word == "where" { return "Where" }
if word == "test" { return "Test" }
if word == "seed" { return "Seed" }
if word == "assert" { return "Assert" }
if word == "protocol" { return "Protocol" }
if word == "impl" { return "Impl" }
if word == "retry" { return "Retry" }
if word == "times" { return "Times" }
if word == "fallback" { return "Fallback" }
@@ -184,6 +204,7 @@ fn keyword_kind(word: String) -> String {
if word == "false" { return "Bool" }
if word == "cgi" { return "Cgi" }
if word == "service" { return "Service" }
if word == "program" { return "Program" }
if word == "manager" { return "Manager" }
if word == "engine" { return "Engine" }
if word == "accessor" { return "Accessor" }
@@ -520,6 +541,12 @@ fn scan_interp_brace(src: String, start: Int, total: Int) -> Map<String, Any> {
// interp_tokens_append_all - copy every (kind, value) pair from flat src list
// into flat dst list, skipping the trailing Eof pair that lex() always appends.
// Splices re-lexed interpolation tokens into the stream. This walks the token
// list DIRECTLY rather than through tok_append, so it carries its own copy of
// the stride -- which is why giving tokens a line broke the compiler's second
// generation and not its first: the compiler's own source uses string
// interpolation, so gen1 (built by the old compiler) was fine and gen2 emitted
// a corrupted stream.
fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] {
let src_len: Int = native_list_len(src)
let j = 0
@@ -530,9 +557,11 @@ fn interp_tokens_append_all(dst: [Any], src: [Any]) -> [Any] {
let j = src_len
} else {
let val: String = native_list_get(src, j + 1)
let ln: String = native_list_get(src, j + 2)
let result = native_list_append(result, kind)
let result = native_list_append(result, val)
let j = j + 2
let result = native_list_append(result, ln)
let j = j + 3
}
}
result
@@ -763,8 +792,14 @@ fn lex(source: String) -> [Any] {
let total: Int = str_len(source)
let tokens: [Any] = native_list_empty()
let i: Int = 0
state_set("__lex_line", "1")
let line_no: Int = 1
while i < total {
if str_eq(str_slice(source, i, i + 1), "\n") {
let line_no = line_no + 1
state_set("__lex_line", native_int_to_str(line_no))
}
let c: Int = str_char_code(source, i)
// Skip whitespace (space=32, tab=9, newline=10, CR=13)
lang/el-compiler/src/parser.el
+177 -14
View File
@@ -17,8 +17,8 @@
// programs. All callers use these helpers -- only these three need updating.
fn tok_at(tokens: [Any], pos: Int) -> Map<String, Any> {
let kind: String = native_list_get(tokens, pos * 2)
let value: String = native_list_get(tokens, pos * 2 + 1)
let kind: String = native_list_get(tokens, pos * 3)
let value: String = native_list_get(tokens, pos * 3 + 1)
{ "kind": kind, "value": value }
}
@@ -28,25 +28,32 @@ fn tok_kind(tokens: [Any], pos: Int) -> String {
// single trailing Eof token returns runtime null (el_list_get OOB -> 0),
// which matches no delimiter, letting inner parse loops append AST nodes
// forever on malformed input -> unbounded allocation -> OOM.
let n: Int = native_list_len(tokens) / 2
let n: Int = native_list_len(tokens) / 3
if pos < 0 {
return "Eof"
}
if pos >= n {
return "Eof"
}
native_list_get(tokens, pos * 2)
native_list_get(tokens, pos * 3)
}
fn tok_line(tokens: [Any], pos: Int) -> String {
let n: Int = native_list_len(tokens) / 3
if pos < 0 { return "0" }
if pos >= n { return "0" }
native_list_get(tokens, pos * 3 + 2)
}
fn tok_value(tokens: [Any], pos: Int) -> String {
let n: Int = native_list_len(tokens) / 2
let n: Int = native_list_len(tokens) / 3
if pos < 0 {
return ""
}
if pos >= n {
return ""
}
native_list_get(tokens, pos * 2 + 1)
native_list_get(tokens, pos * 3 + 1)
}
// parse_progress_fatal — robustness backstop. Called by the token-consuming
@@ -1230,7 +1237,7 @@ fn parse_block(tokens: [Any], pos: Int) -> Map<String, Any> {
// Runaway backstop: a block can hold at most (token count) statements, since
// every iteration consumes >= 1 token. If we exceed that, the cursor has run
// off the end without terminating (malformed input) -> fail fast, don't hang.
let blk_total: Int = native_list_len(tokens) / 2
let blk_total: Int = native_list_len(tokens) / 3
let blk_iters: Int = 0
while running {
let blk_iters = blk_iters + 1
@@ -1550,7 +1557,10 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map<String, Any> {
let p = r2["pos"]
// r2 result map fully consumed — release to free peak heap.
el_release(r2)
return make_result({ "stmt": "FnDef", "name": name, "params": params, "body": body, "ret_type": ret_type }, p)
// The definition carries the line it was written on. Without it no
// diagnostic can name a place, and after textual inlining there is no
// way to say which FILE a definition came from.
return make_result({ "stmt": "FnDef", "name": name, "params": params, "body": body, "ret_type": ret_type, "line": tok_line(tokens, pos) }, p)
}
// type definition: `type Name = { field: Type, ... }`
@@ -1842,6 +1852,7 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map<String, Any> {
"params": inner["params"],
"body": inner["body"],
"ret_type": inner["ret_type"],
"line": inner["line"],
"decorator": dec_name,
"decorators": dlist
}
@@ -1967,6 +1978,142 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map<String, Any> {
}, p)
}
// program block: program "name" { singleton: "id", env NAME: Type = "default", ... }
//
// The program block is El's declaration surface for CROSS-CUTTING CONCERNS —
// properties of the whole process rather than of any one function, which
// otherwise degrade into "remember to call this at every site" conventions.
//
// singleton: "id" — process identity. The runtime takes an exclusive
// lock at startup; a SECOND start is refused, loudly,
// instead of two processes sharing one data dir.
// guards: <expr> — WHAT that singleton protects: an expression yielding
// the path of the guarded state directory, evaluated at
// startup. MANDATORY with `singleton:`, because a lock
// keyed on a program's NAME rather than on its STATE is
// not a guard — measured 2026-08-16, the name-keyed
// version refused unrelated instances (different data
// dirs) AND permitted concurrent ones (same data dir,
// different $TMPDIR). It is an expression and not a
// string so a program can point at the resolver that
// already OWNS the path (§18.4) instead of restating
// its default here, which would give the path two
// owners that can disagree.
// env NAME: T = "d" — one configuration entry. Its type and its default
// are declared ONCE, here, and resolved+validated
// before main() body runs.
// env NAME: T required
// — no default; the program refuses to start unless the
// variable is set.
//
// Both compile into calls injected at the head of main() — the same boundary
// seam `cgi` already uses (codegen.el emit_program_init). No call site in the
// program body has to remember anything, which is the whole point.
if k == "Program" {
let p = pos + 1
let name = tok_value(tokens, p)
let p = p + 1
let p = expect(tokens, p, "LBrace")
let singleton = ""
let has_singleton = false
let guards_node = { "expr": "Str", "value": "" }
let has_guards = false
let entries = native_list_empty()
// Entry-scratch declared at loop-body level (not inside the branch) so
// that inner `let` forms compile to assignment rather than a C-scoped
// redeclaration — the same idiom the service block above relies on.
let ename = ""
let etype = ""
let edefault = ""
let has_default = false
let erequired = false
let fname = ""
let fval = ""
let running = true
while running {
let k2 = tok_kind(tokens, p)
if k2 == "RBrace" {
let running = false
} else {
if k2 == "Eof" {
let running = false
} else {
let fname = tok_value(tokens, p)
let p = p + 1
if str_eq(fname, "env") {
// env NAME: Type [= "default"] [required]
let ename = tok_value(tokens, p)
let p = p + 1
let p = expect(tokens, p, "Colon")
let etype = tok_value(tokens, p)
let p = p + 1
let edefault = ""
let has_default = false
let erequired = false
let k3 = tok_kind(tokens, p)
if str_eq(k3, "Eq") {
let p = p + 1
let edefault = tok_value(tokens, p)
let has_default = true
let p = p + 1
}
let k4 = tok_kind(tokens, p)
if str_eq(k4, "Ident") {
let w = tok_value(tokens, p)
if str_eq(w, "required") {
let erequired = true
let p = p + 1
}
}
let entries = native_list_append(entries, {
"name": ename,
"etype": etype,
"default": edefault,
"has_default": has_default,
"required": erequired
})
} else {
if str_eq(fname, "guards") {
// guards: <expr> — the STATE the singleton protects.
// Parsed as a full expression, not a string literal, so
// it can name the resolver that owns the path
// (`guards: engram_resolve_data_dir()`) rather than
// duplicating that resolver's default here.
let p = expect(tokens, p, "Colon")
let g_r = parse_expr(tokens, p)
let guards_node = g_r["node"]
let p = g_r["pos"]
let has_guards = true
} else {
// scalar field: `name: "value"`
let p = expect(tokens, p, "Colon")
let fval = tok_value(tokens, p)
let p = p + 1
if str_eq(fname, "singleton") {
let singleton = fval
let has_singleton = true
}
}
}
let k5 = tok_kind(tokens, p)
if k5 == "Comma" {
let p = p + 1
}
}
}
}
let p = expect(tokens, p, "RBrace")
return make_result({
"stmt": "ProgramBlock",
"name": name,
"singleton": singleton,
"has_singleton": has_singleton,
"guards": guards_node,
"has_guards": has_guards,
"entries": entries
}, p)
}
// assert <cond_expr> [ , <msg_expr> ]
// The message is optional — if the next token after the condition is not a
// Comma, emit an empty string placeholder so the test still works.
@@ -2022,7 +2169,7 @@ fn parse_stmt(tokens: [Any], pos: Int) -> Map<String, Any> {
fn parse(tokens: [Any]) -> [Map<String, Any>] {
// Flat list: 2 entries per token, so divide by 2 for token count.
let total: Int = native_list_len(tokens) / 2
let total: Int = native_list_len(tokens) / 3
let stmts: [Map<String, Any>] = native_list_empty()
let pos: Int = 0
let running = true
@@ -2065,7 +2212,7 @@ fn parse_one(tokens: [Any], pos: Int) -> Map<String, Any> {
// On entry, pos must point at the LBrace token.
// Returns the position of the token AFTER the matching RBrace.
fn skip_to_rbrace(tokens: [Any], pos: Int) -> Int {
let total: Int = native_list_len(tokens) / 2
let total: Int = native_list_len(tokens) / 3
let p: Int = pos + 1
let depth: Int = 1
let going: Bool = true
@@ -2117,7 +2264,7 @@ fn is_stmt_start_kind(k: String) -> Bool {
// token that could start a new top-level statement, staying depth-aware
// so that braces inside expressions don't fool us.
fn skip_expr_to_stmt_boundary(tokens: [Any], pos: Int) -> Int {
let total: Int = native_list_len(tokens) / 2
let total: Int = native_list_len(tokens) / 3
let p: Int = pos
let depth: Int = 0
let going: Bool = true
@@ -2283,7 +2430,7 @@ fn scan_params_el(tokens: [Any], pos: Int) -> Map<String, Any> {
//
// Peak memory: O(tokens) with no expression AST allocation.
fn scan_fn_sigs_el(tokens: [Any]) -> [Map<String, Any>] {
let total: Int = native_list_len(tokens) / 2
let total: Int = native_list_len(tokens) / 3
let sigs: [Map<String, Any>] = native_list_empty()
let pos: Int = 0
let going: Bool = true
@@ -2419,12 +2566,13 @@ fn scan_params_c(tokens: [Any], pos: Int) -> Map<String, Any> {
// toplevel_let: { "kind": "toplevel_let", "name": String, "ltype": String }
// cgi_block: { "kind": "cgi_block", "name": String }
// service_block: { "kind": "service_block", "name": String }
// program_block: { "kind": "program_block", "name": String }
//
// Import/TypeDef/EnumDef nodes are skipped (codegen treats them as no-ops).
//
// The scan allocates only small string values per entry, keeping peak RSS low.
fn scan_fn_sigs(tokens: [Any]) -> [Map<String, Any>] {
let total: Int = native_list_len(tokens) / 2
let total: Int = native_list_len(tokens) / 3
let sigs: [Map<String, Any>] = native_list_empty()
let pos: Int = 0
let going: Bool = true
@@ -2546,13 +2694,28 @@ fn scan_fn_sigs(tokens: [Any]) -> [Map<String, Any>] {
"name": name
})
let pos = p
} else {
// --- program block ---
if str_eq(k, "Program") {
let p: Int = pos + 1
let name: String = tok_value(tokens, p)
let p = p + 1
let k2: String = tok_kind(tokens, p)
if str_eq(k2, "LBrace") {
let p = skip_to_rbrace(tokens, p)
}
let sigs = native_list_append(sigs, {
"kind": "program_block",
"name": name
})
let pos = p
} else {
// Import, Type, Enum, From, or any other token.
// Skip ahead to the next statement boundary.
let p: Int = pos + 1
let p = skip_expr_to_stmt_boundary(tokens, p)
let pos = p
}}}}}
}}}}}}
}
}
}
lang/examples/transduce.el
+228
View File
@@ -0,0 +1,228 @@
// transduce.el — transduction decomposes a signal into components and the
// relations between them. Runnable: this is the worked example for the
// transduce surface, and it exits non-zero if any claim in it stops being true.
//
// elc lang/examples/transduce.el > transduce.c
// cc -std=c11 -O2 -I lang/runtime -o transduce transduce.c \
// lang/runtime/el_runtime.c lang/runtime/el_seed.c \
// lang/runtime/engram_store.c lang/runtime/engram_vindex.c \
// lang/runtime/engram_cognition.c lang/runtime/engram_geometry.c \
// lang/runtime/engram_reason.c lang/runtime/engram_verify.c \
// -lcurl -lpthread -lm
// ./transduce # exits 0 only if every check passes
//
// It writes to an IN-MEMORY engram (leave ENGRAM_STORE unset) and contacts no
// server. The same claims are asserted by the native harness in
// lang/tests/native/test_transduce.el.
//
// WHAT CHANGED, AND WHY IT MATTERS. #144 shipped
// `transduce(signal, modality) -> Geometry`: one vector per signal. That made
// transduction a CONVERSION — take a thing, encode it, store a position — and
// what a conversion returns is a fingerprint. A fingerprint can be matched and
// ranked, and that is all it can ever do. It cannot be decomposed, cannot have
// one part grounded while another is not, and cannot be contradicted in one
// part while holding in another, because it has no parts.
//
// A song is not a point. It decomposes into pitch, interval, rhythm, harmonic
// function — components, each with its own geometry, plus the relations among
// them. THE SONG IS THE STRUCTURE OF THE RELATIONS. transduce now returns a
// Manifold, and a realizer's job is to say what its modality's components ARE.
fn check(ok: Int, label: String) -> Int {
if ok > 0 {
println(" ok " + label)
return 0
}
println(" FAIL " + label)
exit(1)
return 1
}
fn near(a: Float, b: Float) -> Int {
let d: Float = a - b
if d > 0.001 { return 0 }
if d < -0.001 { return 0 }
return 1
}
fn eq_int(a: Int, b: Int) -> Int {
if a == b { return 1 }
return 0
}
// ── A DECOMPOSING realizer, written entirely in El ──────────────────────────
// "tone" signals are note letters, e.g. "CEG". This does NOT return one vector
// for the chord. It returns the PARTS — one component per note, one per
// interval between adjacent notes — and the relations that make those parts a
// chord rather than an unordered bag of pitches.
//
// The interval is deliberately a COMPONENT, not a field on a note. An interval
// is a thing with its own geometry belonging to neither endpoint; modelling it
// as an attribute of one of them is the same collapse, one level down.
fn tone_realizer(signal: String) -> Manifold {
let m: Manifold = manifold_new()
let n: Int = str_len(signal)
let i: Int = 0
while i < n {
let code: Int = str_char_code(signal, i)
let g: Geometry = geometry_new(2)
let s0: Int = geometry_set(g, 0, int_to_float(code))
let s1: Int = geometry_set(g, 1, int_to_float(i))
let idx: Int = manifold_add(m, "note:" + int_to_str(i), "pitch", g)
let f: Int = geometry_free(g)
i = i + 1
}
let j: Int = 1
while j < n {
let a: Int = str_char_code(signal, j - 1)
let b: Int = str_char_code(signal, j)
let lo: String = "note:" + int_to_str(j - 1)
let hi: String = "note:" + int_to_str(j)
let key: String = "interval:" + int_to_str(j - 1) + "-" + int_to_str(j)
let g: Geometry = geometry_new(1)
let s: Int = geometry_set(g, 0, int_to_float(b - a))
let idx: Int = manifold_add(m, key, "interval", g)
let f: Int = geometry_free(g)
let e1: Int = manifold_relate(m, key, "spans", lo, 0.9)
let e2: Int = manifold_relate(m, key, "spans", hi, 0.9)
let e3: Int = manifold_relate(m, lo, "sounds_before", hi, 0.8)
j = j + 1
}
m
}
// #144's contract, kept as a control: one vector for the whole signal.
fn fingerprint_realizer(signal: String) -> Geometry {
let g: Geometry = geometry_new(4)
let n: Int = str_len(signal)
let a: Int = geometry_set(g, 0, int_to_float(n))
g
}
fn main() -> Void {
println("a realizer declared in El is a first-class realizer")
let reg: Int = realizer_register("tone", "tone_realizer")
let _c: Int = check(reg, "an El fn registers as a realizer by name")
let _c: Int = check(realizer_has("tone"), "the modality now has an organ")
println("transduction decomposes a signal into parts")
let m: Manifold = transduce("CEG", "tone")
let _c: Int = check(manifold_is(m), "transduce returns a real Manifold")
let sz: Int = manifold_size(m)
let _c: Int = check(eq_int(sz, 5), "three notes and two intervals are five parts")
let rc: Int = manifold_rel_count(m)
let _c: Int = check(eq_int(rc, 6), "and they stand in six stated relations")
println("every part is addressable BY KEY, which is what survives persistence")
let i_c: Int = manifold_index_of(m, "note:0")
let _c: Int = check(1 - eq_int(i_c, -1), "the first note is addressable on its own")
let i_iv: Int = manifold_index_of(m, "interval:0-1")
let _c: Int = check(1 - eq_int(i_iv, -1), "so is the interval between the first two")
let miss: Int = manifold_index_of(m, "never_added")
let _c: Int = check(eq_int(miss, -1), "an unknown key is -1, not component 0")
println("parts carry their own geometry, and may differ in width")
let gn: Geometry = manifold_geometry(m, i_c)
let _c: Int = check(eq_int(geometry_dim(gn), 2), "a note component is 2 wide")
let _c: Int = check(near(geometry_get(gn, 0), 67.0), "and it is C — the signal reached the realizer")
let gi: Geometry = manifold_geometry(m, i_iv)
let _c: Int = check(eq_int(geometry_dim(gi), 1), "an interval component is 1 wide")
// A single vector per signal cannot represent parts of unequal width at all.
let _c: Int = check(near(geometry_get(gi, 0), 2.0), "C to E is two semitones")
let f1: Int = geometry_free(gn)
let f2: Int = geometry_free(gi)
println("the relations are content no single part carries")
// That "2" above is not a property of C and not a property of E. It exists
// only BETWEEN them, so a representation with no relations cannot hold it.
let spans: Int = 0
let k: Int = 0
while k < rc {
if str_eq(manifold_rel_name(m, k), "spans") {
if str_eq(manifold_rel_from(m, k), "interval:0-1") { spans = spans + 1 }
}
k = k + 1
}
let _c: Int = check(eq_int(spans, 2), "the interval is wired to both notes it spans")
println("relation weight IS the grounding (correspondence-and-censorship §1)")
let wk: Int = 0
let found: Int = 0
while wk < rc {
if str_eq(manifold_rel_name(m, wk), "sounds_before") {
if near(manifold_rel_weight(m, wk), 0.8) > 0 { found = 1 }
}
wk = wk + 1
}
let _c: Int = check(found, "the ordering relation carries the weight its realizer stated")
println("the decomposition persists as real, separately addressable nodes")
let ids: [String] = el_list_empty()
let n0: Int = engram_node_count()
let e0: Int = engram_edge_count()
let pi: Int = 0
while pi < sz {
let key: String = manifold_key(m, pi)
let g: Geometry = manifold_geometry(m, pi)
let id: String = engram_node("component " + key, "Concept", 0.6)
let att: Int = node_attach_geometry(id, g)
ids = el_list_append(ids, id)
let ff: Int = geometry_free(g)
pi = pi + 1
}
let ri: Int = 0
while ri < rc {
let fi: Int = manifold_index_of(m, manifold_rel_from(m, ri))
let ti: Int = manifold_index_of(m, manifold_rel_to(m, ri))
engram_connect(el_list_get(ids, fi), el_list_get(ids, ti),
manifold_rel_weight(m, ri), manifold_rel_name(m, ri))
ri = ri + 1
}
let _c: Int = check(eq_int(engram_node_count() - n0, 5), "one signal became five nodes")
let _c: Int = check(eq_int(engram_edge_count() - e0, 6), "and six edges between them")
println("each part's geometry is independently readable back off its node")
let id_c: String = el_list_get(ids, manifold_index_of(m, "note:0"))
let id_iv: String = el_list_get(ids, manifold_index_of(m, "interval:0-1"))
let _c: Int = check(eq_int(node_geometry_dim(id_c), 2), "note:0 node carries a 2-wide geometry")
let _c: Int = check(eq_int(node_geometry_dim(id_iv), 1), "interval:0-1 node carries a 1-wide one")
println("one part can be grounded without touching its siblings")
let ear: String = engram_node("evidence: heard a C in the recording", "Memory", 0.7)
engram_connect(ear, id_c, 0.95, "corroborates")
let _c: Int = check(engram_edge_between(ear, id_c), "evidence attaches to note:0 specifically")
let id_g: String = el_list_get(ids, manifold_index_of(m, "note:2"))
let _c: Int = check(1 - engram_edge_between(ear, id_g), "and NOT to note:2 — the sibling is untouched")
// This is the whole gain, and it is impossible with a fingerprint: with one
// node per signal, "the C is corroborated" and "the G is not" have the same
// grounding target and cannot both be recorded.
let _c: Int = check(eq_int(node_geometry_dim(id_g), 2), "note:2 geometry is intact regardless")
println("a fingerprint realizer transduces NOTHING")
// #144's contract exactly: signal in, one Geometry out. It resolves, so the
// organ is present — but it does not decompose, so it does not transduce.
// "No organ" and "an organ that only fingerprints" must not look alike.
let rf: Int = realizer_register("fingerprint", "fingerprint_realizer")
let _c: Int = check(rf, "the symbol resolves, so registration succeeds")
let mf: Manifold = transduce("x", "fingerprint")
let _c: Int = check(1 - manifold_is(mf), "a single vector is not a transduction")
println("the one-part case is a size-one manifold, not a bare vector")
let g1: Geometry = geometry_new(3)
let s1: Int = geometry_set(g1, 0, 5.0)
let ms: Manifold = manifold_single("level", "scalar", g1)
let _c: Int = check(manifold_is(ms), "manifold_single yields a real Manifold")
let _c: Int = check(eq_int(manifold_size(ms), 1), "of size one — visibly degenerate, not hidden")
let fg: Int = geometry_free(g1)
let fs: Int = manifold_free(ms)
println("no organ is still reported as no organ")
let me: Manifold = transduce("anything", "echolocation")
let _c: Int = check(1 - manifold_is(me), "no realizer means no manifold, not a fake one")
let fm: Int = manifold_free(m)
// Reaching here means nothing called exit(1) along the way.
println("")
println("all checks passed")
}
lang/install.sh
+41 -9
View File
@@ -49,21 +49,49 @@ download() {
TMP_DIR="$(mktemp -d)"
trap 'rm -rf "${TMP_DIR}"' EXIT
download "${RELEASE_BASE}/elc" "${TMP_DIR}/elc"
download "${RELEASE_BASE}/el_runtime.c" "${TMP_DIR}/el_runtime.c"
download "${RELEASE_BASE}/el_runtime.h" "${TMP_DIR}/el_runtime.h"
# The runtime is MULTI-FILE. el_runtime.c #includes six engram headers and makes
# hard cross-TU calls into all six sibling .c files, so installing el_runtime.c
# alone produces a lib/ that CANNOT LINK — `ld` fails with undefined
# engram_ground_json / engram_activate_inner / eg_find_relation / cog_assert_two_axis.
# This list mirrors lang/runtime/SOURCES (the in-repo source of truth); keep them
# in step. install.sh is standalone by design — it runs on machines with no repo
# checkout — so it cannot call scripts/el-runtime-sources.sh.
RUNTIME_SOURCES=(
el_runtime.c el_seed.c
engram_store.c engram_vindex.c engram_geometry.c
engram_reason.c engram_verify.c engram_cognition.c
engram_text.c
eg_cosine_batch.c eg_cosine_batch_strategy_cpu.c
)
RUNTIME_HEADERS=(
el_runtime.h el_seed.h
engram_store.h engram_vindex.h engram_geometry.h
engram_reason.h engram_verify.h engram_cognition.h
engram_text.h
eg_cosine_batch.h eg_cosine_batch_strategy.h
)
download "${RELEASE_BASE}/elc" "${TMP_DIR}/elc"
for f in "${RUNTIME_SOURCES[@]}" "${RUNTIME_HEADERS[@]}"; do
download "${RELEASE_BASE}/${f}" "${TMP_DIR}/${f}"
done
# Install
install -m 755 "${TMP_DIR}/elc" "${BIN_DIR}/elc"
install -m 644 "${TMP_DIR}/el_runtime.c" "${LIB_DIR}/el_runtime.c"
install -m 644 "${TMP_DIR}/el_runtime.h" "${LIB_DIR}/el_runtime.h"
install -m 755 "${TMP_DIR}/elc" "${BIN_DIR}/elc"
for f in "${RUNTIME_SOURCES[@]}" "${RUNTIME_HEADERS[@]}"; do
install -m 644 "${TMP_DIR}/${f}" "${LIB_DIR}/${f}"
done
# Record the link set so downstream Makefiles can read it instead of hardcoding.
printf '%s\n' "${RUNTIME_SOURCES[@]}" > "${TMP_DIR}/SOURCES"
install -m 644 "${TMP_DIR}/SOURCES" "${LIB_DIR}/SOURCES"
echo
echo "==> El SDK installed successfully"
echo
echo " elc binary : ${BIN_DIR}/elc"
echo " runtime : ${LIB_DIR}/el_runtime.c"
echo " header : ${LIB_DIR}/el_runtime.h"
echo " runtime : ${LIB_DIR}/ (${#RUNTIME_SOURCES[@]} .c files, ${#RUNTIME_HEADERS[@]} headers)"
echo " link set : ${LIB_DIR}/SOURCES"
echo
echo "Add the following to your Makefile to build El programs:"
echo
@@ -71,10 +99,14 @@ echo " EL_LIB := ${LIB_DIR}"
echo " ELC := elc"
echo " CC := cc"
echo " CFLAGS := -std=c11 -O2 -I\$(EL_LIB)"
echo " LDLIBS := -lcurl -lssl -lcrypto -lpthread -lm"
echo
echo " # The runtime is multi-file — link the whole set, not el_runtime.c alone."
echo " EL_RUNTIME := \$(addprefix \$(EL_LIB)/,\$(shell cat \$(EL_LIB)/SOURCES))"
echo
echo " dist/myapp.c: src/myapp.el"
echo " \t\$(ELC) src/myapp.el > dist/myapp.c"
echo
echo " dist/myapp: dist/myapp.c"
echo " \t\$(CC) \$(CFLAGS) -o dist/myapp dist/myapp.c \$(EL_LIB)/el_runtime.c -lcurl -lpthread"
echo " \t\$(CC) \$(CFLAGS) -o dist/myapp dist/myapp.c \$(EL_RUNTIME) \$(LDLIBS)"
echo
lang/runtime/BUDGET
+41
View File
@@ -0,0 +1,41 @@
# BUDGET — a RATCHET on lang/runtime/el_runtime.c. Enforced by
# scripts/check-runtime-growth.sh. These numbers may only ever go DOWN.
#
# WHY THIS FILE EXISTS
# --------------------
# scripts/check-single-runtime.sh guards against el_runtime.c being COPIED.
# Nothing guarded against it GROWING. It grew from 10,607 lines to 20,527 —
# 94% — in 3.5 months, while under an explicit commit-message promise that it
# was a temporary shim about to be deleted.
#
# It grew because lang/AGENTS.md told every agent to grow it: it claimed
# el_runtime.c was "the authoritative single-file link target" and that a new
# C builtin "must live there to be linkable". That is false — placement is a
# link-time concern, `builtin_arity` is an arity guard not a dispatch table,
# and the shipped elc already links from ten translation units. The claim is
# corrected, and this file is the mechanism that keeps it corrected.
#
# THIS IS A RATCHET, NOT A LIMIT
# ------------------------------
# The budget is set at the CURRENT size. There is no headroom, deliberately.
# The file cannot grow by even one line. Any new code goes in the .c that owns
# the concern — that is the whole point, and every other runtime file is
# deliberately UNCAPPED.
#
# When you move code OUT, lower the number in the same commit. The guard tells
# you to when you have earned it.
#
# FORMAT: <key> <value> — `#` comments and blank lines ignored.
# Maximum lines in lang/runtime/el_runtime.c.
# 2026-08-16: 20,527 — the high-water mark.
# 2026-08-16: 20,427 — engram_text.c extracted (tokenize, token hygiene,
# word-boundary match, damage signature). Ratcheted down.
max_lines 20427
# Maximum top-level engram/eg_/cog_ function definitions in el_runtime.c.
# ~47.5% of the file is engram code, and engram already owns six dedicated
# sibling files (engram_{store,vindex,geometry,reason,verify,cognition}.c).
# Every one of these belongs in one of them. This is the Stage 3 scoreboard.
# 2026-08-16: 279 -> 275 (4 moved to engram_text.c).
max_engram_fns 275
lang/runtime/SOURCES
+57
View File
@@ -0,0 +1,57 @@
# SOURCES — the canonical El runtime link set.
#
# THIS FILE IS THE SINGLE SOURCE OF TRUTH for "what do I compile and link to
# get the El runtime". Every build path — CI, install.sh, the SDK release, the
# docs, elb, the engram test harnesses — reads it via scripts/el-runtime-sources.sh
# instead of hardcoding its own list.
#
# WHY THIS FILE EXISTS
# --------------------
# The runtime has been multi-translation-unit since the engram siblings landed:
# el_runtime.c #includes engram_{store,vindex,geometry,reason,verify,cognition}.h
# and makes hard cross-TU calls into all six. Linking el_runtime.c ALONE has been
# broken since then — `ld` fails with undefined symbols (engram_ground_json,
# engram_activate_inner, eg_find_relation, cog_assert_two_axis, ...).
#
# It stayed broken because the link set was written out longhand in ~8 different
# places, each of which drifted independently. A list copied 8 times is a list
# that is wrong in 8 places. It is now written once, here.
#
# HOW TO USE IT
# -------------
# scripts/el-runtime-sources.sh # bare names, one per line
# scripts/el-runtime-sources.sh lang/runtime # prefixed with a directory
# cc ... $(scripts/el-runtime-sources.sh lang/runtime) -lcurl -lssl -lcrypto -lpthread -lm
#
# ADDING A FILE
# -------------
# Add the .c here and it is picked up by every build path at once. That is the
# point: a new concern gets its own translation unit and costs one line, instead
# of being appended to el_runtime.c because appending was the cheaper edit.
#
# Order is link order. Blank lines and `#` comments are ignored.
# --- EL core language runtime -------------------------------------------------
el_runtime.c
el_seed.c
# --- Engram: store, index, geometry, reasoning, verification, cognition -------
# These are the six concern-owned translation units el_runtime.c calls into.
engram_store.c
engram_vindex.c
engram_geometry.c
engram_reason.c
engram_verify.c
engram_cognition.c
# --- Text: tokenization, token hygiene, damage signature ---------------------
# Extracted from el_runtime.c 2026-08-16. Plain C over <ctype.h>/<string.h> —
# touches no EL value type and no engram store type. New text helpers go HERE.
engram_text.c
# --- Vector math: batch cosine + its CPU strategy ----------------------------
# The ggml strategy (eg_cosine_batch_strategy_ggml.c) is an OPTIONAL swap-in and
# is deliberately NOT in the default set — it needs ggml headers. Link it in
# place of the cpu strategy when you have them.
eg_cosine_batch.c
eg_cosine_batch_strategy_cpu.c
lang/runtime/build_vindex_bench.sh
Executable
+51
View File
@@ -0,0 +1,51 @@
#!/bin/bash
# build_vindex_bench.sh — build the vindex_bench oracle/proof harness, with
# the real ggml + hand-rolled-Metal batch-cosine strategies on Darwin and a
# zero-dependency CPU-only stub everywhere else. Mirrors the two-step recipe
# documented in vindex_bench.c's own header comment; this script exists so
# that recipe is one command, not a copy-pasted paragraph.
#
# Darwin build links FOUR strategy translation units:
# eg_cosine_batch.c — the Factory (always)
# eg_cosine_batch_strategy_cpu.c — universal fallback (always)
# eg_cosine_batch_strategy_ggml.c — ggml + dynamic Metal backend plugin
# eg_cosine_batch_strategy_metal_hand.m — PR #114's original hand-rolled
# Metal shader, preserved as one
# selectable strategy
# plus -DEG_HAVE_STRATEGY_GGML -DEG_HAVE_STRATEGY_METAL_HAND so the Factory
# (and vindex_bench.c's own direct strategy comparison) knows both exist.
#
# ggml is resolved via `brew --prefix ggml` when available (portable across
# Intel /usr/local and Apple Silicon /opt/homebrew installs), falling back to
# /opt/homebrew if brew isn't on PATH. Override with GGML_PREFIX=... env var.
#
# Usage: ./build_vindex_bench.sh [output_path]
set -euo pipefail
cd "$(dirname "$0")"
OUT="${1:-./vindex_bench}"
CC="${CC:-cc}"
if [ "$(uname -s)" = "Darwin" ]; then
GGML_PREFIX="${GGML_PREFIX:-$(brew --prefix ggml 2>/dev/null || echo /opt/homebrew)}"
echo "== Darwin: building with the ggml + hand-rolled-Metal strategies (ggml prefix: $GGML_PREFIX) =="
"$CC" -O2 -std=c11 -x objective-c \
-c eg_cosine_batch_strategy_metal_hand.m -o /tmp/eg_cosine_batch_strategy_metal_hand.o \
-framework Metal -framework Foundation
"$CC" -O2 -std=c11 -DEG_HAVE_STRATEGY_GGML -DEG_HAVE_STRATEGY_METAL_HAND -w \
-I"$GGML_PREFIX/include" \
vindex_bench.c engram_vindex.c \
eg_cosine_batch.c eg_cosine_batch_strategy_cpu.c eg_cosine_batch_strategy_ggml.c \
/tmp/eg_cosine_batch_strategy_metal_hand.o \
-L"$GGML_PREFIX/lib" -lggml -lggml-base \
-Wl,-rpath,"$GGML_PREFIX/lib" \
-lm -framework Metal -framework Foundation -o "$OUT"
else
echo "== non-Darwin: building with the CPU-only fallback strategy (no ggml, no Metal) =="
"$CC" -O2 -std=c11 -w vindex_bench.c engram_vindex.c \
eg_cosine_batch.c eg_cosine_batch_strategy_cpu.c \
-lm -o "$OUT"
fi
echo "built: $OUT"
lang/runtime/eg_cosine_batch.c
+121
View File
@@ -0,0 +1,121 @@
/* eg_cosine_batch.c — the Factory. Implements the stable public interface
* declared in eg_cosine_batch.h by selecting ONE concrete
* EgCosineBatchStrategy (eg_cosine_batch_strategy.h) and dispatching every
* call to it. This is the ONLY file that branches on EG_HAVE_STRATEGY_*
* (build-time: which strategy .c/.m files were actually compiled in for
* this platform) — call sites never see those macros.
*
* Selection is lazy (first call) and cached — mirrors the lazy-init caching
* every individual strategy already does internally, so there is no added
* per-call cost after the first.
*
* Selection mechanism (env var + build-time + runtime capability probe, all
* three, exactly as directed):
* - BUILD-TIME decides which strategies exist to choose from at all: a
* Darwin build compiles+links the ggml strategy and the hand-rolled
* Metal strategy (EG_HAVE_STRATEGY_GGML / EG_HAVE_STRATEGY_METAL_HAND
* both defined); a non-Darwin build compiles neither, matching PR #114's
* original Linux behavior exactly (CPU-fallback only, no Objective-C
* compiler or Metal frameworks required).
* - RUNTIME CAPABILITY PROBE: each candidate strategy's own available()
* does the real, cheap-after-first-call check (device present, backend
* plugin loaded, pipeline compiled) — never assumed from build-time
* alone. A build that HAS the ggml strategy compiled in but is running
* on hardware/software where it can't actually initialize (backend
* plugin missing, no GPU) correctly falls through to the next candidate.
* - ENV VAR gives explicit, debuggable override for either axis:
* EL_COSINE_BATCH_STRATEGY = "ggml" | "metal" | "cpu" | unset/"auto"
* forces a specific strategy (falling back to cpu if the forced one
* isn't actually available), or leaves the default auto-preference
* order in place.
* EL_METAL_COSINE = 0/n/N/f/F (back-compat with PR #114's vindex_bench
* gate) disables ALL GPU-backed strategies outright, same as before.
*
* DEFAULT preference order when nothing is forced: ggml, then hand-rolled
* Metal, then CPU fallback — first candidate whose available() reports true
* wins. This is what makes "stop hand-rolling GPU kernels, use ggml" real
* rather than nominal: ggml is what actually runs by default on this
* machine today (see the PR body for the measured numbers backing that).
*/
#include "eg_cosine_batch.h"
#include "eg_cosine_batch_strategy.h"
#include <stdlib.h>
#include <string.h>
static bool g_selected = false;
static const EgCosineBatchStrategy* g_active = NULL;
static bool eg_env_truthy_off(const char* v) {
return v && (v[0]=='0' || v[0]=='n' || v[0]=='N' || v[0]=='f' || v[0]=='F');
}
static const EgCosineBatchStrategy* eg_select_strategy(void) {
if (g_selected) return g_active;
g_selected = true;
const EgCosineBatchStrategy* cpu = eg_cosine_batch_strategy_cpu();
const char* force = getenv("EL_COSINE_BATCH_STRATEGY");
const char* legacy_off = getenv("EL_METAL_COSINE");
if (eg_env_truthy_off(legacy_off)) { g_active = cpu; return g_active; }
if (force && strcmp(force, "cpu") == 0) { g_active = cpu; return g_active; }
if (force && strcmp(force, "ggml") == 0) {
#ifdef EG_HAVE_STRATEGY_GGML
const EgCosineBatchStrategy* s = eg_cosine_batch_strategy_ggml();
if (s->available()) { g_active = s; return g_active; }
#endif
g_active = cpu; return g_active;
}
if (force && strcmp(force, "metal") == 0) {
#ifdef EG_HAVE_STRATEGY_METAL_HAND
const EgCosineBatchStrategy* s = eg_cosine_batch_strategy_metal_hand();
if (s->available()) { g_active = s; return g_active; }
#endif
g_active = cpu; return g_active;
}
/* auto (unset, or any other value): ggml -> metal-hand -> cpu, first
* available wins. */
#ifdef EG_HAVE_STRATEGY_GGML
{
const EgCosineBatchStrategy* s = eg_cosine_batch_strategy_ggml();
if (s->available()) { g_active = s; return g_active; }
}
#endif
#ifdef EG_HAVE_STRATEGY_METAL_HAND
{
const EgCosineBatchStrategy* s = eg_cosine_batch_strategy_metal_hand();
if (s->available()) { g_active = s; return g_active; }
}
#endif
g_active = cpu;
return g_active;
}
bool eg_cosine_batch_available(void) {
return eg_select_strategy()->available();
}
const char* eg_cosine_batch_strategy_name(void) {
return eg_select_strategy()->name;
}
bool eg_cosine_batch(const float* query, int32_t qdim,
const float* const* node_ptrs,
const int32_t* node_dims,
int32_t n,
double* out_scores) {
return eg_select_strategy()->batch(query, qdim, node_ptrs, node_dims, n, out_scores);
}
bool eg_cosine_batch_multi(const float* queries, int32_t qdim, int32_t nq,
const float* const* node_ptrs,
const int32_t* node_dims,
int32_t n,
double* out_scores) {
return eg_select_strategy()->batch_multi(queries, qdim, nq, node_ptrs, node_dims, n, out_scores);
}
lang/runtime/eg_cosine_batch.h
+106
View File
@@ -0,0 +1,106 @@
/* eg_cosine_batch.h — stable Adapter interface over batch-cosine-similarity
* BACKEND STRATEGIES. This header is the ONE thing call sites (el_runtime.c,
* vindex_bench.c, ...) talk to. Plain C11, safe to #include on every
* platform — the symbols declared here always exist and always link,
* regardless of what backend actually runs underneath. Zero #ifdef at call
* sites: which concrete strategy executes (ggml/Metal, hand-rolled Metal, or
* the always-false CPU fallback) is resolved once, lazily, inside
* eg_cosine_batch.c's factory — see eg_cosine_batch_strategy.h for that.
*
* This supersedes eg_metal_cosine.h (PR #114's single hand-rolled-Metal-only
* bridge). The contract is UNCHANGED — same shapes, same sentinel, same
* never-partial guarantee, same "caller must always be prepared to fall back
* to its own scalar per-node loop" rule — only the name changed, because the
* thing behind it is no longer "the Metal bridge," it is "whichever batch-
* cosine strategy the factory picked." eg_metal_cosine.h's original doc
* comments (byte-for-byte, this file is the direct descendant) are preserved
* below since they remain the precise spec any strategy must honor.
*
* On ANY failure at ANY step — no compute device, compile/init error, alloc
* failure, bad args — every function here returns false and writes nothing.
* Out-params are either fully populated or left completely untouched, never
* partial. Callers MUST always be prepared to fall back to their own scalar
* per-node CPU loop unconditionally. These functions must never crash, throw,
* or hang the calling process — several call sites run inside a long-lived
* daemon's request-handling hot path.
*/
#ifndef EG_COSINE_BATCH_H
#define EG_COSINE_BATCH_H
#include <stdint.h>
#include <stdbool.h>
#ifdef __cplusplus
extern "C" {
#endif
/* Batched cosine similarity: one query vector against `n` node vectors.
*
* query — qdim floats, the query embedding. Raw/unnormalized.
* qdim — query dimensionality (e.g. 768 for nomic-embed-text).
* node_ptrs — array of n pointers, node_ptrs[i] pointing at a (possibly
* differently-owned, possibly NULL) float vector for node i.
* NOT required to be contiguous — every strategy performs the
* gather into a packed row-major matrix internally, exactly
* mirroring how EngramNode.emb is one malloc per node.
* node_dims — array of n ints, node_dims[i] = that node's real emb_dim
* (0 or mismatched vs qdim => that node scores -2.0, matching
* eg_cosine's null/dim-mismatch/zero-norm sentinel exactly).
* n — number of nodes.
* out_scores — caller-owned array of n doubles; out_scores[i] is filled
* with the cosine similarity of node i against query, or
* -2.0 for a null/dim-mismatched/zero-norm node — bit-for-bit
* the same contract as eg_cosine(node_ptrs[i], query, qdim).
*
* Returns true iff a real backend strategy ran and out_scores was fully
* populated. Returns false (out_scores left untouched) on ANY failure or
* unavailability — no compute device, compile/init failure, allocation
* failure, n<=0, qdim<=0, null query/node_ptrs/node_dims/out_scores.
*/
bool eg_cosine_batch(const float* query, int32_t qdim,
const float* const* node_ptrs,
const int32_t* node_dims,
int32_t n,
double* out_scores);
/* True iff a real (non-CPU-fallback) strategy is available right now (cheap
* after the first call — cached). Purely informational (e.g. a startup log
* line or /api/stats field); callers should still treat a false return from
* eg_cosine_batch()/eg_cosine_batch_multi() itself as the authoritative
* fallback signal, not this function. */
bool eg_cosine_batch_available(void);
/* Which concrete strategy is currently selected — "ggml", "metal-hand",
* or "cpu-fallback". Purely informational/diagnostic, same spirit as
* eg_cosine_batch_available(). Never NULL. */
const char* eg_cosine_batch_strategy_name(void);
/* Multi-query batched cosine: nq query vectors against the SAME n node
* vectors, in one call. A real strategy uploads/prepares the node population
* ONCE and reuses it for every query, instead of nq separate
* eg_cosine_batch() calls each paying the full gather+upload cost — PR #114
* measured this necessary: at N~=13.7k/dim=768, repeating the single-query
* call per query was slower than the CPU baseline; batching queries together
* is what makes a GPU-backed path a real win at this shape. Use this
* whenever multiple queries will run against an unchanged (or
* rarely-changing) node population; use eg_cosine_batch() for a genuinely
* one-off comparison.
*
* queries — nq*qdim floats, row-major (query i at queries+i*qdim).
* out_scores — caller-owned nq*n doubles, row-major
* (out_scores[i*n+j] = cosine(queries[i], node j)), same
* -2.0 sentinel semantics as eg_cosine_batch().
*
* Returns true iff a real strategy ran and out_scores was fully populated
* (all nq*n entries); false (untouched) on any failure/unavailability. */
bool eg_cosine_batch_multi(const float* queries, int32_t qdim, int32_t nq,
const float* const* node_ptrs,
const int32_t* node_dims,
int32_t n,
double* out_scores);
#ifdef __cplusplus
}
#endif
#endif /* EG_COSINE_BATCH_H */
lang/runtime/eg_cosine_batch.metal
+156
View File
@@ -0,0 +1,156 @@
/* eg_cosine_batch.metal — batched cosine similarity, one query vs N node vectors.
*
* GPU-shaped counterpart to eg_cosine() in el_runtime.c: same math, same
* dim-mismatch/zero-norm sentinel (-2.0), applied to N independent rows in
* parallel instead of one pair at a time in a CPU loop.
*
* Semantics MUST match eg_cosine() exactly:
* - inputs are raw, UNNORMALIZED vectors (nomic-embed-text magnitudes are
* not 1.0) — this kernel computes the full dot/(|a|*|b|) cosine, not a
* plain dot product.
* - a node whose declared dim differs from the query dim, or whose norm is
* zero, scores exactly -2.0 (below any valid cosine in [-1,1]), so a
* caller doing `if (score > threshold)` behaves identically whether the
* scalar or the batched path filled the array.
*
* Precision: Apple GPUs do not support double in Metal Shading Language —
* everything here is float32. eg_cosine accumulates in CPU double, but its
* *inputs* are float32 embeddings, so the achievable precision ceiling is
* bounded by the input data regardless of accumulator width. To keep the
* float32 reduction from drifting relative to the double-accumulated CPU
* result across dim=768 terms, each thread accumulates with 4 independent
* partial sums (unrolled) rather than one running scalar — the same
* error-reduction trick already used by the CPU brute-force loop in
* vindex_bench.c. The measured float-vs-double delta is reported in the PR
* description; this is not assumed to be "close enough" without measurement.
*/
#include <metal_stdlib>
using namespace metal;
/* Per-dispatch invariants. `dim` is the query's dimensionality — the
* dimensionality every comparable node vector must match. */
struct EgCosineParams {
uint n; /* number of node rows */
uint dim; /* vector width (both query and node rows are `dim` wide in
* the packed buffer; node_dims[] carries each node's REAL
* embedded dim for the mismatch check) */
};
/* One thread per node row. node_matrix is n*dim floats, row-major, packed at
* `dim` stride regardless of a row's real dim (the CPU side zero-pads or
* skips packing rows that don't match — see eg_cosine_batch_metal in
* eg_metal_cosine.m for the exact packing contract). node_dims[i] is the
* node's true emb_dim, used only for the mismatch sentinel — never used to
* index, since every row is packed at uniform `dim` stride. */
kernel void eg_cosine_batch_kernel(
device const float* query [[buffer(0)]],
device const float* node_matrix [[buffer(1)]],
device const int* node_dims [[buffer(2)]],
constant EgCosineParams& p [[buffer(3)]],
device float* out_scores [[buffer(4)]],
uint gid [[thread_position_in_grid]])
{
if (gid >= p.n) return;
if (node_dims[gid] != int(p.dim)) {
out_scores[gid] = -2.0f;
return;
}
device const float* row = node_matrix + (uint64_t)gid * (uint64_t)p.dim;
/* 4-way partial accumulation — same shape as vindex_bench.c's brute_topk
* unroll, done here for float32 accuracy rather than raw throughput. */
float dot0 = 0.0f, dot1 = 0.0f, dot2 = 0.0f, dot3 = 0.0f;
float na0 = 0.0f, na1 = 0.0f, na2 = 0.0f, na3 = 0.0f;
float nb0 = 0.0f, nb1 = 0.0f, nb2 = 0.0f, nb3 = 0.0f;
uint d = 0;
uint dim4 = p.dim & ~3u;
for (; d < dim4; d += 4) {
float a0 = row[d], b0 = query[d];
float a1 = row[d+1], b1 = query[d+1];
float a2 = row[d+2], b2 = query[d+2];
float a3 = row[d+3], b3 = query[d+3];
dot0 += a0*b0; dot1 += a1*b1; dot2 += a2*b2; dot3 += a3*b3;
na0 += a0*a0; na1 += a1*a1; na2 += a2*a2; na3 += a3*a3;
nb0 += b0*b0; nb1 += b1*b1; nb2 += b2*b2; nb3 += b3*b3;
}
float dot = (dot0 + dot1) + (dot2 + dot3);
float na = (na0 + na1) + (na2 + na3);
float nb = (nb0 + nb1) + (nb2 + nb3);
for (; d < p.dim; d++) {
float a = row[d], b = query[d];
dot += a*b; na += a*a; nb += b*b;
}
if (na <= 0.0f || nb <= 0.0f) {
out_scores[gid] = -2.0f;
return;
}
out_scores[gid] = dot / sqrt(na * nb);
}
/* ── multi-query variant ──────────────────────────────────────────────────
* Same per-pair math as eg_cosine_batch_kernel, but amortizes ONE upload of
* node_matrix (the expensive part at real store size — 13k*768 floats is
* ~42MB) across `nq` queries instead of re-uploading it once per query.
* Measured need: a naive one-query-at-a-time loop calling the single-query
* kernel nq times was SLOWER than the CPU oracle at N≈13.7k (re-gather +
* re-upload dominated the actual compute) — this is the fix, not a
* hypothetical optimization.
*
* 2D grid: x = node index [0,n), y = query index [0,nq). out_scores is
* nq*n, row-major by query (out_scores[qid*n + nid]). */
struct EgCosineMultiParams { uint n; uint dim; uint nq; };
kernel void eg_cosine_batch_multi_kernel(
device const float* queries [[buffer(0)]], /* nq*dim */
device const float* node_matrix [[buffer(1)]], /* n*dim */
device const int* node_dims [[buffer(2)]], /* n */
constant EgCosineMultiParams& p [[buffer(3)]],
device float* out_scores [[buffer(4)]], /* nq*n */
uint2 gid [[thread_position_in_grid]])
{
uint nid = gid.x, qid = gid.y;
if (nid >= p.n || qid >= p.nq) return;
uint64_t out_idx = (uint64_t)qid * (uint64_t)p.n + (uint64_t)nid;
if (node_dims[nid] != int(p.dim)) {
out_scores[out_idx] = -2.0f;
return;
}
device const float* row = node_matrix + (uint64_t)nid * (uint64_t)p.dim;
device const float* query = queries + (uint64_t)qid * (uint64_t)p.dim;
float dot0 = 0.0f, dot1 = 0.0f, dot2 = 0.0f, dot3 = 0.0f;
float na0 = 0.0f, na1 = 0.0f, na2 = 0.0f, na3 = 0.0f;
float nb0 = 0.0f, nb1 = 0.0f, nb2 = 0.0f, nb3 = 0.0f;
uint d = 0;
uint dim4 = p.dim & ~3u;
for (; d < dim4; d += 4) {
float a0 = row[d], b0 = query[d];
float a1 = row[d+1], b1 = query[d+1];
float a2 = row[d+2], b2 = query[d+2];
float a3 = row[d+3], b3 = query[d+3];
dot0 += a0*b0; dot1 += a1*b1; dot2 += a2*b2; dot3 += a3*b3;
na0 += a0*a0; na1 += a1*a1; na2 += a2*a2; na3 += a3*a3;
nb0 += b0*b0; nb1 += b1*b1; nb2 += b2*b2; nb3 += b3*b3;
}
float dot = (dot0 + dot1) + (dot2 + dot3);
float na = (na0 + na1) + (na2 + na3);
float nb = (nb0 + nb1) + (nb2 + nb3);
for (; d < p.dim; d++) {
float a = row[d], b = query[d];
dot += a*b; na += a*a; nb += b*b;
}
if (na <= 0.0f || nb <= 0.0f) {
out_scores[out_idx] = -2.0f;
return;
}
out_scores[out_idx] = dot / sqrt(na * nb);
}
lang/runtime/eg_cosine_batch_strategy.h
+82
View File
@@ -0,0 +1,82 @@
/* eg_cosine_batch_strategy.h — internal Strategy interface, NOT for call
* sites (they use eg_cosine_batch.h). Only eg_cosine_batch.c's factory and
* the concrete strategy implementation files include this.
*
* Each concrete strategy exposes exactly one getter returning a pointer to a
* static, immutable EgCosineBatchStrategy vtable. Which getters actually
* exist as linkable symbols is a BUILD-TIME concern (decided by
* build_vindex_bench.sh / the engram daemon's own build, via which .c/.m
* files get compiled per platform) gated by the EG_HAVE_STRATEGY_* macros
* below — the factory in eg_cosine_batch.c is the ONLY place that branches
* on those macros. Call sites never see them; that's the whole point of the
* Adapter in eg_cosine_batch.h.
*
* Three concrete strategies exist:
* eg_cosine_batch_strategy_ggml() — ggml + dynamically-loaded Metal
* backend plugin. Darwin only in
* this build; the default
* preferred strategy wherever
* available. EG_HAVE_STRATEGY_GGML.
* eg_cosine_batch_strategy_metal_hand() — the original hand-rolled Metal
* compute shader from PR #114
* (eg_cosine_batch.metal),
* preserved verbatim as a
* selectable fallback strategy,
* not deleted. Darwin only.
* EG_HAVE_STRATEGY_METAL_HAND.
* eg_cosine_batch_strategy_cpu() — universal always-false
* fallback. Always compiled, on
* every platform; this is what a
* non-Darwin build links
* exclusively (matching PR #114's
* eg_metal_cosine_stub.c), and
* what any platform falls back
* to when no real strategy is
* available at runtime.
*/
#ifndef EG_COSINE_BATCH_STRATEGY_H
#define EG_COSINE_BATCH_STRATEGY_H
#include <stdint.h>
#include <stdbool.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef struct EgCosineBatchStrategy {
/* Stable, short, lowercase-hyphenated identifier — what
* eg_cosine_batch_strategy_name() surfaces. Never NULL. */
const char* name;
/* Cheap after the first call (lazy init, cached internally). Must never
* throw/crash/hang — mirrors eg_cosine_batch_available()'s contract. */
bool (*available)(void);
/* Same shape/contract as eg_cosine_batch() in eg_cosine_batch.h. */
bool (*batch)(const float* query, int32_t qdim,
const float* const* node_ptrs, const int32_t* node_dims,
int32_t n, double* out_scores);
/* Same shape/contract as eg_cosine_batch_multi() in eg_cosine_batch.h. */
bool (*batch_multi)(const float* queries, int32_t qdim, int32_t nq,
const float* const* node_ptrs, const int32_t* node_dims,
int32_t n, double* out_scores);
} EgCosineBatchStrategy;
#ifdef EG_HAVE_STRATEGY_GGML
const EgCosineBatchStrategy* eg_cosine_batch_strategy_ggml(void);
#endif
#ifdef EG_HAVE_STRATEGY_METAL_HAND
const EgCosineBatchStrategy* eg_cosine_batch_strategy_metal_hand(void);
#endif
/* Always declared/linked, on every platform/build. */
const EgCosineBatchStrategy* eg_cosine_batch_strategy_cpu(void);
#ifdef __cplusplus
}
#endif
#endif /* EG_COSINE_BATCH_STRATEGY_H */
lang/runtime/eg_cosine_batch_strategy_cpu.c
+45
View File
@@ -0,0 +1,45 @@
/* eg_cosine_batch_strategy_cpu.c — plain-C, zero-dependency universal
* fallback strategy. Always returns false / unavailable. Direct descendant
* of PR #114's eg_metal_cosine_stub.c, generalized from "the Metal stub" to
* "the strategy vtable's universal fallback entry" now that multiple real
* strategies can exist.
*
* Always compiled, on every platform. On Darwin builds it is the last-resort
* strategy the factory falls back to when neither ggml nor the hand-rolled
* Metal strategy is available at runtime (no device, compile failure, ...).
* On non-Darwin builds it is the ONLY strategy compiled in at all — no
* Objective-C, no Metal frameworks, no ggml/Metal backend plugin — so
* eg_cosine_batch()/eg_cosine_batch_multi() always return false there and
* every call site's existing CPU fallback runs unconditionally, exactly as
* before this PR.
*/
#include "eg_cosine_batch_strategy.h"
static bool cpu_available(void) {
return false;
}
static bool cpu_batch(const float* query, int32_t qdim,
const float* const* node_ptrs, const int32_t* node_dims,
int32_t n, double* out_scores) {
(void)query; (void)qdim; (void)node_ptrs; (void)node_dims; (void)n; (void)out_scores;
return false;
}
static bool cpu_batch_multi(const float* queries, int32_t qdim, int32_t nq,
const float* const* node_ptrs, const int32_t* node_dims,
int32_t n, double* out_scores) {
(void)queries; (void)qdim; (void)nq; (void)node_ptrs; (void)node_dims; (void)n; (void)out_scores;
return false;
}
static const EgCosineBatchStrategy g_cpu_strategy = {
.name = "cpu-fallback",
.available = cpu_available,
.batch = cpu_batch,
.batch_multi = cpu_batch_multi,
};
const EgCosineBatchStrategy* eg_cosine_batch_strategy_cpu(void) {
return &g_cpu_strategy;
}
lang/runtime/eg_cosine_batch_strategy_ggml.c
+515
View File
@@ -0,0 +1,515 @@
/* eg_cosine_batch_strategy_ggml.c — the GGML Strategy, and the preferred
* default whenever it is available (see the factory's selection order in
* eg_cosine_batch.c).
*
* WHY: directive from Will Anderson — stop hand-rolling GPU kernels, use a
* real, proven, permissively-licensed library instead. ggml (the compute
* library underneath llama.cpp, MIT licensed) is already installed on this
* machine as a standalone Homebrew package (`brew info ggml`), independent
* of llama.cpp itself. This file is a genuinely bounded COMPUTE UTILITY —
* batch cosine-similarity math — analogous to a VBD Accessor calling out to
* infrastructure. It is explicitly NOT the engram's reasoning/persistence
* core; using ggml here does not cross the "own the core" line, because
* batch cosine math is infrastructure, not the graph traversal / activation
* spreading / "thinking" that IS the core and stays 100% own-code.
*
* ── The real API shape (verified against the installed headers + a
* standalone probe program, not assumed from memory of other tensor
* libraries) ──────────────────────────────────────────────────────────
*
* ggml ships its CPU and Metal implementations as DYNAMICALLY LOADED PLUGIN
* .so files (confirmed by nm: `ggml_backend_metal_init` is NOT an exported
* symbol of libggml.dylib/libggml-base.dylib — it exists ONLY inside
* libggml-metal.so under $(brew --prefix ggml)/libexec/). You cannot link
* `-lggml-metal`; you must go through ggml's backend REGISTRY:
*
* 1. ggml_backend_load_all_from_path(dir) — dlopen()s every backend plugin
* .so found in `dir` and registers its device(s). We point this at
* $(brew --prefix ggml)/libexec (resolved once, at build+init time; see
* eg_ggml_backend_dir() below) rather than relying on
* ggml_backend_load_all()'s own default search heuristics, which are
* tuned for an installed llama.cpp-style app bundle layout, not an
* arbitrary `cc`-built binary invoked from an arbitrary cwd — the exact
* same "must not silently fall back to CPU for reasons that have
* nothing to do with GPU availability" concern PR #114's hand-rolled
* bridge already documented for its own embedded-shader-source choice.
* 2. ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU) — find the
* registered Metal device.
* 3. ggml_backend_dev_init(dev, NULL) — get a live ggml_backend_t.
* 4. Build a tiny ggml_context (no_alloc=true; it holds only tensor
* metadata, not data), declare 2D F32 tensors, ggml_mul_mat(nodes,
* query) — ggml's documented convention: A is [k cols, n rows], B is
* [k cols, m rows] (transposed internally), result is [n cols, m rows]
* — i.e. mul_mat(node_matrix[dim,n], query_matrix[dim,nq]) yields
* out[n,nq] where out[j*n+i] = dot(node_i, query_j). A row-major
* (dim,n) node matrix and a row-major (dim,nq) query matrix is EXACTLY
* the packed layout the hand-rolled Metal kernel already used — one
* matmul replaces the whole per-row dot-product loop.
* 5. ggml_backend_alloc_ctx_tensors(ctx, backend) to actually allocate
* device buffers for those tensors, ggml_backend_tensor_set() to upload,
* ggml_backend_graph_compute() to run, ggml_backend_tensor_get() to
* read back.
*
* This exact sequence was verified end-to-end in a standalone probe (build
* it yourself: see the PR description) against a plain-C CPU dot product —
* bit-for-bit correct within float rounding. Real numbers against the
* el_runtime.c CPU oracle are reported in the PR body via vindex_bench.
*
* ggml_mul_mat only computes the raw dot products — it has no notion of
* "cosine" or of this codebase's -2.0 dim-mismatch/null/zero-norm sentinel.
* Per the adapter's directive: gather only VALID, uniform-dim rows into the
* packed matrix sent to the GPU (skipping null/mismatched rows entirely,
* rather than the hand-rolled kernel's zero-pad-and-sentinel-in-shader
* approach), then scatter -2.0 back for every row that was excluded — same
* gather/scatter contract eg_cosine_batch.h documents. Norms (||node||,
* ||query||) are computed on the CPU host in the same pass that already
* touches every element to gather/convert — essentially free — using the
* same 4-way-partial-sum accumulation the hand-rolled kernel and the CPU
* oracle both use, so the float32 error profile stays comparable across all
* three strategies. Only the O(n*dim*nq) dot-product matmul — the actual
* expensive part — is offloaded to the GPU.
*
* ── Precision: the ne11<=8 chunking, and why it is not optional ──────────
*
* The claim in the first version of this file — "ggml_mul_mat on F32 x F32
* inputs computes in F32 on the Metal backend" — is WRONG, and the 0.9933
* id-recall it shipped with (vs the hand-rolled kernel's 0.9997) was the
* symptom. ggml-metal has two F32xF32 matmul kernels and picks between them
* purely on ne11 (the number of B rows == our query count):
*
* ne11 <= 8 -> kernel_mul_mv_ext_f32_f32_* / kernel_mul_mv_f32_f32_*
* templated <float, float> — genuine F32 accumulation.
* ne11 > 8 -> kernel_mul_mm_f32_f32, which is templated
* <half, half4x4, simdgroup_half8x8, half, half2x4,
* simdgroup_half8x8, ...> — i.e. BOTH operands are narrowed
* to F16 and accumulated in simdgroup_half8x8 tiles, even
* though the tensors are GGML_TYPE_F32 on both sides.
*
* (Read it yourself, no guessing — the kernel templates are literal strings
* in the shipped plugin:
* strings $(brew --prefix ggml)/libexec/libggml-metal.so \
* | grep -E 'host_name\("kernel_mul_m[mv]_f32_f32'
* and the runtime pick is visible with GGML_METAL_DEBUG-style logging as
* "compiling pipeline: base = 'kernel_mul_mm_f32_f32'".)
*
* The previous code issued ONE ggml_mul_mat with ne11 = nq (300 in the
* benchmark), landing squarely on the F16 mul_mm path. Measured on this
* machine (M4 Pro), n=13415 x dim=768 x nq=300, against a CPU double-
* accumulated oracle:
*
* ne11=300 (one mul_mat, the old code) : mean |Δdot| = 1.038e-05
* ne11=8 (chunked, this code) : mean |Δdot| = 3.863e-09
*
* — a ~2700x reduction in dot-product error, which is exactly the gap that
* showed up as 0.9933-vs-0.9997 recall.
*
* ggml_mul_mat_set_prec(t, GGML_PREC_F32) does NOT fix this. It was tried:
* the error was bit-identical with and without it (1.038e-05 either way),
* because ggml-metal only consults the prec flag on paths that have an F32
* variant to switch to, and there is no F32-accumulating mul_mm kernel in
* this build to select. The ONLY lever from outside ggml is ne11.
*
* So: instead of one mul_mat with ne11=nq, we emit ceil(nq/8) mul_mats, each
* over an ne11<=8 ggml_view_2d slice of the same query tensor, all into ONE
* graph and ONE ggml_backend_graph_compute. The node matrix is still uploaded
* exactly once and still read by the GPU as one shared operand — the whole
* point of batch_multi is preserved.
*
* The cost is real, and stated rather than buried. Timing the whole
* batch_multi() call (gather + norms + upload + GPU + scatter) on the real
* shape, median of 15 reps after a discarded warm-up, three separate runs:
*
* unchunked (old, F16 mm) : 13.19 / 13.35 / 14.42 ms -> ~0.044 ms/query
* chunked (this code) : 19.92 / 20.08 / 20.23 ms -> ~0.067 ms/query
* hand-rolled Metal : 17.74 / 17.88 / 17.99 ms -> ~0.060 ms/query
*
* So correctness here costs about +6.7ms per 300-query batch (~1.5x on this
* call), and leaves us ~12% behind the hand-rolled kernel instead of ~35%
* ahead of it. That is not free and should not be sold as free. The reason it
* cannot be recovered inside ggml: an fp32 matmul on Metal has to re-stream
* the whole node matrix once per <=8 queries (38 dispatches x ~41MB here),
* where the F16 mul_mm kernel tiles it in threadgroup memory and reads it far
* fewer times. ggml's Metal backend ships no fp32 TILED matmul, so on this
* backend "fast" and "fp32" are genuinely exclusive — the hand-rolled kernel
* escapes the choice only because it is an fp32 kernel written for this one
* shape. Trading precision back for speed is a one-line env change; trading
* the other way was not available before this commit at all.
*
* 8 is not a magic number we invented — it is ggml-metal's own mul_mm
* threshold, measured by sweeping ne11 and watching both the error and which
* pipeline ggml compiles (9 flips to mul_mm and the error jumps back to
* 1.0e-05 in the same step). EL_GGML_MULMAT_CHUNK overrides it: raise it to
* trade this precision back for throughput, or set it >= nq to reproduce the
* old single-mul_mat behaviour exactly. If a future ggml moves the threshold,
* the worst case is that we silently land back on mul_mm — the same accuracy
* we shipped before, never a correctness break.
*
* ── Cold start: what is and is not ours to fix ───────────────────────────
*
* The ~7.8s first-call cost reported for the first version of this file is
* NOT this file re-initialising per call (init is, and always was, cached
* behind g_init_attempted below). It is Apple's Metal shader cache missing
* on ggml's embedded metallib — ggml-metal ships ~650 kernels in one
* __ggml_metallib section, and the first newLibraryWithData of it on a given
* machine costs seconds ("ggml_metal_library_init: loaded in 7.670 sec")
* while the driver populates ~/…/C/com.apple.metal/. That cache is keyed on
* the library, not on our binary, and is shared across processes: the very
* next run of a DIFFERENT binary linking the same ggml reports
* "loaded in 0.009 sec". So it is a once-per-machine, per-ggml-version cost,
* not a per-process one, and nothing this file does can avoid it — the
* hand-rolled strategy escapes it only because its shader is two small
* kernels instead of six hundred.
*
* The residual warm init IS ours to look at, and the answer there is "there
* was nothing much to win": ggml_backend_load_all_from_path() dlopens every
* plugin in the directory (three CPU micro-arch variants + BLAS + Metal) when
* we only ever use Metal, so we now load the single Metal plugin instead —
* but measured warm that is 44.7-52.4ms against 46.9-58.9ms, i.e. the same
* number inside noise, because libggml-metal.so's own init dominates. Warm
* ggml init lands at 44-53ms, against 36-117ms for the hand-rolled strategy's
* device+pipeline setup. Cold start was never the real defect here; precision
* was.
*/
#include "eg_cosine_batch_strategy.h"
#include <ggml.h>
#include <ggml-backend.h>
#include <ggml-alloc.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
/* ── lazy, one-time backend init, cached ─────────────────────────────────── */
static bool g_init_attempted = false;
static bool g_init_ok = false;
static ggml_backend_t g_backend = NULL;
/* Where to look for the dynamically-loaded backend plugin .so files.
* EL_GGML_BACKEND_PATH overrides for non-standard installs; otherwise we try
* the Homebrew opt-prefix symlink (stable across ggml point-version bumps —
* $(brew --prefix ggml)/libexec — confirmed to exist and contain
* libggml-metal.so / libggml-cpu-*.so / libggml-blas.so on this machine),
* falling back to ggml's own default search (ggml_backend_load_all()) in
* case a different install layout (e.g. a from-source build with a
* standard-prefix install) makes that succeed instead. */
static const char* eg_ggml_backend_dir(void) {
const char* s = getenv("EL_GGML_BACKEND_PATH");
if (s && *s) return s;
return "/opt/homebrew/opt/ggml/libexec";
}
/* "<dir>/libggml-metal.so" in a static buffer. Only ever called once, from
* eg_ggml_ensure_init(), before any thread could race it. */
static const char* eg_ggml_metal_plugin_path(const char* dir) {
static char buf[1024];
snprintf(buf, sizeof buf, "%s/libggml-metal.so", dir);
return buf;
}
/* Largest ne11 (query-batch rows per ggml_mul_mat) that keeps ggml-metal on
* its F32 mul_mv kernels instead of the F16-accumulating mul_mm kernel — see
* the precision discussion in this file's header. EL_GGML_MULMAT_CHUNK
* overrides; a value <= 0 means "use the default". */
#define EG_GGML_MULMAT_CHUNK_DEFAULT 8
static int32_t eg_ggml_mulmat_chunk(void) {
static bool resolved = false;
static int32_t chunk = EG_GGML_MULMAT_CHUNK_DEFAULT;
if (!resolved) {
resolved = true;
const char* s = getenv("EL_GGML_MULMAT_CHUNK");
if (s && *s) {
long v = strtol(s, NULL, 10);
if (v > 0 && v <= INT32_MAX) chunk = (int32_t)v;
}
}
return chunk;
}
/* Which ggml device this strategy computes on. GPU (Metal) is the default
* because offloading is the architectural point — the engram's own graph
* traversal and activation spreading are CPU work, and a "GPU" strategy that
* quietly saturates the CPU steals from them.
*
* ACCEL (ggml's BLAS/Accelerate plugin) is reachable here mainly as a
* portability fallback and a diagnostic, and it is documented as MEASURED AND
* REJECTED rather than as a recommendation. In an isolated probe that timed
* only ggml_backend_graph_compute, BLAS looked excellent — 3.4-4.0ms for the
* 300-query batch at mean |Δdot| 1.5e-08, i.e. as fast as the old F16 path and
* far more accurate. End to end on the real store through vindex_bench it does
* not hold up: 0.191 ms/query at id-recall 0.9973, against 0.125-0.142 ms/query
* at 0.9987 for the Metal default. It is dominated on BOTH axes, because the
* isolated probe was not competing with the rest of the batch for the same CPU
* cores and the real call path is. Kept because a machine with no usable Metal
* device still wants a working ggml strategy — not because it is faster. */
static enum ggml_backend_dev_type eg_ggml_device_type(void) {
const char* s = getenv("EL_GGML_DEVICE");
if (s && *s) {
if (strcmp(s, "accel") == 0) return GGML_BACKEND_DEVICE_TYPE_ACCEL;
if (strcmp(s, "cpu") == 0) return GGML_BACKEND_DEVICE_TYPE_CPU;
}
return GGML_BACKEND_DEVICE_TYPE_GPU;
}
static bool eg_ggml_ensure_init(void) {
if (g_init_attempted) return g_init_ok;
g_init_attempted = true;
const char* dir = eg_ggml_backend_dir();
const enum ggml_backend_dev_type want = eg_ggml_device_type();
/* Metal is the only backend this strategy uses by default, so load just
* that one plugin rather than dlopening the whole directory (three CPU
* micro-arch variants + BLAS + Metal here).
*
* Be honest about what this buys: almost nothing in wall time. Measured
* warm, three runs each — load-everything 58.9/46.9/55.1ms, Metal-only
* 52.4/51.2/44.7ms. The cost is dominated by dlopening and initialising
* libggml-metal.so itself, not by the four plugins we skip, so the two
* overlap inside noise. It is kept because registering four device types
* we will never dispatch to is untidy and makes ggml_backend_dev_by_type
* ambiguous, not because it is a speedup — do not cite it as one.
*
* ggml_backend_load() returns NULL for a missing or unloadable path,
* which simply falls through to the broader searches below; it is never
* fatal. Any non-default device needs the full directory scan to find
* its plugin, so skip the fast path there. */
if (want == GGML_BACKEND_DEVICE_TYPE_GPU)
ggml_backend_load(eg_ggml_metal_plugin_path(dir));
ggml_backend_dev_t dev = ggml_backend_dev_by_type(want);
if (!dev) {
/* Non-standard layout, a ggml built with a differently-named Metal
* plugin, or a non-default device: dlopen every plugin in `dir`. */
ggml_backend_load_all_from_path(dir);
dev = ggml_backend_dev_by_type(want);
}
if (!dev) {
/* Fall back to ggml's own default search heuristics only if the
* explicit path above found nothing — avoids double-registering the
* same plugins (ggml does not dedupe two different paths that
* happen to resolve to the same files, e.g. our stable opt-prefix
* symlink vs. its own Cellar-relative guess) in the common case
* where the explicit path already worked. */
ggml_backend_load_all();
dev = ggml_backend_dev_by_type(want);
}
if (!dev) return false;
ggml_backend_t backend = ggml_backend_dev_init(dev, NULL);
if (!backend) return false;
g_backend = backend;
g_init_ok = true;
return true;
}
static bool ggml_strategy_available(void) {
return eg_ggml_ensure_init();
}
/* ── shared core: gather valid rows + norms, matmul, scatter ────────────── */
/* 4-way partial-sum squared-norm accumulation over `dim` floats — same shape
* as eg_cosine_batch.metal's per-thread accumulation and vindex_bench.c's
* CPU brute_topk unroll, kept consistent on purpose so the float32 error
* profile is comparable across all three strategies. */
static float eg_norm_sq_f32(const float* v, int32_t dim) {
float s0 = 0, s1 = 0, s2 = 0, s3 = 0;
int32_t d = 0, dim4 = dim & ~3;
for (; d < dim4; d += 4) {
s0 += v[d] * v[d]; s1 += v[d+1] * v[d+1];
s2 += v[d+2] * v[d+2]; s3 += v[d+3] * v[d+3];
}
float s = (s0 + s1) + (s2 + s3);
for (; d < dim; d++) s += v[d] * v[d];
return s;
}
/* Runs one ggml_mul_mat(node_matrix[dim,n_valid], query_matrix[dim,nq]) and
* combines it with CPU-computed norms into cosine scores, scattering into
* out_scores at ORIGINAL (ungathered) indices. out_scores must already be
* fully sized for n*nq (or n for the single-query case, nq=1) — every entry
* gets written (valid rows get a real cosine, invalid rows get -2.0), so
* this never leaves a partial result. Returns false only on a genuine
* failure (alloc, compute) at which point out_scores is left as whatever a
* caller-supplied scratch buffer already contained — callers here always
* pass a fresh buffer they discard on false, matching the adapter contract
* of "on failure, out_scores is treated as untouched" from the caller's
* point of view. */
static bool eg_ggml_run(const float* queries, int32_t qdim, int32_t nq,
const float* const* node_ptrs, const int32_t* node_dims,
int32_t n, double* out_scores) {
if (!queries || qdim <= 0 || nq <= 0 || !node_ptrs || !node_dims || n <= 0 || !out_scores)
return false;
if (!eg_ggml_ensure_init()) return false;
/* Pass 1 (CPU): gather valid rows (non-NULL ptr, dim == qdim) into a
* packed (dim, n_valid) row-major matrix, remembering the original index
* of each packed row, and compute each valid row's squared norm in the
* same pass. Rows excluded here get -2.0 scattered for every query
* below without ever touching the GPU. */
int32_t* valid_orig = (int32_t*)malloc((size_t)n * sizeof(int32_t));
float* node_norm_sq = (float*)malloc((size_t)n * sizeof(float)); /* indexed by packed position */
float* node_matrix = NULL;
if (!valid_orig || !node_norm_sq) { free(valid_orig); free(node_norm_sq); return false; }
int32_t n_valid = 0;
for (int32_t i = 0; i < n; i++) {
if (node_ptrs[i] && node_dims[i] == qdim) n_valid++;
}
if (n_valid > 0) {
node_matrix = (float*)malloc((size_t)n_valid * (size_t)qdim * sizeof(float));
if (!node_matrix) { free(valid_orig); free(node_norm_sq); return false; }
int32_t w = 0;
for (int32_t i = 0; i < n; i++) {
if (!node_ptrs[i] || node_dims[i] != qdim) continue;
memcpy(node_matrix + (size_t)w * qdim, node_ptrs[i], (size_t)qdim * sizeof(float));
node_norm_sq[w] = eg_norm_sq_f32(node_ptrs[i], qdim);
valid_orig[w] = i;
w++;
}
}
/* Query norms — nq is typically small (1 or the size of one batch of
* comparison queries), so this loop is cheap regardless. */
float* q_norm_sq = (float*)malloc((size_t)nq * sizeof(float));
if (!q_norm_sq) { free(valid_orig); free(node_norm_sq); free(node_matrix); return false; }
for (int32_t j = 0; j < nq; j++) q_norm_sq[j] = eg_norm_sq_f32(queries + (size_t)j * qdim, qdim);
/* Nothing valid to compare against: every output is -2.0. Still a fully
* and correctly populated result — no GPU dispatch was needed to know
* that. */
if (n_valid == 0) {
for (size_t k = 0; k < (size_t)n * (size_t)nq; k++) out_scores[k] = -2.0;
free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq);
return true;
}
/* Pass 2 (GPU via ggml): dot[j*n_valid + i] = dot(node_i, query_j),
* computed as ceil(nq/chunk) separate ggml_mul_mat ops over ne11<=chunk
* ggml_view_2d slices of ONE query tensor, all expanded into ONE graph
* and run by ONE ggml_backend_graph_compute. Chunking is what keeps
* ggml-metal on its F32 mul_mv kernels rather than the F16-accumulating
* mul_mm kernel (see this file's header); sharing one graph and one
* t_nodes tensor is what keeps the node matrix uploaded exactly once,
* which is the entire reason batch_multi exists. */
const int32_t chunk = eg_ggml_mulmat_chunk();
const int32_t ngroups = (nq + chunk - 1) / chunk;
/* Tensors held by the context: t_nodes, t_query, plus one view and one
* mul_mat result per group. The graph holds at most one node per view and
* one per mul_mat. Slack on both so a ggml that bookkeeps slightly
* differently cannot silently overflow the arena. */
const size_t n_tensors = (size_t)2 * (size_t)ngroups + 8;
const size_t graph_size = (size_t)2 * (size_t)ngroups + 16;
struct ggml_init_params gp = {
.mem_size = ggml_tensor_overhead() * n_tensors
+ ggml_graph_overhead_custom(graph_size, false),
.mem_buffer = NULL,
.no_alloc = true,
};
struct ggml_context* ctx = ggml_init(gp);
if (!ctx) { free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; }
struct ggml_tensor** t_dots = (struct ggml_tensor**)malloc((size_t)ngroups * sizeof(*t_dots));
if (!t_dots) { ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; }
struct ggml_tensor* t_nodes = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, qdim, n_valid);
struct ggml_tensor* t_query = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, qdim, nq);
struct ggml_cgraph* gf = t_nodes && t_query
? ggml_new_graph_custom(ctx, graph_size, false) : NULL;
if (!gf) { free(t_dots); ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; }
bool built = true;
for (int32_t g = 0; g < ngroups; g++) {
const int32_t start = g * chunk;
const int32_t count = (start + chunk <= nq) ? chunk : (nq - start);
struct ggml_tensor* t_qv = ggml_view_2d(ctx, t_query, qdim, count,
t_query->nb[1],
(size_t)start * t_query->nb[1]);
t_dots[g] = t_qv ? ggml_mul_mat(ctx, t_nodes, t_qv) : NULL;
if (!t_dots[g]) { built = false; break; }
ggml_build_forward_expand(gf, t_dots[g]);
}
if (!built) { free(t_dots); ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; }
struct ggml_backend_buffer* buf = ggml_backend_alloc_ctx_tensors(ctx, g_backend);
if (!buf) { free(t_dots); ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; }
ggml_backend_tensor_set(t_nodes, node_matrix, 0, (size_t)n_valid * qdim * sizeof(float));
ggml_backend_tensor_set(t_query, queries, 0, (size_t)nq * qdim * sizeof(float));
free(node_matrix); /* uploaded; the packed CPU copy is no longer needed */
enum ggml_status st = ggml_backend_graph_compute(g_backend, gf);
if (st != GGML_STATUS_SUCCESS) {
free(t_dots); ggml_backend_buffer_free(buf); ggml_free(ctx);
free(valid_orig); free(node_norm_sq); free(q_norm_sq);
return false;
}
/* Each group's result is [n_valid, count] contiguous, so reading group g
* into dot + start*n_valid reconstructs exactly the same flat
* dot[j*n_valid + w] layout a single ne11=nq mul_mat would have produced —
* Pass 3 below is unchanged by the chunking. */
float* dot = (float*)malloc((size_t)n_valid * (size_t)nq * sizeof(float));
if (!dot) { free(t_dots); ggml_backend_buffer_free(buf); ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(q_norm_sq); return false; }
for (int32_t g = 0; g < ngroups; g++) {
const int32_t start = g * chunk;
const int32_t count = (start + chunk <= nq) ? chunk : (nq - start);
ggml_backend_tensor_get(t_dots[g], dot + (size_t)start * n_valid, 0,
(size_t)count * (size_t)n_valid * sizeof(float));
}
free(t_dots);
/* Pass 3 (CPU): combine dot/(||a||*||b||) per (query,node) pair, scatter
* into out_scores at ORIGINAL node indices; every excluded row gets
* -2.0 for every query. out_scores is fully populated either way. */
for (int32_t j = 0; j < nq; j++) {
double* orow = out_scores + (size_t)j * n;
for (int32_t i = 0; i < n; i++) orow[i] = -2.0; /* default: excluded */
for (int32_t w = 0; w < n_valid; w++) {
float na = node_norm_sq[w], nb = q_norm_sq[j];
int32_t oi = valid_orig[w];
if (na <= 0.0f || nb <= 0.0f) { orow[oi] = -2.0; continue; }
float d = dot[(size_t)j * n_valid + w];
orow[oi] = (double)(d / sqrtf(na * nb));
}
}
free(dot);
ggml_backend_buffer_free(buf);
ggml_free(ctx);
free(valid_orig); free(node_norm_sq); free(q_norm_sq);
return true;
}
static bool ggml_strategy_batch(const float* query, int32_t qdim,
const float* const* node_ptrs, const int32_t* node_dims,
int32_t n, double* out_scores) {
if (!query || qdim <= 0 || !node_ptrs || !node_dims || n <= 0 || !out_scores) return false;
/* out_scores here is n doubles (nq=1); eg_ggml_run writes n*nq = n of
* them, laid out identically to the single-query contract. */
return eg_ggml_run(query, qdim, 1, node_ptrs, node_dims, n, out_scores);
}
static bool ggml_strategy_batch_multi(const float* queries, int32_t qdim, int32_t nq,
const float* const* node_ptrs, const int32_t* node_dims,
int32_t n, double* out_scores) {
return eg_ggml_run(queries, qdim, nq, node_ptrs, node_dims, n, out_scores);
}
static const EgCosineBatchStrategy g_ggml_strategy = {
.name = "ggml",
.available = ggml_strategy_available,
.batch = ggml_strategy_batch,
.batch_multi = ggml_strategy_batch_multi,
};
const EgCosineBatchStrategy* eg_cosine_batch_strategy_ggml(void) {
return &g_ggml_strategy;
}
lang/runtime/eg_cosine_batch_strategy_metal_hand.m
+358
View File
@@ -0,0 +1,358 @@
/* eg_cosine_batch_strategy_metal_hand.m — the HAND-ROLLED-METAL Strategy.
*
* This is PR #114's original Objective-C bridge (formerly eg_metal_cosine.m)
* exposing the hand-written Metal compute shader (eg_cosine_batch.metal) as
* one concrete EgCosineBatchStrategy. It is preserved here almost verbatim —
* real, carefully verified work, not discarded — now living behind the
* Adapter/Strategy/Factory restructuring (see eg_cosine_batch.h and
* eg_cosine_batch_strategy.h) alongside the new ggml-Metal strategy
* (eg_cosine_batch_strategy_ggml.c) and the universal CPU fallback
* (eg_cosine_batch_strategy_cpu.c). The factory in eg_cosine_batch.c prefers
* ggml by default when both are available; this strategy remains selectable
* via EL_COSINE_BATCH_STRATEGY=metal, and is what the factory falls back to
* if ggml's backend plugin fails to load/init for any reason.
*
* Apple-only (Metal has no other platform). This file is excluded from the
* build entirely on non-Darwin — see build_vindex_bench.sh, which only
* compiles/links this file and defines EG_HAVE_STRATEGY_METAL_HAND when
* `uname` is Darwin. On Linux the factory never sees this strategy at all —
* callers must always be prepared for the "no real strategy available"
* fallback via the CPU strategy, which is also exactly what happens here on
* Apple hardware with no usable GPU.
*
* Design (unchanged from PR #114):
* - Device/queue/pipeline are created lazily, once, and cached in static
* globals — every call after the first only allocates buffers + submits.
* - The Metal shader source is embedded as a C string literal (kMetalSrc
* below) rather than loaded from a file at runtime or shipped as a
* precompiled .metallib. Chosen over newLibraryWithFile: /a .metallib
* because the engram binary can be invoked from an arbitrary working
* directory (launchd job, nsbx sandbox, CI) and a file-path shader would
* be one relocation away from silently falling back to CPU for reasons
* that have nothing to do with Metal availability. Embedding costs one
* runtime shader compile (~tens of ms) on first use, amortized over the
* process lifetime, in exchange for a genuinely self-contained binary.
* Source of truth for review/tooling is eg_cosine_batch.metal — this
* string MUST be kept byte-identical to that file (a comment marks both
* ends of the copy).
* - Buffers use MTLResourceStorageModeShared: on Apple Silicon's unified
* memory, CPU and GPU read the same physical pages, so filling a buffer
* is a plain memcpy and there is no separate "upload" step.
* - ANY failure at ANY step (no device, pipeline compile error, buffer
* allocation failure, bad args) returns false and leaves out_scores
* untouched. This function is called from the request-handling hot path
* of a long-lived daemon — it must never throw, crash, or hang it.
*/
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#include "eg_cosine_batch_strategy.h"
#include <string.h>
#include <stdlib.h>
/* ── BEGIN embedded shader source (keep in sync with eg_cosine_batch.metal) ── */
static const char* kEgCosineBatchMetalSrc =
"#include <metal_stdlib>\n"
"using namespace metal;\n"
"struct EgCosineParams { uint n; uint dim; };\n"
"kernel void eg_cosine_batch_kernel(\n"
" device const float* query [[buffer(0)]],\n"
" device const float* node_matrix [[buffer(1)]],\n"
" device const int* node_dims [[buffer(2)]],\n"
" constant EgCosineParams& p [[buffer(3)]],\n"
" device float* out_scores [[buffer(4)]],\n"
" uint gid [[thread_position_in_grid]])\n"
"{\n"
" if (gid >= p.n) return;\n"
" if (node_dims[gid] != int(p.dim)) { out_scores[gid] = -2.0f; return; }\n"
" device const float* row = node_matrix + (uint64_t)gid * (uint64_t)p.dim;\n"
" float dot0 = 0.0f, dot1 = 0.0f, dot2 = 0.0f, dot3 = 0.0f;\n"
" float na0 = 0.0f, na1 = 0.0f, na2 = 0.0f, na3 = 0.0f;\n"
" float nb0 = 0.0f, nb1 = 0.0f, nb2 = 0.0f, nb3 = 0.0f;\n"
" uint d = 0;\n"
" uint dim4 = p.dim & ~3u;\n"
" for (; d < dim4; d += 4) {\n"
" float a0 = row[d], b0 = query[d];\n"
" float a1 = row[d+1], b1 = query[d+1];\n"
" float a2 = row[d+2], b2 = query[d+2];\n"
" float a3 = row[d+3], b3 = query[d+3];\n"
" dot0 += a0*b0; dot1 += a1*b1; dot2 += a2*b2; dot3 += a3*b3;\n"
" na0 += a0*a0; na1 += a1*a1; na2 += a2*a2; na3 += a3*a3;\n"
" nb0 += b0*b0; nb1 += b1*b1; nb2 += b2*b2; nb3 += b3*b3;\n"
" }\n"
" float dot = (dot0 + dot1) + (dot2 + dot3);\n"
" float na = (na0 + na1) + (na2 + na3);\n"
" float nb = (nb0 + nb1) + (nb2 + nb3);\n"
" for (; d < p.dim; d++) {\n"
" float a = row[d], b = query[d];\n"
" dot += a*b; na += a*a; nb += b*b;\n"
" }\n"
" if (na <= 0.0f || nb <= 0.0f) { out_scores[gid] = -2.0f; return; }\n"
" out_scores[gid] = dot / sqrt(na * nb);\n"
"}\n"
"struct EgCosineMultiParams { uint n; uint dim; uint nq; };\n"
"kernel void eg_cosine_batch_multi_kernel(\n"
" device const float* queries [[buffer(0)]],\n"
" device const float* node_matrix [[buffer(1)]],\n"
" device const int* node_dims [[buffer(2)]],\n"
" constant EgCosineMultiParams& p [[buffer(3)]],\n"
" device float* out_scores [[buffer(4)]],\n"
" uint2 gid [[thread_position_in_grid]])\n"
"{\n"
" uint nid = gid.x, qid = gid.y;\n"
" if (nid >= p.n || qid >= p.nq) return;\n"
" uint64_t out_idx = (uint64_t)qid * (uint64_t)p.n + (uint64_t)nid;\n"
" if (node_dims[nid] != int(p.dim)) { out_scores[out_idx] = -2.0f; return; }\n"
" device const float* row = node_matrix + (uint64_t)nid * (uint64_t)p.dim;\n"
" device const float* query = queries + (uint64_t)qid * (uint64_t)p.dim;\n"
" float dot0 = 0.0f, dot1 = 0.0f, dot2 = 0.0f, dot3 = 0.0f;\n"
" float na0 = 0.0f, na1 = 0.0f, na2 = 0.0f, na3 = 0.0f;\n"
" float nb0 = 0.0f, nb1 = 0.0f, nb2 = 0.0f, nb3 = 0.0f;\n"
" uint d = 0;\n"
" uint dim4 = p.dim & ~3u;\n"
" for (; d < dim4; d += 4) {\n"
" float a0 = row[d], b0 = query[d];\n"
" float a1 = row[d+1], b1 = query[d+1];\n"
" float a2 = row[d+2], b2 = query[d+2];\n"
" float a3 = row[d+3], b3 = query[d+3];\n"
" dot0 += a0*b0; dot1 += a1*b1; dot2 += a2*b2; dot3 += a3*b3;\n"
" na0 += a0*a0; na1 += a1*a1; na2 += a2*a2; na3 += a3*a3;\n"
" nb0 += b0*b0; nb1 += b1*b1; nb2 += b2*b2; nb3 += b3*b3;\n"
" }\n"
" float dot = (dot0 + dot1) + (dot2 + dot3);\n"
" float na = (na0 + na1) + (na2 + na3);\n"
" float nb = (nb0 + nb1) + (nb2 + nb3);\n"
" for (; d < p.dim; d++) {\n"
" float a = row[d], b = query[d];\n"
" dot += a*b; na += a*a; nb += b*b;\n"
" }\n"
" if (na <= 0.0f || nb <= 0.0f) { out_scores[out_idx] = -2.0f; return; }\n"
" out_scores[out_idx] = dot / sqrt(na * nb);\n"
"}\n";
/* ── END embedded shader source ── */
typedef struct EgCosineParamsC { uint32_t n; uint32_t dim; } EgCosineParamsC;
typedef struct EgCosineMultiParamsC { uint32_t n; uint32_t dim; uint32_t nq; } EgCosineMultiParamsC;
static id<MTLDevice> g_device = nil;
static id<MTLCommandQueue> g_queue = nil;
static id<MTLComputePipelineState> g_pipeline = nil; /* single-query kernel */
static id<MTLComputePipelineState> g_pipeline_multi = nil; /* multi-query kernel */
static bool g_init_attempted = false;
static bool g_init_ok = false;
/* Lazy, one-time setup. Never throws — every Metal call here is the
* "returns nil/NSError on failure" flavor, not an exception-throwing one. */
static bool eg_metal_ensure_init(void) {
if (g_init_attempted) return g_init_ok;
g_init_attempted = true;
@autoreleasepool {
id<MTLDevice> dev = MTLCreateSystemDefaultDevice();
if (!dev) return false;
id<MTLCommandQueue> q = [dev newCommandQueue];
if (!q) return false;
NSError* err = nil;
NSString* src = [NSString stringWithUTF8String:kEgCosineBatchMetalSrc];
MTLCompileOptions* opts = [MTLCompileOptions new];
id<MTLLibrary> lib = [dev newLibraryWithSource:src options:opts error:&err];
if (!lib) return false;
id<MTLFunction> fn = [lib newFunctionWithName:@"eg_cosine_batch_kernel"];
if (!fn) return false;
id<MTLComputePipelineState> pipe = [dev newComputePipelineStateWithFunction:fn error:&err];
if (!pipe) return false;
id<MTLFunction> fnMulti = [lib newFunctionWithName:@"eg_cosine_batch_multi_kernel"];
if (!fnMulti) return false;
id<MTLComputePipelineState> pipeMulti = [dev newComputePipelineStateWithFunction:fnMulti error:&err];
if (!pipeMulti) return false;
g_device = dev;
g_queue = q;
g_pipeline = pipe;
g_pipeline_multi = pipeMulti;
g_init_ok = true;
return true;
}
}
static bool mh_available(void) {
return eg_metal_ensure_init();
}
static bool mh_batch(const float* query, int32_t qdim,
const float* const* node_ptrs,
const int32_t* node_dims,
int32_t n,
double* out_scores) {
if (!query || qdim <= 0 || !node_ptrs || !node_dims || n <= 0 || !out_scores) return false;
if (!eg_metal_ensure_init()) return false;
@autoreleasepool {
const size_t dim = (size_t)qdim;
const size_t nu = (size_t)n;
/* Gather into a packed row-major matrix — EngramNode.emb is one
* malloc per node, not a contiguous array, so this copy is
* unavoidable regardless of backend. Rows whose real dim doesn't
* match qdim are zero-filled (harmless: the kernel sentinels them
* via node_dims before ever reading the row). */
float* matrix = (float*)calloc(nu * dim, sizeof(float));
int32_t* dims_i32 = (int32_t*)malloc(nu * sizeof(int32_t));
if (!matrix || !dims_i32) { free(matrix); free(dims_i32); return false; }
for (size_t i = 0; i < nu; i++) {
dims_i32[i] = node_dims[i];
if (node_ptrs[i] && node_dims[i] == qdim) {
memcpy(matrix + i * dim, node_ptrs[i], dim * sizeof(float));
}
/* else: leave zero-filled; node_dims[i] != qdim (or missing)
* makes the kernel sentinel it to -2.0 without reading the row. */
}
id<MTLBuffer> bufQuery = [g_device newBufferWithBytes:query
length:dim * sizeof(float)
options:MTLResourceStorageModeShared];
id<MTLBuffer> bufMatrix = [g_device newBufferWithBytes:matrix
length:nu * dim * sizeof(float)
options:MTLResourceStorageModeShared];
id<MTLBuffer> bufDims = [g_device newBufferWithBytes:dims_i32
length:nu * sizeof(int32_t)
options:MTLResourceStorageModeShared];
EgCosineParamsC params = { (uint32_t)nu, (uint32_t)dim };
id<MTLBuffer> bufParams = [g_device newBufferWithBytes:&params
length:sizeof(params)
options:MTLResourceStorageModeShared];
id<MTLBuffer> bufOut = [g_device newBufferWithLength:nu * sizeof(float)
options:MTLResourceStorageModeShared];
free(matrix); free(dims_i32);
if (!bufQuery || !bufMatrix || !bufDims || !bufParams || !bufOut) return false;
id<MTLCommandBuffer> cmd = [g_queue commandBuffer];
if (!cmd) return false;
id<MTLComputeCommandEncoder> enc = [cmd computeCommandEncoder];
if (!enc) return false;
[enc setComputePipelineState:g_pipeline];
[enc setBuffer:bufQuery offset:0 atIndex:0];
[enc setBuffer:bufMatrix offset:0 atIndex:1];
[enc setBuffer:bufDims offset:0 atIndex:2];
[enc setBuffer:bufParams offset:0 atIndex:3];
[enc setBuffer:bufOut offset:0 atIndex:4];
NSUInteger tgSize = g_pipeline.maxTotalThreadsPerThreadgroup;
if (tgSize > 256) tgSize = 256;
if (tgSize < 1) tgSize = 1;
MTLSize gridSize = MTLSizeMake(nu, 1, 1);
MTLSize threadgroupSize = MTLSizeMake(tgSize, 1, 1);
[enc dispatchThreads:gridSize threadsPerThreadgroup:threadgroupSize];
[enc endEncoding];
[cmd commit];
[cmd waitUntilCompleted];
if (cmd.status != MTLCommandBufferStatusCompleted) return false;
const float* results = (const float*)bufOut.contents;
if (!results) return false;
for (size_t i = 0; i < nu; i++) out_scores[i] = (double)results[i];
return true;
}
}
static bool mh_batch_multi(const float* queries, int32_t qdim, int32_t nq,
const float* const* node_ptrs,
const int32_t* node_dims,
int32_t n,
double* out_scores) {
if (!queries || qdim <= 0 || nq <= 0 || !node_ptrs || !node_dims || n <= 0 || !out_scores) return false;
if (!eg_metal_ensure_init()) return false;
@autoreleasepool {
const size_t dim = (size_t)qdim;
const size_t nu = (size_t)n;
const size_t nqu = (size_t)nq;
float* matrix = (float*)calloc(nu * dim, sizeof(float));
int32_t* dims_i32 = (int32_t*)malloc(nu * sizeof(int32_t));
if (!matrix || !dims_i32) { free(matrix); free(dims_i32); return false; }
for (size_t i = 0; i < nu; i++) {
dims_i32[i] = node_dims[i];
if (node_ptrs[i] && node_dims[i] == qdim) {
memcpy(matrix + i * dim, node_ptrs[i], dim * sizeof(float));
}
}
/* This is the ONE upload of node_matrix for the whole nq-query batch —
* the fix for the measured re-upload-per-query slowdown. */
id<MTLBuffer> bufMatrix = [g_device newBufferWithBytes:matrix
length:nu * dim * sizeof(float)
options:MTLResourceStorageModeShared];
id<MTLBuffer> bufDims = [g_device newBufferWithBytes:dims_i32
length:nu * sizeof(int32_t)
options:MTLResourceStorageModeShared];
id<MTLBuffer> bufQueries = [g_device newBufferWithBytes:queries
length:nqu * dim * sizeof(float)
options:MTLResourceStorageModeShared];
EgCosineMultiParamsC params = { (uint32_t)nu, (uint32_t)dim, (uint32_t)nqu };
id<MTLBuffer> bufParams = [g_device newBufferWithBytes:&params
length:sizeof(params)
options:MTLResourceStorageModeShared];
id<MTLBuffer> bufOut = [g_device newBufferWithLength:nqu * nu * sizeof(float)
options:MTLResourceStorageModeShared];
free(matrix); free(dims_i32);
if (!bufMatrix || !bufDims || !bufQueries || !bufParams || !bufOut) return false;
id<MTLCommandBuffer> cmd = [g_queue commandBuffer];
if (!cmd) return false;
id<MTLComputeCommandEncoder> enc = [cmd computeCommandEncoder];
if (!enc) return false;
[enc setComputePipelineState:g_pipeline_multi];
[enc setBuffer:bufQueries offset:0 atIndex:0];
[enc setBuffer:bufMatrix offset:0 atIndex:1];
[enc setBuffer:bufDims offset:0 atIndex:2];
[enc setBuffer:bufParams offset:0 atIndex:3];
[enc setBuffer:bufOut offset:0 atIndex:4];
/* 2D dispatch: x over nodes, y over queries. Threadgroup width picked
* from the pipeline's own limit, height fixed at 1 — nq is typically
* small (tens to low hundreds) relative to n (thousands+), so tiling
* the wide axis (n) is what matters for occupancy. */
NSUInteger tgWidth = g_pipeline_multi.maxTotalThreadsPerThreadgroup;
if (tgWidth > 256) tgWidth = 256;
if (tgWidth < 1) tgWidth = 1;
MTLSize gridSize = MTLSizeMake(nu, nqu, 1);
MTLSize threadgroupSize = MTLSizeMake(tgWidth, 1, 1);
[enc dispatchThreads:gridSize threadsPerThreadgroup:threadgroupSize];
[enc endEncoding];
[cmd commit];
[cmd waitUntilCompleted];
if (cmd.status != MTLCommandBufferStatusCompleted) return false;
const float* results = (const float*)bufOut.contents;
if (!results) return false;
for (size_t i = 0; i < nqu * nu; i++) out_scores[i] = (double)results[i];
return true;
}
}
static const EgCosineBatchStrategy g_metal_hand_strategy = {
.name = "metal-hand",
.available = mh_available,
.batch = mh_batch,
.batch_multi = mh_batch_multi,
};
const EgCosineBatchStrategy* eg_cosine_batch_strategy_metal_hand(void) {
return &g_metal_hand_strategy;
}
lang/runtime/el_audio_darwin.m
+525
View File
@@ -0,0 +1,525 @@
/* el_audio_darwin.m — the SPEAKER realizer. El's native audio output on Darwin.
*
* WHY THIS FILE EXISTS.
*
* Neuron could already turn meaning into samples — the render path in
* elp/src/speech.el superposes formant resonances over a glottal source and
* produces PCM. What it could not do was make a sound. Every path from those
* samples to the air ran outside the language: a 939-line Swift program
* (peripheral/src/periph.swift) that shelled out to /usr/bin/afplay. So the
* voice was not a capability of El or of Neuron. It was a separate binary
* standing next to them, and "speak" meant "ask that binary to speak."
*
* A speaker is not a language feature the way a string is, but it is exactly
* the kind of thing a runtime owns: a device. El already owns the filesystem,
* the network, the clock, and a graph. It should own the one output device that
* makes it audible. After this file, `speak` is an El operation.
*
* WHY IT IS A REALIZER AND NOT PURE EL.
*
* This is the boundary the whole design turns on. Everything ABOVE the sample
* buffer is arithmetic and belongs in El: formant geometry, superposition,
* envelopes, WAV framing, the voice signature. Everything in this file is the
* part that cannot be arithmetic — handing a buffer to CoreAudio and waiting
* for the hardware to drain it. There is no way to express "the DAC has now
* played these samples" in El, and there should not be. So the split is: El
* computes the sound, the realizer emits it, and the realizer is as thin as it
* can possibly be — it makes no decisions about content, it has no opinion
* about audio, and it cannot synthesize anything.
*
* The precedent is eg_cosine_batch_strategy_metal_hand.m: a platform-bound
* capability compiled as its OWN translation unit, declared in el_runtime.h,
* and linked in where the platform supports it. Deliberately NOT a patch to
* el_runtime.c — adding a device to El must not mean editing the core runtime,
* for the same reason adding a modality must not (see el_runtime.c's realizer
* registry: a realizer is resolved by name, so new organs never touch the
* middle of the language). el_audio_null.c is the same two entry points for
* every platform that is not Darwin, so El code that speaks still links
* everywhere and simply reports that it has no speaker.
*
* WHY AudioQueue AND NOT afplay.
*
* afplay is a process. Using it means the sound Neuron makes is a file it wrote
* and asked something else to open — which forces every utterance through the
* disk, cannot start until the whole utterance exists, and puts a fork/exec
* between the intent to speak and the sound. AudioQueue takes the samples
* directly out of memory. Nothing is written, nothing is spawned, and a caller
* that wants to stream can push buffers as it renders them.
*
* AudioToolbox ships with macOS, so this stays own-core: no cloud, no library
* to install, no model. The output is the local speaker and nothing leaves the
* machine — there is no network path in this file at all, by construction.
*/
#import <AudioToolbox/AudioToolbox.h>
#import <Foundation/Foundation.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdio.h>
#include "el_runtime.h"
/* Three buffers is the standard AudioQueue depth: one being played by the
* hardware, one queued behind it, one being refilled. Fewer risks a gap on a
* busy machine; more only adds latency before the first sound. */
#define EL_AQ_NBUF 3
#define EL_AQ_FRAMES 8192
typedef struct {
const int16_t* pcm;
int64_t frames;
int64_t pos;
volatile int inflight; /* buffers CoreAudio still owns */
volatile int drained; /* set once the last buffer has been played */
} ElAqState;
/* Called on an AudioQueue-internal thread each time a buffer finishes playing.
* Refills and re-enqueues while samples remain; when the source is exhausted it
* lets the buffer die and counts it out. `drained` flips only when the queue is
* holding nothing, which is what makes the play call synchronous without
* clipping the tail — the same reason periph.swift used .dataPlayedBack rather
* than treating "consumed" as "heard". */
static void el_aq_callback(void* userData, AudioQueueRef q, AudioQueueBufferRef buf) {
ElAqState* st = (ElAqState*)userData;
int64_t remain = st->frames - st->pos;
if (remain <= 0) {
if (--st->inflight <= 0) st->drained = 1;
return;
}
int64_t n = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES;
memcpy(buf->mAudioData, st->pcm + st->pos, (size_t)n * sizeof(int16_t));
buf->mAudioDataByteSize = (UInt32)(n * (int64_t)sizeof(int16_t));
st->pos += n;
if (AudioQueueEnqueueBuffer(q, buf, 0, NULL) != noErr) {
if (--st->inflight <= 0) st->drained = 1;
}
}
/* Play a 16-bit mono PCM buffer out the default output device, blocking until
* the hardware has actually finished. Returns 1 on success, 0 on any failure —
* never throws, never hangs indefinitely. */
static int el_audio_play_raw(const int16_t* pcm, int64_t frames, int32_t sample_rate) {
if (!pcm || frames <= 0 || sample_rate <= 0) return 0;
AudioStreamBasicDescription fmt;
memset(&fmt, 0, sizeof(fmt));
fmt.mSampleRate = (Float64)sample_rate;
fmt.mFormatID = kAudioFormatLinearPCM;
fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked;
fmt.mFramesPerPacket = 1;
fmt.mChannelsPerFrame = 1;
fmt.mBitsPerChannel = 16;
fmt.mBytesPerFrame = 2;
fmt.mBytesPerPacket = 2;
ElAqState st;
memset(&st, 0, sizeof(st));
st.pcm = pcm;
st.frames = frames;
AudioQueueRef q = NULL;
/* NULL run loop => callbacks arrive on an AudioQueue-internal thread, so
* this function can simply wait rather than having to pump a run loop it
* does not own. El programs are not required to have one. */
if (AudioQueueNewOutput(&fmt, el_aq_callback, &st, NULL, NULL, 0, &q) != noErr || !q) {
return 0;
}
AudioQueueBufferRef bufs[EL_AQ_NBUF];
int prepared = 0;
for (int i = 0; i < EL_AQ_NBUF; i++) {
if (AudioQueueAllocateBuffer(q, EL_AQ_FRAMES * sizeof(int16_t), &bufs[i]) != noErr) break;
prepared++;
}
if (prepared == 0) { AudioQueueDispose(q, true); return 0; }
/* Prime: fill what we can before starting, so playback begins immediately
* rather than after the first underrun. */
for (int i = 0; i < prepared; i++) {
int64_t remain = st.frames - st.pos;
if (remain <= 0) break;
int64_t n = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES;
memcpy(bufs[i]->mAudioData, st.pcm + st.pos, (size_t)n * sizeof(int16_t));
bufs[i]->mAudioDataByteSize = (UInt32)(n * (int64_t)sizeof(int16_t));
st.pos += n;
if (AudioQueueEnqueueBuffer(q, bufs[i], 0, NULL) != noErr) break;
st.inflight++;
}
if (st.inflight == 0) { AudioQueueDispose(q, true); return 0; }
if (AudioQueueStart(q, NULL) != noErr) { AudioQueueDispose(q, true); return 0; }
/* Bound the wait by the material's own duration plus a margin. A speaker
* that wedges a program is worse than a speaker that gives up. */
double seconds = (double)frames / (double)sample_rate;
int64_t max_us = (int64_t)((seconds + 5.0) * 1000000.0);
int64_t waited = 0;
const int64_t tick = 5000; /* 5 ms */
while (!st.drained && waited < max_us) {
usleep((useconds_t)tick);
waited += tick;
}
AudioQueueStop(q, true);
AudioQueueDispose(q, true);
return st.drained ? 1 : 0;
}
/* ── El entry points ────────────────────────────────────────────────────────
* Declared in el_runtime.h; see there for the El-facing contract. */
/* 1 when this build has a real speaker behind it. El code should ask before
* speaking so the no-speaker case is a reported condition, not a silence that
* looks like success. */
el_val_t speaker_available(void) {
return (el_val_t)1;
}
el_val_t speaker_name(void) {
return EL_STR("coreaudio-audioqueue");
}
/* Play an El [Int] of 16-bit samples. Values are clamped, not wrapped: a
* render that overshoots should distort at the rails the way real clipping
* does, rather than invert phase and produce a sound nothing in the signal
* chain intended. */
el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate) {
int64_t n = (int64_t)el_list_len(samples);
int32_t sr = (int32_t)sample_rate;
if (n <= 0 || sr <= 0) return (el_val_t)0;
int16_t* pcm = (int16_t*)malloc((size_t)n * sizeof(int16_t));
if (!pcm) return (el_val_t)0;
for (int64_t i = 0; i < n; i++) {
int64_t v = (int64_t)el_list_get(samples, (el_val_t)i);
if (v > 32767) v = 32767;
if (v < -32768) v = -32768;
pcm[i] = (int16_t)v;
}
int ok = el_audio_play_raw(pcm, n, sr);
free(pcm);
return (el_val_t)(ok ? 1 : 0);
}
/* ── Asynchronous playback ───────────────────────────────────────────────────
*
* converse needs this and a blocking play cannot give it. Barge-in means
* stopping ON THE SPOT when the user starts talking — not at the end of the
* current buffer, and certainly not at the end of the utterance. So the async
* path keeps one queue alive, reports how far the hardware actually got, and
* can be halted mid-buffer.
*
* `played_frames` is what makes an interrupted utterance resumable at the
* sample rather than at the segment: it is the position the DAC reached, not
* the position we enqueued to, and those differ by up to the full queue depth.
*
* One utterance at a time. A second async play stops the first — a mouth that
* can say two things at once is not a feature. */
static AudioQueueRef g_aq = NULL;
static ElAqState* g_aq_state = NULL;
static int16_t* g_aq_pcm = NULL;
static int32_t g_aq_sr = 0;
static void el_audio_teardown(void) {
if (g_aq) {
AudioQueueStop(g_aq, true);
AudioQueueDispose(g_aq, true);
g_aq = NULL;
}
free(g_aq_pcm); g_aq_pcm = NULL;
free(g_aq_state); g_aq_state = NULL;
g_aq_sr = 0;
}
el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate) {
el_audio_teardown();
int64_t n = (int64_t)el_list_len(samples);
int32_t sr = (int32_t)sample_rate;
if (n <= 0 || sr <= 0) return (el_val_t)0;
g_aq_pcm = (int16_t*)malloc((size_t)n * sizeof(int16_t));
if (!g_aq_pcm) return (el_val_t)0;
for (int64_t i = 0; i < n; i++) {
int64_t v = (int64_t)el_list_get(samples, (el_val_t)i);
if (v > 32767) v = 32767;
if (v < -32768) v = -32768;
g_aq_pcm[i] = (int16_t)v;
}
g_aq_state = (ElAqState*)calloc(1, sizeof(ElAqState));
if (!g_aq_state) { el_audio_teardown(); return (el_val_t)0; }
g_aq_state->pcm = g_aq_pcm;
g_aq_state->frames = n;
g_aq_sr = sr;
AudioStreamBasicDescription fmt;
memset(&fmt, 0, sizeof(fmt));
fmt.mSampleRate = (Float64)sr;
fmt.mFormatID = kAudioFormatLinearPCM;
fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked;
fmt.mFramesPerPacket = 1;
fmt.mChannelsPerFrame = 1;
fmt.mBitsPerChannel = 16;
fmt.mBytesPerFrame = 2;
fmt.mBytesPerPacket = 2;
if (AudioQueueNewOutput(&fmt, el_aq_callback, g_aq_state, NULL, NULL, 0, &g_aq) != noErr || !g_aq) {
el_audio_teardown();
return (el_val_t)0;
}
for (int i = 0; i < EL_AQ_NBUF; i++) {
int64_t remain = g_aq_state->frames - g_aq_state->pos;
if (remain <= 0) break;
AudioQueueBufferRef b = NULL;
if (AudioQueueAllocateBuffer(g_aq, EL_AQ_FRAMES * sizeof(int16_t), &b) != noErr) break;
int64_t k = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES;
memcpy(b->mAudioData, g_aq_state->pcm + g_aq_state->pos, (size_t)k * sizeof(int16_t));
b->mAudioDataByteSize = (UInt32)(k * (int64_t)sizeof(int16_t));
g_aq_state->pos += k;
if (AudioQueueEnqueueBuffer(g_aq, b, 0, NULL) != noErr) break;
g_aq_state->inflight++;
}
if (g_aq_state->inflight == 0) { el_audio_teardown(); return (el_val_t)0; }
if (AudioQueueStart(g_aq, NULL) != noErr) { el_audio_teardown(); return (el_val_t)0; }
return (el_val_t)1;
}
el_val_t speaker_playing(void) {
if (!g_aq || !g_aq_state) return (el_val_t)0;
return (el_val_t)(g_aq_state->drained ? 0 : 1);
}
/* Frames the DAC has actually rendered. AudioQueueGetCurrentTime's mSampleTime
* is relative to queue start, which is exactly the "where was I really" figure
* a resumable utterance needs. Falls back to the enqueued position if the
* timeline is unavailable (it is, briefly, right after start). */
el_val_t speaker_played_frames(void) {
if (!g_aq || !g_aq_state) return (el_val_t)0;
AudioTimeStamp ts;
memset(&ts, 0, sizeof(ts));
Boolean discontinuity = false;
if (AudioQueueGetCurrentTime(g_aq, NULL, &ts, &discontinuity) == noErr &&
(ts.mFlags & kAudioTimeStampSampleTimeValid)) {
int64_t played = (int64_t)ts.mSampleTime;
if (played < 0) played = 0;
if (played > g_aq_state->frames) played = g_aq_state->frames;
return (el_val_t)played;
}
return (el_val_t)g_aq_state->pos;
}
/* Pause where we are, keeping the queue and its position intact.
*
* This is the difference between barge-in and "finish the buffer". The moment
* the microphone hears speech, output must stop AT THAT SAMPLE — a listener
* experiences even 200ms of continued talking as being talked over. Pause
* rather than stop because the interruption might turn out to be a backchannel
* ("mm-hm"), and the right response to a backchannel is to carry on as though
* nothing happened, which requires the queue to still be exactly where it was.
* A stop-and-restart would re-attack the buffer and be audible as a stutter. */
el_val_t speaker_pause(void) {
if (!g_aq) return (el_val_t)0;
return (el_val_t)(AudioQueuePause(g_aq) == noErr ? 1 : 0);
}
el_val_t speaker_resume(void) {
if (!g_aq) return (el_val_t)0;
return (el_val_t)(AudioQueueStart(g_aq, NULL) == noErr ? 1 : 0);
}
el_val_t speaker_stop(void) {
if (!g_aq) return (el_val_t)0;
/* immediate: do NOT let the queue finish what it is holding */
AudioQueueStop(g_aq, true);
el_audio_teardown();
return (el_val_t)1;
}
/* Decode a 16-bit RIFF/WAVE into a freshly malloc'd mono int16 buffer.
* Returns frames, or 0 on any failure; *out is set only on success. Shared by
* the blocking and async WAV paths. */
static int64_t el_wav_load(const char* path, int16_t** out, int32_t* out_sr) {
if (!path || !out) return 0;
FILE* f = fopen(path, "rb");
if (!f) return 0;
if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return 0; }
long size = ftell(f);
if (size <= 44) { fclose(f); return 0; }
rewind(f);
unsigned char* d = (unsigned char*)malloc((size_t)size);
if (!d) { fclose(f); return 0; }
size_t got = fread(d, 1, (size_t)size, f);
fclose(f);
if (got != (size_t)size) { free(d); return 0; }
if (memcmp(d, "RIFF", 4) != 0 || memcmp(d + 8, "WAVE", 4) != 0) { free(d); return 0; }
int32_t sr = 0, channels = 0, bits = 0;
long dataOff = -1, dataLen = 0, o = 12;
/* Chunk-walk rather than assuming fmt-then-data at fixed offsets: recorders
* routinely interleave JUNK/FLLR padding, and a fixed-offset parser reads
* padding as audio. */
while (o + 8 <= size) {
long sz = (long)d[o+4] | ((long)d[o+5] << 8) | ((long)d[o+6] << 16) | ((long)d[o+7] << 24);
if (sz < 0) break;
if (memcmp(d + o, "fmt ", 4) == 0 && o + 24 <= size) {
channels = (int32_t)(d[o+10] | (d[o+11] << 8));
sr = (int32_t)((long)d[o+12] | ((long)d[o+13] << 8) | ((long)d[o+14] << 16) | ((long)d[o+15] << 24));
bits = (int32_t)(d[o+22] | (d[o+23] << 8));
} else if (memcmp(d + o, "data", 4) == 0) {
dataOff = o + 8;
dataLen = sz;
if (dataOff + dataLen > size) dataLen = size - dataOff;
}
o += 8 + sz + (sz & 1);
}
if (dataOff < 0 || sr <= 0 || bits != 16 || channels < 1 || dataLen <= 0) { free(d); return 0; }
long frames = dataLen / (2 * channels);
int16_t* pcm = (int16_t*)malloc((size_t)frames * sizeof(int16_t));
if (!pcm) { free(d); return 0; }
for (long i = 0; i < frames; i++) {
long b = dataOff + i * 2 * channels;
pcm[i] = (int16_t)((unsigned)d[b] | ((unsigned)d[b+1] << 8));
}
free(d);
*out = pcm;
if (out_sr) *out_sr = sr;
return (int64_t)frames;
}
/* Async WAV playback. converse speaks PRE-RENDERED segments and must keep
* listening while it does, so it needs the file on the queue without blocking
* and needs to be able to stop it mid-buffer. Going through the file rather
* than an El [Int] also avoids marshalling a million-element list per segment
* for audio the caller never intends to look at. */
el_val_t speaker_play_wav_async(el_val_t path) {
const char* p = EL_CSTR(path);
if (!p) return (el_val_t)0;
el_audio_teardown();
int32_t sr = 0;
int16_t* pcm = NULL;
int64_t frames = el_wav_load(p, &pcm, &sr);
if (frames <= 0 || !pcm) { free(pcm); return (el_val_t)0; }
g_aq_pcm = pcm;
g_aq_sr = sr;
g_aq_state = (ElAqState*)calloc(1, sizeof(ElAqState));
if (!g_aq_state) { el_audio_teardown(); return (el_val_t)0; }
g_aq_state->pcm = g_aq_pcm;
g_aq_state->frames = frames;
AudioStreamBasicDescription fmt;
memset(&fmt, 0, sizeof(fmt));
fmt.mSampleRate = (Float64)sr;
fmt.mFormatID = kAudioFormatLinearPCM;
fmt.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked;
fmt.mFramesPerPacket = 1;
fmt.mChannelsPerFrame = 1;
fmt.mBitsPerChannel = 16;
fmt.mBytesPerFrame = 2;
fmt.mBytesPerPacket = 2;
if (AudioQueueNewOutput(&fmt, el_aq_callback, g_aq_state, NULL, NULL, 0, &g_aq) != noErr || !g_aq) {
el_audio_teardown();
return (el_val_t)0;
}
for (int i = 0; i < EL_AQ_NBUF; i++) {
int64_t remain = g_aq_state->frames - g_aq_state->pos;
if (remain <= 0) break;
AudioQueueBufferRef b = NULL;
if (AudioQueueAllocateBuffer(g_aq, EL_AQ_FRAMES * sizeof(int16_t), &b) != noErr) break;
int64_t k = remain < EL_AQ_FRAMES ? remain : EL_AQ_FRAMES;
memcpy(b->mAudioData, g_aq_state->pcm + g_aq_state->pos, (size_t)k * sizeof(int16_t));
b->mAudioDataByteSize = (UInt32)(k * (int64_t)sizeof(int16_t));
g_aq_state->pos += k;
if (AudioQueueEnqueueBuffer(g_aq, b, 0, NULL) != noErr) break;
g_aq_state->inflight++;
}
if (g_aq_state->inflight == 0) { el_audio_teardown(); return (el_val_t)0; }
if (AudioQueueStart(g_aq, NULL) != noErr) { el_audio_teardown(); return (el_val_t)0; }
return (el_val_t)1;
}
/* Total frames and sample rate of a WAV, without playing it — wav-info, and the
* duration converse needs to compute progress through a segment. */
el_val_t wav_frames(el_val_t path) {
const char* p = EL_CSTR(path);
int16_t* pcm = NULL; int32_t sr = 0;
int64_t n = el_wav_load(p, &pcm, &sr);
free(pcm);
return (el_val_t)n;
}
el_val_t wav_rate(el_val_t path) {
const char* p = EL_CSTR(path);
int16_t* pcm = NULL; int32_t sr = 0;
int64_t n = el_wav_load(p, &pcm, &sr);
free(pcm);
return (el_val_t)(n > 0 ? sr : 0);
}
/* Play a 16-bit mono RIFF/WAVE file. Present because the render already knows
* how to write a WAV and a caller may reasonably want to hear one back without
* re-rendering it; the parse is deliberately minimal and chunk-walking, so the
* JUNK/FLLR padding that recorders emit does not defeat it. */
el_val_t speaker_play_wav(el_val_t path) {
const char* p = EL_CSTR(path);
if (!p) return (el_val_t)0;
FILE* f = fopen(p, "rb");
if (!f) return (el_val_t)0;
if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return (el_val_t)0; }
long size = ftell(f);
if (size <= 44) { fclose(f); return (el_val_t)0; }
rewind(f);
unsigned char* d = (unsigned char*)malloc((size_t)size);
if (!d) { fclose(f); return (el_val_t)0; }
size_t got = fread(d, 1, (size_t)size, f);
fclose(f);
if (got != (size_t)size) { free(d); return (el_val_t)0; }
if (memcmp(d, "RIFF", 4) != 0 || memcmp(d + 8, "WAVE", 4) != 0) { free(d); return (el_val_t)0; }
int32_t sr = 0, channels = 0, bits = 0;
long dataOff = -1, dataLen = 0;
long o = 12;
while (o + 8 <= size) {
long sz = (long)d[o+4] | ((long)d[o+5] << 8) | ((long)d[o+6] << 16) | ((long)d[o+7] << 24);
if (sz < 0) break;
if (memcmp(d + o, "fmt ", 4) == 0 && o + 24 <= size) {
channels = (int32_t)(d[o+10] | (d[o+11] << 8));
sr = (int32_t)((long)d[o+12] | ((long)d[o+13] << 8) | ((long)d[o+14] << 16) | ((long)d[o+15] << 24));
bits = (int32_t)(d[o+22] | (d[o+23] << 8));
} else if (memcmp(d + o, "data", 4) == 0) {
dataOff = o + 8;
dataLen = sz;
if (dataOff + dataLen > size) dataLen = size - dataOff;
}
o += 8 + sz + (sz & 1);
}
if (dataOff < 0 || sr <= 0 || bits != 16 || channels < 1 || dataLen <= 0) { free(d); return (el_val_t)0; }
long frames = dataLen / (2 * channels);
int16_t* pcm = (int16_t*)malloc((size_t)frames * sizeof(int16_t));
if (!pcm) { free(d); return (el_val_t)0; }
/* Take channel 0; the organ is mono by design and downmixing would be an
* opinion about content this layer is not entitled to have. */
for (long i = 0; i < frames; i++) {
long b = dataOff + i * 2 * channels;
pcm[i] = (int16_t)((unsigned)d[b] | ((unsigned)d[b+1] << 8));
}
free(d);
int ok = el_audio_play_raw(pcm, frames, sr);
free(pcm);
return (el_val_t)(ok ? 1 : 0);
}
lang/runtime/el_capture_darwin.m
+841
View File
@@ -0,0 +1,841 @@
/* el_capture_darwin.m — the MICROPHONE and CAMERA realizers. El's afferent
* organ on Darwin: the two entry points through which the world gets in.
*
* WHY THIS FILE EXISTS, AND WHY IT IS A REALIZER RATHER THAN PURE EL.
*
* el_audio_darwin.m argued the efferent half of this: El can compute a sound
* but it cannot make one, because "the DAC has now played these samples" is not
* a fact any amount of arithmetic can produce. This file is the same argument
* run backwards. El can compute *about* a sound — it can window it, take its
* autocorrelation, run Levinson-Durbin over that, find the formant peaks in the
* resulting all-pole envelope, and hand back a voiceprint — but it cannot ASK.
* There is no expression in El, and there must not be, whose value is "the next
* 1024 frames the microphone hears" or "what the camera is pointed at right
* now." Those are not computed; they are *requested*, from an operating system
* that owns the device, mediates consent for it, and delivers the answer on a
* thread of its choosing whenever it feels like it. Asking is the one primitive
* operation here. Everything else in this file is bookkeeping around the ask.
*
* So the line is drawn exactly where el_audio_darwin.m drew it, at the sample
* buffer, and it is drawn on purpose:
*
* BELOW the line (here): open the device, honour the OS permission gate,
* install a tap or a frame delegate, convert whatever the hardware happens to
* emit into the one shape El asked for, and hand it up. No opinions about
* content. No analysis. No decisions.
*
* ABOVE the line (El): energy, zero-crossing rate, spectral centroid, F0 by
* autocorrelation, LPC, formants F1-F5, the compact descriptors, the
* scene-geometry grid, the yield-or-hold turn-taking decision. All of it is
* arithmetic over a buffer, all of it belongs in El, and none of it appears
* below. The reference this file ports — peripheral/src/periph.swift — held
* both halves, and that was the problem worth fixing: the descriptors were
* trapped in a 939-line binary standing next to the language instead of being
* written in it. Porting the *whole* of periph.swift down here would have
* reproduced that mistake in C. Only the ask came down.
*
* The precedent for the file's SHAPE is eg_cosine_batch_strategy_metal_hand.m:
* a platform-bound capability compiled as its own translation unit, declared in
* el_runtime.h, linked in where the platform supports it, and deliberately NOT
* a patch to the middle of el_runtime.c. Acquiring a device must not mean
* editing the language, for the same reason acquiring a modality must not (see
* the realizer registry: organs are resolved by name). el_peripheral_null.c is
* the same entry points everywhere else, so El code that listens still links on
* every platform and merely reports having no ear.
*
* FAIL CLOSED, ALWAYS.
*
* A capture path that returns plausible-looking zeros when it was denied is
* worse than one that returns nothing, because the caller cannot tell the
* difference between a silent room and a refused microphone. Every entry point
* here checks AVCaptureDevice's authorization status BEFORE touching hardware
* and returns the empty value — an empty list, a 0 map — on anything short of
* .authorized. mic_available() and camera_available() report that state WITHOUT
* prompting, so El can ask "may I?" without the act of asking being a prompt.
*
* NEVER HANG.
*
* Every wait in this file is bounded: 30s on a permission prompt (the user has
* to walk to the dialog), seconds+5 on a capture of `seconds`, 10s on a camera
* frame. An organ that wedges the program holding it is not an organ, it is a
* fault. Every path also tears the device down on the way out, including the
* failure paths, so a timed-out capture does not leave the mic light on.
*
* OWN-CORE AND LOCAL BY CONSTRUCTION.
*
* AVFoundation, CoreVideo, CoreGraphics and ImageIO ship with macOS. There is
* no third-party library here, no model, and — the part that matters — no
* network path of any kind. Samples and pixels move from local hardware into an
* El value and stop. periph.swift had a URLSession in it; this file has no
* socket, no URL, and nothing that could grow one without being obvious in
* review. Consent is enforced above this layer in El and below it by the OS;
* this layer's whole contribution to that is refusing to proceed.
*
* DISCLOSURE.
*
* Every actual device touch writes one line to stderr and flushes it, before
* the device opens. stderr and not stdout: a program that announces "I am about
* to open the microphone" on stdout has corrupted its own output, and the
* caller must be able to separate the answer from how it was obtained. One line
* per touch, no more — a disclosure rail that spams is a rail people learn to
* ignore.
*/
#if defined(__APPLE__)
#import <Foundation/Foundation.h>
#import <AVFoundation/AVFoundation.h>
#import <CoreMedia/CoreMedia.h>
#import <CoreVideo/CoreVideo.h>
#import <CoreGraphics/CoreGraphics.h>
#import <ImageIO/ImageIO.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <unistd.h>
#include <pthread.h>
#include "el_runtime.h"
/* ── Disclosure ──────────────────────────────────────────────────────────────
* One flushed line per real device touch, on stderr. Flushed rather than
* buffered so the line reaches the terminal BEFORE the mic light comes on
* rather than whenever the buffer happens to drain. */
static void el_cap_disclose(const char* what) {
fprintf(stderr, " [peripheral] %s\n", what);
fflush(stderr);
}
/* ── Permission ──────────────────────────────────────────────────────────────
* authorizationStatus is a pure read of the TCC database: it never prompts and
* never blocks, which is what lets mic_available()/camera_available() answer
* honestly without the question itself becoming an event. requestAccess DOES
* prompt, so it lives behind its own entry point and nothing calls it
* implicitly. */
static int el_cap_authorized(AVMediaType media) {
@try {
return [AVCaptureDevice authorizationStatusForMediaType:media]
== AVAuthorizationStatusAuthorized ? 1 : 0;
} @catch (NSException* e) {
(void)e;
return 0;
}
}
static int el_cap_device_present(AVMediaType media) {
@try {
return [AVCaptureDevice defaultDeviceWithMediaType:media] != nil ? 1 : 0;
} @catch (NSException* e) {
(void)e;
return 0;
}
}
/* Prompt once and wait, bounded. 30 seconds is the same budget periph.swift
* used: long enough for a human to notice a dialog and decide, short enough
* that an unattended run fails rather than parks forever. A timeout is reported
* as "not granted", which is the safe reading — we genuinely do not know that
* it was. */
static int el_cap_request(AVMediaType media) {
__block int granted = 0;
dispatch_semaphore_t sem = dispatch_semaphore_create(0);
@try {
[AVCaptureDevice requestAccessForMediaType:media
completionHandler:^(BOOL ok) {
granted = ok ? 1 : 0;
dispatch_semaphore_signal(sem);
}];
} @catch (NSException* e) {
(void)e;
return 0;
}
if (dispatch_semaphore_wait(sem,
dispatch_time(DISPATCH_TIME_NOW, (int64_t)(30 * NSEC_PER_SEC))) != 0) {
return 0; /* timed out — treat as refused */
}
return granted;
}
/* ════════════════════════════════════════════════════════════════════════════
* MICROPHONE — one-shot capture
* ══════════════════════════════════════════════════════════════════════════ */
/* The sink the tap block writes into. An object rather than a static so two
* captures can never share state, and so ARC keeps it alive for exactly as long
* as the block that captured it. The lock is real, not decorative: the tap runs
* on an AVAudioEngine-internal thread and the waiter runs on the caller's. */
@interface ElCapMicSink : NSObject
@property (nonatomic, strong) NSMutableData* pcm;
@property (nonatomic, strong) NSLock* lock;
@end
@implementation ElCapMicSink
- (instancetype)init {
self = [super init];
if (self) {
_pcm = [NSMutableData data];
_lock = [[NSLock alloc] init];
}
return self;
}
@end
/* Capture `seconds` of mono 16-bit PCM at `sample_rate`.
*
* The hardware format is NOT assumed. A built-in mic will typically hand back
* float32 at 44.1 or 48 kHz, an aggregate device may be 8 channels at 96 kHz,
* and a caller asking for 16 kHz mono (which is what the formant path wants)
* gets 16 kHz mono either way. AVAudioConverter does the rate conversion and
* the downmix; doing it by hand would mean writing a resampler in the one file
* that is supposed to contain no arithmetic.
*
* The converter is built ONCE, outside the tap, because a sample-rate converter
* carries filter state across buffers — rebuilding it per callback would put a
* discontinuity at every buffer boundary, which is audible and which would then
* show up in El's spectral descriptors as energy that was never in the room. */
el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate) {
el_val_t empty = el_list_empty();
if (!el_cap_authorized(AVMediaTypeAudio)) return empty;
int64_t secs = (int64_t)seconds;
int64_t sr = (int64_t)sample_rate;
if (secs <= 0 || sr <= 0) return empty;
/* Bound the ask. A caller that asks for a year of audio has made a mistake,
* and honouring it would mean an unkillable capture and an OOM. */
if (secs > 300) secs = 300;
if (sr > 384000) sr = 384000;
__block AVAudioEngine* engine = nil;
AVAudioInputNode* input = nil;
AVAudioFormat* hwFmt = nil;
int tapped = 0;
@try {
engine = [[AVAudioEngine alloc] init];
input = [engine inputNode];
hwFmt = [input inputFormatForBus:0];
} @catch (NSException* e) {
(void)e;
return empty;
}
if (!input || !hwFmt || hwFmt.sampleRate <= 0 || hwFmt.channelCount == 0) {
return empty;
}
/* Preferred target: mono int16 at the requested rate. If the converter
* refuses that pairing (some exotic input layouts will not downmix), fall
* back to keeping the hardware's channel count and taking channel 0 on the
* way out — the organ is mono by design and inventing a downmix here would
* be an opinion about content this layer is not entitled to have. */
AVAudioFormat* outFmt =
[[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16
sampleRate:(double)sr
channels:1
interleaved:YES];
AVAudioConverter* conv = outFmt ? [[AVAudioConverter alloc] initFromFormat:hwFmt
toFormat:outFmt] : nil;
AVAudioChannelCount outCh = 1;
if (!conv) {
outFmt = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16
sampleRate:(double)sr
channels:hwFmt.channelCount
interleaved:YES];
conv = outFmt ? [[AVAudioConverter alloc] initFromFormat:hwFmt toFormat:outFmt] : nil;
outCh = hwFmt.channelCount;
}
if (!conv || !outFmt) return empty;
ElCapMicSink* sink = [[ElCapMicSink alloc] init];
const int64_t want = secs * sr; /* frames we are waiting for */
{
char msg[192];
snprintf(msg, sizeof(msg),
"MIC: opening the microphone for %llds -> %lld Hz mono PCM "
"(local, never egresses).", (long long)secs, (long long)sr);
el_cap_disclose(msg);
}
const double ratio = (double)sr / hwFmt.sampleRate;
@try {
[input installTapOnBus:0
bufferSize:4096
format:hwFmt
block:^(AVAudioPCMBuffer* _Nonnull buf, AVAudioTime* _Nonnull when) {
(void)when;
if (!buf || buf.frameLength == 0) return;
AVAudioFrameCount cap =
(AVAudioFrameCount)((double)buf.frameLength * ratio) + 1024;
AVAudioPCMBuffer* out =
[[AVAudioPCMBuffer alloc] initWithPCMFormat:outFmt frameCapacity:cap];
if (!out) return;
__block BOOL fed = NO;
AVAudioConverterInputBlock feed =
^AVAudioBuffer* _Nullable (AVAudioPacketCount need,
AVAudioConverterInputStatus* _Nonnull status) {
(void)need;
if (fed) { *status = AVAudioConverterInputStatus_NoDataNow; return nil; }
fed = YES;
*status = AVAudioConverterInputStatus_HaveData;
return buf;
};
NSError* err = nil;
AVAudioConverterOutputStatus st =
[conv convertToBuffer:out error:&err withInputFromBlock:feed];
if (st == AVAudioConverterOutputStatus_Error || out.frameLength == 0) return;
const int16_t* src = out.int16ChannelData ? out.int16ChannelData[0] : NULL;
if (!src) return;
NSUInteger n = (NSUInteger)out.frameLength;
[sink.lock lock];
if (outCh == 1) {
[sink.pcm appendBytes:src length:n * sizeof(int16_t)];
} else {
/* Interleaved: stride to channel 0. */
for (NSUInteger i = 0; i < n; i++) {
int16_t v = src[i * outCh];
[sink.pcm appendBytes:&v length:sizeof(int16_t)];
}
}
[sink.lock unlock];
}];
tapped = 1;
[engine prepare];
NSError* startErr = nil;
if (![engine startAndReturnError:&startErr]) {
[input removeTapOnBus:0];
return empty;
}
} @catch (NSException* e) {
(void)e;
@try { if (tapped) [input removeTapOnBus:0]; } @catch (NSException* e2) { (void)e2; }
@try { [engine stop]; } @catch (NSException* e2) { (void)e2; }
return empty;
}
/* Wait for `want` frames, bounded by the material's own duration plus a
* margin. A device that stops producing must not become a hang. */
const int64_t deadline_us = (secs + 5) * 1000000;
int64_t waited_us = 0;
const int64_t tick_us = 5000;
for (;;) {
[sink.lock lock];
int64_t have = (int64_t)([sink.pcm length] / sizeof(int16_t));
[sink.lock unlock];
if (have >= want || waited_us >= deadline_us) break;
usleep((useconds_t)tick_us);
waited_us += tick_us;
}
@try { [input removeTapOnBus:0]; } @catch (NSException* e) { (void)e; }
@try { [engine stop]; } @catch (NSException* e) { (void)e; }
engine = nil;
/* Hand up exactly what was asked for, or everything we got if the device
* came up short. Never padded: silence we invented is indistinguishable
* from silence we heard, and El has no way to tell them apart afterwards. */
[sink.lock lock];
int64_t have = (int64_t)([sink.pcm length] / sizeof(int16_t));
int64_t n = have < want ? have : want;
const int16_t* pcm = (const int16_t*)[sink.pcm bytes];
el_val_t list = empty;
for (int64_t i = 0; i < n; i++) {
list = el_list_append(list, (el_val_t)(int64_t)pcm[i]);
}
[sink.lock unlock];
return list;
}
el_val_t mic_available(void) {
if (!el_cap_authorized(AVMediaTypeAudio)) return (el_val_t)0;
return (el_val_t)(el_cap_device_present(AVMediaTypeAudio) ? 1 : 0);
}
el_val_t mic_request_access(void) {
return (el_val_t)(el_cap_request(AVMediaTypeAudio) ? 1 : 0);
}
/* ════════════════════════════════════════════════════════════════════════════
* MICROPHONE — live monitor (the full-duplex ear)
*
* converse needs to keep listening WHILE it speaks, which means the mic is open
* at the same time as the speaker. In a real room that is a feedback path:
* without cancellation Neuron hears its own voice, decides someone is talking,
* and barges in on itself. setVoiceProcessingEnabled: hands the input node to
* the OS voice-processing unit, which subtracts the known output signal from
* the input — the single thing that makes barge-in work outside a headset.
*
* It is not always available (some aggregate and virtual devices refuse it), so
* failure to enable it is reported as a DISTINCT return value (2) rather than
* folded into success. The caller needs to know, because the correct response
* is to raise the VAD floor, and a caller that thinks AEC is on will set that
* floor far too low.
*
* The tap keeps only a running short-window RMS in a static behind a mutex.
* Deliberately not a queue of samples: this path is polled at ~50 Hz by a loop
* that only ever asks "is someone talking", and buffering audio nobody reads
* would be an unbounded allocation in the middle of a conversation.
* ══════════════════════════════════════════════════════════════════════════ */
static AVAudioEngine* g_mon_engine = nil;
static int g_mon_running = 0;
static int g_mon_code = 0; /* what the successful start reported */
static double g_mon_rms = 0.0;
static pthread_mutex_t g_mon_lock = PTHREAD_MUTEX_INITIALIZER;
el_val_t mic_monitor_start(void) {
/* Idempotent, and it re-reports the ORIGINAL code rather than a bare 1: a
* caller that starts twice must not be told AEC is on when the first start
* already discovered it was not. */
if (g_mon_running) return (el_val_t)g_mon_code;
if (!el_cap_authorized(AVMediaTypeAudio)) return (el_val_t)0;
AVAudioEngine* engine = nil;
AVAudioInputNode* input = nil;
AVAudioFormat* fmt = nil;
int aec = 0;
int tapped = 0;
@try {
engine = [[AVAudioEngine alloc] init];
input = [engine inputNode];
} @catch (NSException* e) {
(void)e;
return (el_val_t)0;
}
if (!input) return (el_val_t)0;
/* Enable AEC BEFORE reading the format: the voice-processing unit imposes
* its own input format, and a tap installed with the pre-VP format would be
* rejected at start. */
@try {
NSError* vpErr = nil;
if ([input respondsToSelector:@selector(setVoiceProcessingEnabled:error:)]) {
aec = [input setVoiceProcessingEnabled:YES error:&vpErr] ? 1 : 0;
}
} @catch (NSException* e) {
(void)e;
aec = 0;
}
@try {
fmt = [input inputFormatForBus:0];
} @catch (NSException* e) {
(void)e;
return (el_val_t)0;
}
if (!fmt || fmt.sampleRate <= 0 || fmt.channelCount == 0) return (el_val_t)0;
el_cap_disclose(aec
? "MIC: opening the microphone for live monitoring, echo-cancelled (local)."
: "MIC: opening the microphone for live monitoring, NO echo cancellation (local).");
@try {
[input installTapOnBus:0
bufferSize:1024
format:fmt
block:^(AVAudioPCMBuffer* _Nonnull buf, AVAudioTime* _Nonnull when) {
(void)when;
if (!buf) return;
AVAudioFrameCount n = buf.frameLength;
if (n == 0) return;
double sum = 0.0;
/* Whatever the VP unit hands back — float32 is the norm, int16 and
* int32 are possible on odd hardware — normalise to -1..1 so the
* Float El sees means the same thing on every device. */
if (buf.floatChannelData) {
const float* ch = buf.floatChannelData[0];
for (AVAudioFrameCount i = 0; i < n; i++) sum += (double)ch[i] * (double)ch[i];
} else if (buf.int16ChannelData) {
const int16_t* ch = buf.int16ChannelData[0];
for (AVAudioFrameCount i = 0; i < n; i++) {
double v = (double)ch[i] / 32768.0;
sum += v * v;
}
} else if (buf.int32ChannelData) {
const int32_t* ch = buf.int32ChannelData[0];
for (AVAudioFrameCount i = 0; i < n; i++) {
double v = (double)ch[i] / 2147483648.0;
sum += v * v;
}
} else {
return;
}
double rms = sqrt(sum / (double)n);
if (rms < 0.0) rms = 0.0;
if (rms > 1.0) rms = 1.0;
pthread_mutex_lock(&g_mon_lock);
g_mon_rms = rms;
pthread_mutex_unlock(&g_mon_lock);
}];
tapped = 1;
[engine prepare];
NSError* startErr = nil;
if (![engine startAndReturnError:&startErr]) {
[input removeTapOnBus:0];
return (el_val_t)0;
}
} @catch (NSException* e) {
(void)e;
@try { if (tapped) [input removeTapOnBus:0]; } @catch (NSException* e2) { (void)e2; }
@try { [engine stop]; } @catch (NSException* e2) { (void)e2; }
return (el_val_t)0;
}
pthread_mutex_lock(&g_mon_lock);
g_mon_rms = 0.0;
pthread_mutex_unlock(&g_mon_lock);
g_mon_engine = engine;
g_mon_running = 1;
g_mon_code = aec ? 1 : 2;
return (el_val_t)g_mon_code;
}
/* Float in 0..1. Reads the last window the tap computed; never blocks on the
* audio thread beyond the mutex, because this is polled inside a turn-taking
* loop where a stall IS a missed barge-in. */
el_val_t mic_monitor_rms(void) {
double rms = 0.0;
pthread_mutex_lock(&g_mon_lock);
rms = g_mon_rms;
pthread_mutex_unlock(&g_mon_lock);
return el_from_float(rms);
}
el_val_t mic_monitor_stop(void) {
AVAudioEngine* engine = g_mon_engine;
g_mon_engine = nil;
g_mon_running = 0;
g_mon_code = 0;
if (engine) {
@try { [[engine inputNode] removeTapOnBus:0]; } @catch (NSException* e) { (void)e; }
@try { [engine stop]; } @catch (NSException* e) { (void)e; }
}
pthread_mutex_lock(&g_mon_lock);
g_mon_rms = 0.0;
pthread_mutex_unlock(&g_mon_lock);
return (el_val_t)1;
}
/* ════════════════════════════════════════════════════════════════════════════
* CAMERA
* ══════════════════════════════════════════════════════════════════════════ */
static void el_cap_free_bitmap(void* info, const void* data, size_t size) {
(void)info; (void)size;
free((void*)data);
}
/* CVPixelBuffer -> CGImage, own-core, no CoreImage.
*
* The output is pinned to 32BGRA at the AVCaptureVideoDataOutput (see below)
* precisely so this conversion can be a memcpy and a CGImageCreate. The
* alternative — accepting the camera's native 2vuy/420v and colour-converting
* here — would mean either pulling in CoreImage or writing a YUV->RGB matrix in
* the file that is supposed to contain no arithmetic. Asking the capture output
* for BGRA moves that work into AVFoundation, where it is already written and
* already hardware-accelerated.
*
* The rows are copied out rather than aliased because the CVPixelBuffer is
* recycled by the capture session the moment the delegate returns; a CGImage
* pointing at it would be pointing at the NEXT frame by the time anyone looked. */
static CGImageRef el_cap_cgimage_from_pixelbuffer(CVPixelBufferRef pb) {
if (!pb) return NULL;
if (CVPixelBufferGetPixelFormatType(pb) != kCVPixelFormatType_32BGRA) return NULL;
if (CVPixelBufferLockBaseAddress(pb, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) return NULL;
size_t w = CVPixelBufferGetWidth(pb);
size_t h = CVPixelBufferGetHeight(pb);
size_t src_bpr = CVPixelBufferGetBytesPerRow(pb);
const uint8_t* base = (const uint8_t*)CVPixelBufferGetBaseAddress(pb);
CGImageRef img = NULL;
if (base && w > 0 && h > 0 && src_bpr >= w * 4) {
size_t dst_bpr = w * 4;
uint8_t* copy = (uint8_t*)malloc(dst_bpr * h);
if (copy) {
for (size_t y = 0; y < h; y++) {
memcpy(copy + y * dst_bpr, base + y * src_bpr, dst_bpr);
}
CGDataProviderRef dp =
CGDataProviderCreateWithData(NULL, copy, dst_bpr * h, el_cap_free_bitmap);
if (dp) {
CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB();
if (cs) {
img = CGImageCreate(w, h, 8, 32, dst_bpr, cs,
(CGBitmapInfo)(kCGBitmapByteOrder32Little |
kCGImageAlphaNoneSkipFirst),
dp, NULL, false, kCGRenderingIntentDefault);
CGColorSpaceRelease(cs);
}
CGDataProviderRelease(dp); /* provider owns `copy` from here */
} else {
free(copy);
}
}
}
CVPixelBufferUnlockBaseAddress(pb, kCVPixelBufferLock_ReadOnly);
return img;
}
/* The frame delegate. AVCaptureVideoDataOutput is used rather than
* AVCapturePhotoOutput for the same reason periph.swift used it: the photo path
* wants KVO and a session owned by an app object, and this runs in a plain CLI
* process with no run loop it can assume. A data output just calls back.
*
* The first frames are dropped on purpose. A camera that has just been powered
* on is still converging exposure and white balance, and the first frame is
* reliably darker and greener than the room. El's scene-geometry descriptors
* are brightness and mean-colour statistics, so handing up an unsettled frame
* would not produce a slightly worse answer, it would produce a confidently
* wrong one. */
@interface ElCapFrameGrabber : NSObject <AVCaptureVideoDataOutputSampleBufferDelegate> {
CGImageRef _img;
int _seen;
dispatch_semaphore_t _sem;
}
- (dispatch_semaphore_t)sem;
- (CGImageRef)takeImage; /* transfers ownership to the caller */
@end
@implementation ElCapFrameGrabber
- (instancetype)init {
self = [super init];
if (self) {
_img = NULL;
_seen = 0;
_sem = dispatch_semaphore_create(0);
}
return self;
}
- (dispatch_semaphore_t)sem { return _sem; }
- (CGImageRef)takeImage {
CGImageRef out = _img;
_img = NULL;
return out;
}
- (void)dealloc {
if (_img) { CGImageRelease(_img); _img = NULL; }
}
/* Runs on the serial delegate queue, so no lock is needed among callbacks; the
* waiter only reads _img after the semaphore has been signalled AND the session
* has been stopped, which orders it after the last callback. */
- (void)captureOutput:(AVCaptureOutput*)output
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
fromConnection:(AVCaptureConnection*)connection {
(void)output; (void)connection;
_seen++;
if (_img != NULL || _seen < 5) return; /* let exposure settle */
CVImageBufferRef pb = CMSampleBufferGetImageBuffer(sampleBuffer);
if (!pb) return;
CGImageRef img = el_cap_cgimage_from_pixelbuffer(pb);
if (!img) return;
_img = img;
dispatch_semaphore_signal(_sem);
}
@end
/* Bring the camera up, take exactly one settled frame, put it back down.
* Returns a +1 CGImageRef the caller releases, or NULL. Bounded at 10s: a
* camera held by another process, or one whose TCC grant was revoked between
* the check and the open, must fail rather than park. */
static CGImageRef el_cap_grab_frame(void) {
AVCaptureSession* session = nil;
AVCaptureVideoDataOutput* output = nil;
ElCapFrameGrabber* grabber = nil;
CGImageRef img = NULL;
@try {
AVCaptureDevice* dev = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
if (!dev) return NULL;
NSError* err = nil;
AVCaptureDeviceInput* in = [AVCaptureDeviceInput deviceInputWithDevice:dev error:&err];
if (!in) return NULL;
session = [[AVCaptureSession alloc] init];
session.sessionPreset = AVCaptureSessionPresetPhoto;
if (![session canAddInput:in]) return NULL;
[session addInput:in];
output = [[AVCaptureVideoDataOutput alloc] init];
output.alwaysDiscardsLateVideoFrames = YES;
/* Pin the pixel format so the CGImage conversion above stays a memcpy.
* Every macOS capture device advertises 32BGRA. */
output.videoSettings = @{ (id)kCVPixelBufferPixelFormatTypeKey :
@(kCVPixelFormatType_32BGRA) };
grabber = [[ElCapFrameGrabber alloc] init];
dispatch_queue_t q = dispatch_queue_create("el.capture.camera", DISPATCH_QUEUE_SERIAL);
[output setSampleBufferDelegate:grabber queue:q];
if (![session canAddOutput:output]) return NULL;
[session addOutput:output];
el_cap_disclose("CAMERA: opening the camera for one frame (local, never egresses).");
[session startRunning];
} @catch (NSException* e) {
(void)e;
@try { [session stopRunning]; } @catch (NSException* e2) { (void)e2; }
return NULL;
}
long timed_out = dispatch_semaphore_wait([grabber sem],
dispatch_time(DISPATCH_TIME_NOW, (int64_t)(10 * NSEC_PER_SEC)));
/* Stop first, then detach the delegate, then read. In that order the last
* callback has already returned by the time anyone touches the image. */
@try { [session stopRunning]; } @catch (NSException* e) { (void)e; }
@try { [output setSampleBufferDelegate:nil queue:NULL]; } @catch (NSException* e) { (void)e; }
if (timed_out == 0) img = [grabber takeImage];
return img;
}
el_val_t camera_available(void) {
if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0;
return (el_val_t)(el_cap_device_present(AVMediaTypeVideo) ? 1 : 0);
}
el_val_t camera_request_access(void) {
return (el_val_t)(el_cap_request(AVMediaTypeVideo) ? 1 : 0);
}
/* Longest edge of the grid handed to El. 64 is not a resolution, it is a budget:
* a 1920x1080 frame is 6.2 MILLION packed RGB ints, and building that as an El
* list would cost more time and memory than everything El then does with it.
* The descriptors El computes over this — mean colour, brightness, a 3x3
* luminance grid — are region statistics, and region statistics do not get
* meaningfully better above a 64-wide grid. The TRUE frame dimensions are
* reported separately so nothing downstream has to guess what was thrown away. */
#define EL_CAP_GRID_MAX 64
/* One frame as Map{width, height, grid_w, grid_h, pixels:[Int]}.
*
* "pixels" is packed R,G,B with NO alpha — three ints per grid cell, row-major
* from the TOP-LEFT. (CGBitmapContext lays its buffer out top row first and
* CGContextDrawImage does the flip, so row 0 here is the top of the frame, the
* same convention periph.swift's grid indexing assumed.) Alpha is dropped
* because a camera frame has none worth carrying and it would inflate the list
* by a third to say "opaque" six thousand times. */
el_val_t camera_capture_rgb(void) {
if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0;
CGImageRef img = el_cap_grab_frame();
if (!img) return (el_val_t)0;
size_t w = CGImageGetWidth(img);
size_t h = CGImageGetHeight(img);
if (w == 0 || h == 0) { CGImageRelease(img); return (el_val_t)0; }
/* Preserve aspect ratio, longest edge capped. */
size_t gw = w, gh = h;
size_t longest = w > h ? w : h;
if (longest > EL_CAP_GRID_MAX) {
double s = (double)EL_CAP_GRID_MAX / (double)longest;
gw = (size_t)((double)w * s + 0.5);
gh = (size_t)((double)h * s + 0.5);
if (gw == 0) gw = 1;
if (gh == 0) gh = 1;
}
size_t bpr = gw * 4;
uint8_t* buf = (uint8_t*)calloc(1, bpr * gh);
if (!buf) { CGImageRelease(img); return (el_val_t)0; }
CGColorSpaceRef cs = CGColorSpaceCreateDeviceRGB();
CGContextRef ctx = cs ? CGBitmapContextCreate(buf, gw, gh, 8, bpr, cs,
(CGBitmapInfo)kCGImageAlphaPremultipliedLast)
: NULL;
if (cs) CGColorSpaceRelease(cs);
if (!ctx) { free(buf); CGImageRelease(img); return (el_val_t)0; }
/* Nearest-neighbour. This is a decimation for statistics, not a thumbnail
* for a human to look at; smoothing would only cost time and blur the very
* region boundaries the grid exists to measure. */
CGContextSetInterpolationQuality(ctx, kCGInterpolationNone);
CGContextDrawImage(ctx, CGRectMake(0, 0, (CGFloat)gw, (CGFloat)gh), img);
CGContextRelease(ctx);
CGImageRelease(img);
el_val_t pixels = el_list_empty();
for (size_t y = 0; y < gh; y++) {
const uint8_t* row = buf + y * bpr;
for (size_t x = 0; x < gw; x++) {
const uint8_t* p = row + x * 4; /* RGBA8, premultiplied-last */
pixels = el_list_append(pixels, (el_val_t)(int64_t)p[0]);
pixels = el_list_append(pixels, (el_val_t)(int64_t)p[1]);
pixels = el_list_append(pixels, (el_val_t)(int64_t)p[2]);
}
}
free(buf);
el_val_t m = el_map_new((el_val_t)0);
if (!m) return (el_val_t)0;
m = el_map_set(m, EL_STR("width"), (el_val_t)(int64_t)w);
m = el_map_set(m, EL_STR("height"), (el_val_t)(int64_t)h);
m = el_map_set(m, EL_STR("grid_w"), (el_val_t)(int64_t)gw);
m = el_map_set(m, EL_STR("grid_h"), (el_val_t)(int64_t)gh);
m = el_map_set(m, EL_STR("pixels"), pixels);
return m;
}
/* One frame to disk as JPEG, at FULL resolution — the opposite budget from
* camera_capture_rgb, and for the opposite reason. A file is not being walked
* element-by-element by an interpreter; it costs one ImageIO call and it is the
* artefact a human or a later pass will actually look at. The encoder is
* ImageIO's because a JPEG encoder is a codec, and re-implementing one in El
* would be a large amount of arithmetic that buys nothing: the point of keeping
* work in El is the reasoning, not the entropy coding. */
el_val_t camera_capture_jpeg(el_val_t path) {
const char* p = EL_CSTR(path);
if (!p || !*p) return (el_val_t)0;
if (!el_cap_authorized(AVMediaTypeVideo)) return (el_val_t)0;
CGImageRef img = el_cap_grab_frame();
if (!img) return (el_val_t)0;
int ok = 0;
@autoreleasepool {
NSString* ns = [NSString stringWithUTF8String:p];
NSURL* url = ns ? [NSURL fileURLWithPath:ns] : nil;
if (url) {
CGImageDestinationRef dst =
CGImageDestinationCreateWithURL((__bridge CFURLRef)url, CFSTR("public.jpeg"), 1, NULL);
if (dst) {
CGImageDestinationAddImage(dst, img, NULL);
ok = CGImageDestinationFinalize(dst) ? 1 : 0;
CFRelease(dst);
}
}
}
CGImageRelease(img);
return (el_val_t)ok;
}
#endif /* __APPLE__ */
lang/runtime/el_peripheral_null.c
+89
View File
@@ -0,0 +1,89 @@
/* el_peripheral_null.c — the no-device build of El's I/O organ.
*
* Every entry point declared in el_runtime.h's "Peripheral" block, implemented
* as an honest refusal. This is what a platform without an El audio/capture
* realizer links instead of el_audio_darwin.m + el_capture_darwin.m, so an El
* program that speaks or listens still COMPILES AND LINKS everywhere.
*
* The distinction that matters: these do not pretend. speaker_available() and
* mic_available() return 0, and every operation returns its failure sentinel.
* A program asking "can I speak here?" gets a truthful no, rather than a
* silence it would have to infer something from. Silent success is the failure
* mode this whole change exists to eliminate — El spent this entire codebase's
* history writing WAV files full of zeros and reporting ok=true, and nobody
* caught it because nothing ever said "there is no sound here".
*
* Compiled INSTEAD OF the Darwin realizers, never alongside them — the symbols
* are the same by design, which is the point: the El side never branches on
* platform, it branches on speaker_available().
*/
#include "el_runtime.h"
#if !defined(__APPLE__)
/* ── Speaker ─────────────────────────────────────────────────────────────── */
el_val_t speaker_available(void) { return (el_val_t)0; }
el_val_t speaker_name(void) { return EL_STR("none"); }
el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate) {
(void)samples; (void)sample_rate;
return (el_val_t)0;
}
el_val_t speaker_play_wav(el_val_t path) {
(void)path;
return (el_val_t)0;
}
el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate) {
(void)samples; (void)sample_rate;
return (el_val_t)0;
}
el_val_t speaker_play_wav_async(el_val_t path) {
(void)path;
return (el_val_t)0;
}
el_val_t speaker_pause(void) { return (el_val_t)0; }
el_val_t speaker_resume(void) { return (el_val_t)0; }
el_val_t speaker_playing(void) { return (el_val_t)0; }
el_val_t speaker_stop(void) { return (el_val_t)0; }
el_val_t speaker_played_frames(void) { return (el_val_t)0; }
/* WAV geometry is pure parsing and would work fine here, but reporting a
* duration for audio this build cannot play would invite a caller to sequence
* around a silence. Refuse consistently with the rest of the file. */
el_val_t wav_frames(el_val_t path) { (void)path; return (el_val_t)0; }
el_val_t wav_rate(el_val_t path) { (void)path; return (el_val_t)0; }
/* ── Microphone ──────────────────────────────────────────────────────────── */
el_val_t mic_available(void) { return (el_val_t)0; }
el_val_t mic_request_access(void) { return (el_val_t)0; }
/* Empty list, not 0: the contract says capture returns samples, and a caller
* iterating the result must find nothing rather than dereference a non-list. */
el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate) {
(void)seconds; (void)sample_rate;
return el_list_empty();
}
el_val_t mic_monitor_start(void) { return (el_val_t)0; }
el_val_t mic_monitor_rms(void) { return el_from_float(0.0); }
el_val_t mic_monitor_stop(void) { return (el_val_t)0; }
/* ── Camera ──────────────────────────────────────────────────────────────── */
el_val_t camera_available(void) { return (el_val_t)0; }
el_val_t camera_request_access(void) { return (el_val_t)0; }
el_val_t camera_capture_rgb(void) { return (el_val_t)0; }
el_val_t camera_capture_jpeg(el_val_t path) {
(void)path;
return (el_val_t)0;
}
#endif /* !__APPLE__ */
lang/runtime/el_runtime.c
+3184 -229
View File
File diff suppressed because it is too large Load Diff
lang/runtime/el_runtime.h
+390 -2
View File
@@ -80,6 +80,92 @@ void println(el_val_t s);
void print(el_val_t s);
el_val_t readline(void);
/* stderr counterpart of println (defined in el_seed.c). El could write to
* stdout and nowhere else, which is right for a program's RESULT and wrong for
* everything about how that result was produced. Disclosure especially has to
* leave on a stream the caller can separate from the answer: a program that
* announces "I am about to open the microphone" on stdout has corrupted its own
* output. Flushed on every call, so a disclosure reaches the terminal BEFORE
* the device it describes is touched rather than whenever the buffer drains. */
void eprintln(el_val_t s);
/* ── Peripheral: the speaker, the microphone, the camera ─────────────────────
*
* El's I/O organ. Implemented per platform in its OWN translation unit —
* el_audio_darwin.m / el_capture_darwin.m on Darwin, el_peripheral_null.c
* everywhere else — so El code that speaks or listens links on every platform
* and merely reports having no device where there isn't one. Declared here and
* deliberately NOT implemented in el_runtime.c: acquiring a device must not
* mean editing the middle of the language, the same rule the realizer registry
* follows for modalities.
*
* These are the ONLY parts of the organ that are not El. Everything above the
* sample buffer — WAV encode/decode, LPC autocorrelation, Levinson-Durbin,
* formant extraction, source-filter resynthesis, the compact descriptors, the
* converse decision loop — is arithmetic, and arithmetic belongs in El. What
* remains here is what El cannot express: handing a buffer to the DAC and
* waiting for it to drain, and asking the OS for frames off a capture device.
*
* Local by construction: none of these entry points has a network path. Samples
* and pixels go to and from local hardware and nowhere else. Consent is
* enforced ABOVE this layer in El (peripheral/src/organ.el) for the Neuron-level
* grant, and BELOW it by the OS for TCC; capture fails closed on either. */
/* Speaker (efferent). speaker_play_pcm16 BLOCKS until the audio has actually
* been played rather than merely queued, so a caller can sequence utterances
* without guessing durations and without clipping each tail. */
el_val_t speaker_available(void); /* 1 if a real speaker backs this build */
el_val_t speaker_name(void); /* backend id, e.g. "coreaudio-audioqueue" */
el_val_t speaker_play_pcm16(el_val_t samples, el_val_t sample_rate); /* [Int] 16-bit mono; 1 ok */
el_val_t speaker_play_wav(el_val_t path); /* 16-bit mono RIFF/WAVE; 1 ok */
/* Asynchronous playback — required by converse, which must keep listening while
* it speaks and must be able to stop ON THE SPOT mid-buffer. A blocking play
* cannot be interrupted, and "finish the current buffer" is not barge-in.
* speaker_stop() halts output immediately; speaker_playing() reports whether
* the hardware is still going; speaker_played_frames() is how far it actually
* got, which is what makes an interrupted utterance resumable at the sample. */
el_val_t speaker_play_pcm16_async(el_val_t samples, el_val_t sample_rate);
el_val_t speaker_play_wav_async(el_val_t path);
el_val_t speaker_pause(void); /* stop AT THIS SAMPLE, keep position */
el_val_t speaker_resume(void); /* carry on from exactly there */
el_val_t speaker_playing(void);
el_val_t speaker_stop(void);
el_val_t speaker_played_frames(void);
/* WAV geometry without playing — wav-info, and the segment duration converse
* needs to turn elapsed time into progress. */
el_val_t wav_frames(el_val_t path);
el_val_t wav_rate(el_val_t path);
/* Microphone (afferent). Fails CLOSED: returns 0 unless the OS has granted
* capture access. mic_capture_pcm16 blocks for `seconds` and returns an [Int]
* of 16-bit mono samples at `sample_rate` — the raw stream is handed to El and
* never written anywhere by this layer. mic_available() reports device +
* permission state without prompting. */
el_val_t mic_available(void); /* 1 device present AND OS-authorized */
el_val_t mic_request_access(void); /* prompt once; 1 if granted */
el_val_t mic_capture_pcm16(el_val_t seconds, el_val_t sample_rate); /* [Int], empty on refusal */
/* Live monitoring for full-duplex converse. mic_monitor_start enables the OS
* voice-processing unit (acoustic echo cancellation) so the microphone does not
* hear the speaker — without AEC, Neuron barges in on its own voice and
* turn-taking is unusable in a real room. mic_monitor_rms returns the current
* short-window RMS as a Float in 0..1. */
el_val_t mic_monitor_start(void); /* 1 ok; 2 = started but AEC unavailable */
el_val_t mic_monitor_rms(void); /* Float */
el_val_t mic_monitor_stop(void);
/* Camera (afferent). Fails CLOSED like the microphone. camera_capture_rgb
* returns a Map with width/height and the frame as an [Int] of packed RGB
* bytes, so the descriptor arithmetic can happen in El rather than here.
* camera_capture_jpeg writes an encoded frame via ImageIO, which is a codec and
* not something El should re-implement. */
el_val_t camera_available(void);
el_val_t camera_request_access(void);
el_val_t camera_capture_rgb(void); /* Map{width,height,pixels:[Int]} or 0 */
el_val_t camera_capture_jpeg(el_val_t path); /* 1 ok */
/* ── String builtins ─────────────────────────────────────────────────────── */
el_val_t el_str_concat(el_val_t a, el_val_t b);
@@ -275,6 +361,10 @@ el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value);
el_val_t json_array_len(el_val_t json_str);
el_val_t json_array_get(el_val_t json_str, el_val_t index);
el_val_t json_array_get_string(el_val_t json_str, el_val_t index);
el_val_t json_escape_string(el_val_t sv);
el_val_t json_build_object(el_val_t kvs);
el_val_t json_build_array(el_val_t items);
el_val_t json_array_push(el_val_t arr_v, el_val_t elem_v); /* defined in el_runtime.c */
/* ── Time ────────────────────────────────────────────────────────────────── */
@@ -301,6 +391,8 @@ el_val_t time_diff(el_val_t ts1, el_val_t ts2, el_val_t unit);
el_val_t el_now_instant(void);
el_val_t now(void);
el_val_t now_millis(void); /* wall-clock milliseconds (defined in el_runtime.c) */
el_val_t now_ns(void); /* wall-clock nanoseconds (defined in el_runtime.c) */
el_val_t unix_seconds(el_val_t n);
el_val_t unix_millis(el_val_t n);
el_val_t instant_from_iso8601(el_val_t s);
@@ -580,6 +672,110 @@ void el_runtime_dharma_event_arrive(const char* event_type,
const char* payload,
const char* source);
/* ── Geometry: signal as a first-class El value ──────────────────────────────
*
* A Geometry is an opaque, magic-tagged heap value carried in an el_val_t —
* the same discipline as List/Map. It holds a width and a float32 payload,
* and it is the medium a non-text modality enters in. Declared HERE, above
* the engram block, because transduction is a LANGUAGE concern: every El
* program touching any modality needs it, and the engram is merely one El
* program that happens to hold a graph. See el_runtime.c ("Geometry: signal
* as a first-class el value") for the full rationale.
*
* El-side type annotation is simply `Geometry` — an opaque boxed pointer,
* exactly like Instant / Calendar / Rhythm. No codegen change is required.
*
* OWNERSHIP: a Geometry is owned by the El caller and released with
* geometry_free. node_attach_geometry COPIES, so a node and the caller's
* value have independent lifetimes. */
el_val_t geometry_new(el_val_t dim); /* zero-filled; 0 on failure */
el_val_t geometry_dim(el_val_t g); /* width, 0 if not a Geometry */
el_val_t geometry_is(el_val_t g); /* 1 if a live Geometry */
el_val_t geometry_get(el_val_t g, el_val_t i); /* Float component */
el_val_t geometry_set(el_val_t g, el_val_t i, el_val_t x); /* 1 ok / 0 out of range */
el_val_t geometry_norm(el_val_t g); /* Float L2 — lets a caller
* check a realizer emitted
* signal, not zeros */
el_val_t geometry_free(el_val_t g); /* 1 if freed, 0 if not a Geometry.
* Returns a value (not void) so it
* is safe in any El expression
* position without a codegen
* void-builtin table entry. */
/* Wire ADAPTERS — the only place an encoding appears, and only at the edge.
* `f32le hex` is little-endian float32, 8 hex chars per component: the
* encoding the perception vessel's /voice/embed already emits. The width is
* DERIVED from the input length, never supplied by a caller — which is why
* there is no max-dim constant here to validate a claimed length against. */
el_val_t geometry_from_f32le_hex(el_val_t hex); /* 0 on empty/odd-length/non-hex */
el_val_t geometry_to_f32le_hex(el_val_t g); /* "" if not a Geometry */
/* ── Manifold: the result of a transduction ──────────────────────────────────
* A transduced signal is a SUBGRAPH — named components, each with its own
* geometry, plus typed weighted relations among them — not a single vector.
* One vector is a fingerprint: matchable, rankable, and nothing else. A song
* decomposes into pitch, interval, rhythm, harmonic function; the song IS the
* structure of those relations, and collapsing it to a point discards exactly
* what made it reasonable-about. See el_runtime.c ("Manifold") for the full
* rationale, the key-addressing rule, and the ownership contract.
*
* Components are addressed BY KEY, never by index, because the key is what
* survives persistence: a component becomes a node, and it is separately
* groundable precisely because it is separately named. Relation weight IS the
* grounding (correspondence-and-censorship.md §1) — one quantity, no separate
* score, nothing computed on read.
*
* OWNERSHIP: a Manifold is owned by the El caller and released with
* manifold_free, which also releases every component's geometry. manifold_add
* COPIES the geometry it is given and manifold_geometry RETURNS a copy, so no
* component's vector is ever aliased in either direction. */
el_val_t manifold_new(void); /* empty; 0 on failure */
el_val_t manifold_is(el_val_t m); /* 1 if a live Manifold */
el_val_t manifold_add(el_val_t m, el_val_t key, el_val_t role, el_val_t g);
/* component index, or -1 on empty/duplicate
* key or a value that is not a Geometry */
el_val_t manifold_relate(el_val_t m, el_val_t from, el_val_t rel,
el_val_t to, el_val_t weight);
/* 1 ok / 0 if either endpoint is unknown —
* an unresolvable edge is REFUSED, never
* silently dropped */
el_val_t manifold_size(el_val_t m); /* component count */
el_val_t manifold_rel_count(el_val_t m); /* relation count */
el_val_t manifold_index_of(el_val_t m, el_val_t key); /* index by key, or -1 */
el_val_t manifold_key(el_val_t m, el_val_t i); /* "" if out of range */
el_val_t manifold_role(el_val_t m, el_val_t i); /* "" if out of range */
el_val_t manifold_geometry(el_val_t m, el_val_t i); /* a COPY the caller frees */
el_val_t manifold_rel_from(el_val_t m, el_val_t j); /* source component key */
el_val_t manifold_rel_name(el_val_t m, el_val_t j); /* relation name */
el_val_t manifold_rel_to(el_val_t m, el_val_t j); /* target component key */
el_val_t manifold_rel_weight(el_val_t m, el_val_t j); /* Float — the grounding */
el_val_t manifold_single(el_val_t key, el_val_t role, el_val_t g);
/* the degenerate one-part case, expressible
* but visibly a size-1 manifold rather than
* a parallel path back to a bare vector */
el_val_t manifold_free(el_val_t m); /* 1 if freed, 0 otherwise */
/* ── Realizers + transduce ───────────────────────────────────────────────────
* A REALIZER DECOMPOSES one modality into components and relations. It does
* not encode a signal to a point; that operation is one layer below and is
* called geometry. Registration is by NAME, so a new modality never requires a
* runtime patch: every El `fn name(...)` compiles to a global C symbol with
* that exact name, and the registry resolves it with dlsym against the running
* binary — the same mechanism http_set_handler already relies on.
*
* fn tone_realizer(signal: String) -> Manifold { ... }
* realizer_register("tone", "tone_realizer")
* let m: Manifold = transduce(sample, "tone")
*
* SUPERSEDES #144's `transduce -> Geometry`. A realizer that still returns a
* bare Geometry now transduces NOTHING (transduce returns 0), deliberately: an
* organ that only fingerprints must not be indistinguishable from a working
* one. A modality with genuinely one part says so with manifold_single. */
el_val_t realizer_register(el_val_t modality, el_val_t fn_name); /* 1 ok / 0 unresolved */
el_val_t realizer_has(el_val_t modality); /* 1 if a realizer is registered */
el_val_t transduce(el_val_t signal, el_val_t modality); /* Manifold, or 0 if no organ */
/* ── Engram local graph primitives ───────────────────────────────────────────
* Operate on the CGI's local Engram knowledge graph.
* `engram_activate` queries the local graph only; `dharma_activate` is
@@ -606,7 +802,33 @@ el_val_t engram_get_node(el_val_t id);
void engram_strengthen(el_val_t node_id);
void engram_forget(el_val_t node_id);
el_val_t engram_prune_telemetry(el_val_t older_than_ms);
/* Largest byte length <= max_bytes that does not split a UTF-8 codepoint.
* Bounded by bytes, not codepoints, so truncated strings never grow. */
size_t el_utf8_safe_len(const char* s, size_t max_bytes);
/* Register the ambient-consolidation step and start dreaming. Resolved by
* dlsym, like http_set_handler. The handler performs ONE step and returns
* non-zero if it did work; returning zero parks the dreamer until engagement
* changes. There is no schedule and must never be one. */
void dream_set_handler(el_val_t name);
el_val_t engram_node_count(void);
/* Attach a Geometry to an existing node, and read the attached width back.
* Named for the operation, not the store: a node acquires geometry. This is
* the geometry-valued ingest path — nothing about it is hex, and nothing
* about it assumes the caller's vector matches the canonical text-embedding
* width. node_geometry_dim exists so an attach is VERIFIED by reading it
* back rather than by trusting a success return. */
el_val_t node_attach_geometry(el_val_t node_id, el_val_t g); /* 1 ok / 0 otherwise */
el_val_t node_geometry_dim(el_val_t node_id); /* width, 0 if none */
/* DEPRECATED (shipped in #141, superseded 2026-08-16). Equivalent to
* geometry_from_f32le_hex + node_attach_geometry, and now implemented as
* exactly that. Kept only so anything built against the #141 runtime keeps
* linking; `dim` is accepted but treated as an assertion about the vector's
* width rather than as its source. New code should not call this — a hex
* string is a wire encoding, not a way to move geometry between two pieces
* of El. Returns 1 on success, 0 otherwise. */
el_val_t engram_node_set_emb(el_val_t id, el_val_t hex, el_val_t dim);
el_val_t engram_search(el_val_t query, el_val_t limit);
el_val_t engram_scan_nodes(el_val_t limit, el_val_t offset);
void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation);
@@ -651,8 +873,13 @@ el_val_t engram_geo_analogy_json(el_val_t a_seeds, el_val_t b_seeds);
el_val_t engram_reason_analogy_json(el_val_t a_seeds, el_val_t b_seeds, el_val_t c_seeds);
/* COGNITION (2026-08-14): THE ONE OPERATION + grounding, surfaced live. */
el_val_t engram_think_json(el_val_t seeds, el_val_t faculty);
/* GROUNDING (2026-08-16): grounding is an attribute of the RELATION and it IS the
* hebbian weight. ground reads; ground_record writes; trajectory reads the chain. */
el_val_t engram_ground_json(el_val_t claim, el_val_t evidence, el_val_t for_whom);
el_val_t engram_assert_json(el_val_t claim_id, el_val_t for_whom, el_val_t floor);
el_val_t engram_ground_record_json(el_val_t claim, el_val_t evidence,
el_val_t provenance, el_val_t floor);
el_val_t engram_ground_trajectory_json(el_val_t claim, el_val_t evidence);
el_val_t engram_assert_json(el_val_t claim_id, el_val_t for_whom, el_val_t floor, el_val_t rel_floor);
el_val_t engram_attend_json(el_val_t node_id, el_val_t observer, el_val_t salience);
el_val_t engram_correspondence_beat_json(el_val_t seeds, el_val_t faculty, el_val_t keystone);
el_val_t engram_consolidate_permanence(el_val_t node_id);
@@ -660,7 +887,10 @@ el_val_t engram_age_field(el_val_t delta_ms);
el_val_t engram_age_field_catchup(void);
el_val_t engram_chrono_persist_tick(void);
el_val_t engram_chrono_tick(void);
el_val_t engram_boundary_beat(el_val_t op_name); /* API-reshape decorator-seam auto-emit */
el_val_t engram_boundary_beat(el_val_t op_name, el_val_t construct);
int el_tagged(el_val_t v, uint32_t magic); /* the gate: validate a slot BEFORE dereferencing it */
el_val_t el_seam_run(el_val_t fn_name, el_val_t phase, el_val_t result); /* runtime construct seam */
el_val_t el_seam_wrap(el_val_t fn_name, el_val_t (*body)(void*), void* env); /* runtime invocation control */ /* API-reshape decorator-seam auto-emit; construct = the decorator that caused the beat */
el_val_t engram_self_anchor_capture(void);
el_val_t engram_self_drift_json(void);
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
@@ -712,6 +942,14 @@ el_val_t engram_label_df(el_val_t term);
el_val_t engram_salient_term(el_val_t node_id, el_val_t max_df,
el_val_t min_df, el_val_t tabu);
el_val_t engram_embed_backfill(el_val_t count);
/* op_assert seam: grounded assertion envelope {subject,grounding} for the realizer. */
el_val_t engram_op_assert_json(el_val_t node_id, el_val_t depth);
/* Parametric mutation (purview write-side): purview==0 => G=live (default), else refuse. */
el_val_t engram_node_full_in(el_val_t purview, el_val_t content, el_val_t node_type, el_val_t label,
el_val_t salience, el_val_t importance, el_val_t confidence,
el_val_t tier, el_val_t tags);
void engram_connect_in(el_val_t purview, el_val_t from_id, el_val_t to_id,
el_val_t weight, el_val_t relation);
el_val_t engram_list_layers_json(void);
/* Working memory introspection — count, mean weight, and top-N snapshot.
* Ported from runtime on 2026-06-30 self-review. */
@@ -892,6 +1130,156 @@ el_val_t trace_span_start(el_val_t name);
el_val_t trace_span_end(el_val_t span_handle);
el_val_t emit_event(el_val_t name, el_val_t duration_ms);
el_val_t __thread_create(el_val_t fn_name_v, el_val_t arg_v);
el_val_t __thread_join(el_val_t tid_v);
/* Mutex + channel seed primitives (defined in el_runtime.c). Declared here so
* that compiled El programs which use runtime/thread.el's with_mutex helper or
* runtime/channel.el's Go-style channels see real prototypes instead of an
* implicit int-return declaration (which the C11 ABI mis-truncates el_val_t). */
el_val_t __mutex_new(void);
void __mutex_lock(el_val_t m_v);
void __mutex_unlock(el_val_t m_v);
el_val_t __channel_new(el_val_t capacity_v);
el_val_t __channel_send(el_val_t ch_v, el_val_t msg_v);
el_val_t __channel_recv(el_val_t ch_v);
el_val_t __channel_try_recv(el_val_t ch_v);
el_val_t __channel_close(el_val_t ch_v);
/* ── __ prefixed aliases (self-hosting compiler ABI) ─────────────────────────
* The El self-hosting compiler emits calls to __-prefixed names. These are
* forwarding wrappers around the existing el_runtime functions above. */
/* I/O */
el_val_t __println(el_val_t s);
el_val_t __print(el_val_t s);
el_val_t __readline(void);
/* String */
el_val_t __int_to_str(el_val_t n);
el_val_t __str_to_int(el_val_t s);
el_val_t __float_to_str(el_val_t f);
el_val_t __str_to_float(el_val_t s);
el_val_t __str_len(el_val_t s);
el_val_t __str_char_at(el_val_t s, el_val_t i);
el_val_t __str_cmp(el_val_t a, el_val_t b);
el_val_t __str_ncmp(el_val_t a, el_val_t b, el_val_t n);
el_val_t __str_concat_raw(el_val_t a, el_val_t b);
el_val_t __str_slice_raw(el_val_t s, el_val_t start, el_val_t end);
el_val_t __str_alloc(el_val_t n);
el_val_t __str_set_char(el_val_t s, el_val_t i, el_val_t c);
/* URL encoding */
el_val_t __url_encode(el_val_t s);
el_val_t __url_decode(el_val_t s);
/* Environment */
el_val_t __env_get(el_val_t key);
/* Cross-cutting concerns declared by a `program` block (spec §18).
* All three are COMPILER-INJECTED at the head of main() — they are not meant to
* be written by hand, which is the point: the guarantee cannot be forgotten at a
* call site because there is no call site. */
el_val_t el_singleton_acquire(el_val_t id, el_val_t state); /* §18.2 process identity — keyed on the guarded state */
el_val_t el_config_declare(el_val_t name, el_val_t type,
el_val_t deflt, el_val_t has_default,
el_val_t required); /* §18.2 config schema */
el_val_t el_config_validate(el_val_t program_name); /* §18.2 startup validate */
/* config(key) — the READ side, and the only one programs write by hand. With a
* schema declared it is a validated lookup; without one it degrades to getenv.
* (Defined in el_runtime.c but previously never prototyped here, so any program
* calling it failed to compile under -Werror=implicit-function-declaration.) */
el_val_t config(el_val_t key);
/* Subprocess */
el_val_t __exec(el_val_t cmd);
el_val_t __exec_bg(el_val_t cmd);
/* Process */
el_val_t __exit_program(el_val_t code);
/* Filesystem */
el_val_t __fs_exists(el_val_t path);
el_val_t __fs_mkdir(el_val_t path);
el_val_t __fs_read(el_val_t path);
el_val_t __fs_write(el_val_t path, el_val_t content);
el_val_t __fs_write_bytes(el_val_t path, el_val_t bytes, el_val_t n);
el_val_t __fs_list_raw(el_val_t path);
/* HTTP server */
el_val_t __http_response(el_val_t status, el_val_t headers_json, el_val_t body);
el_val_t __http_serve(el_val_t port, el_val_t handler);
el_val_t __http_serve_v2(el_val_t port, el_val_t handler);
/* HTTP conn fd / SSE (weak; overridden by el_seed.c when linked together) */
el_val_t __http_conn_fd(void);
el_val_t __http_sse_open(el_val_t conn_id);
el_val_t __http_sse_send(el_val_t conn_id, el_val_t data);
el_val_t __http_sse_close(el_val_t conn_id);
/* HTTP client (requires HAVE_CURL; stubs provided for no-curl builds) */
el_val_t __http_do(el_val_t method, el_val_t url, el_val_t body,
el_val_t headers_map, el_val_t timeout_ms);
el_val_t __http_do_map(el_val_t method, el_val_t url, el_val_t body,
el_val_t headers_json, el_val_t timeout_ms);
el_val_t __http_do_map_to_file(el_val_t method, el_val_t url, el_val_t body,
el_val_t headers_json, el_val_t output_path);
/* JSON */
el_val_t __json_array_get(el_val_t json, el_val_t index);
el_val_t __json_array_get_string(el_val_t json, el_val_t index);
el_val_t __json_array_len(el_val_t json);
el_val_t __json_get(el_val_t json, el_val_t key);
el_val_t __json_get_raw(el_val_t json, el_val_t key);
el_val_t __json_set(el_val_t json, el_val_t key, el_val_t value);
el_val_t __json_parse_map(el_val_t json_str);
el_val_t __json_stringify_val(el_val_t val);
/* Hashing */
el_val_t __sha256_hex(el_val_t s);
/* State K/V */
el_val_t __state_del(el_val_t key);
el_val_t __state_get(el_val_t key);
el_val_t __state_keys(void);
el_val_t __state_set(el_val_t key, el_val_t val);
/* UUID */
el_val_t __uuid_v4(void);
/* Args */
el_val_t __args_json(void);
/* Compiler-support builtins — called by the El compiler's own source
* (compiler.el, codegen.el) and registered in codegen.el's builtin_arity. */
el_val_t stdout_to_file(el_val_t path);
el_val_t stdout_restore(void);
el_val_t el_mem_check(void);
/* Allocation accounting — the deterministic signal behind complexity gating.
* Gate on counts/bytes; peak RSS is context only. */
el_val_t el_alloc_count(void);
el_val_t el_alloc_bytes(void);
el_val_t el_peak_rss(void);
el_val_t el_black_box(el_val_t v);
/* Semantic retrieval surface. NOT interchangeable with engram_search_json,
* which is lexical by design — see the note at the definition. */
el_val_t engram_recall_json(el_val_t query, el_val_t limit);
/* Edges straight from the store — replaces the engram_save()+fs_read()
* whole-graph round trip that /api/graph/edges used to do. */
el_val_t engram_edges_json(el_val_t limit, el_val_t offset);
/* Buffer-pool interoception as JSON — live pool health for observation. */
el_val_t engram_pool_stats_json(void);
/* CGI identity accessors (read-only). */
el_val_t cgi_principal(void);
el_val_t cgi_network(void);
el_val_t cgi_engram(void);
#ifdef __cplusplus
}
#endif
lang/runtime/el_seed.c
+462 -3
View File
@@ -37,6 +37,82 @@
#include <dlfcn.h>
#include <curl/curl.h>
/* el_runtime.c bridge prototypes.
*
* A block of __-prefixed wrappers further down in this file (http serving,
* JSON access, key-val state, URL/HTML escaping, and the whole engram_*
* node/edge/layer/search surface -- 51 symbols in total) delegate to
* unprefixed counterparts that are implemented in el_runtime.c, not here.
* Porting them into native el_seed.c or El has not happened yet.
* tools/install.sh compiles el_seed.c and el_runtime.c as separate objects
* and archives both into libel.a, so the symbols are always present at link
* time. el_seed.c alone was just missing the prototypes, which made even a
* standalone -c compile of this one file fail on a toolchain that now treats
* an implicit function declaration as a hard error under C11.
*
* A plain include of el_runtime.h was tried first and rejected: it redefines
* el_to_float and el_from_float, which el_seed.h already provides. Narrow
* prototypes, copied verbatim from el_runtime.h, avoid that collision without
* pulling in the rest of the retiring runtime header.
*/
el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body);
void http_serve(el_val_t port, el_val_t handler);
void http_serve_v2(el_val_t port, el_val_t handler);
el_val_t json_get(el_val_t json, el_val_t key);
el_val_t json_get_string(el_val_t json_str, el_val_t key);
el_val_t json_get_int(el_val_t json_str, el_val_t key);
el_val_t json_get_float(el_val_t json_str, el_val_t key);
el_val_t json_get_bool(el_val_t json_str, el_val_t key);
el_val_t json_get_raw(el_val_t json_str, el_val_t key);
el_val_t json_parse(el_val_t s);
el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value);
el_val_t json_stringify(el_val_t v);
el_val_t json_array_len(el_val_t json_str);
el_val_t json_array_get(el_val_t json_str, el_val_t index);
el_val_t json_array_get_string(el_val_t json_str, el_val_t index);
el_val_t state_set(el_val_t key, el_val_t value);
el_val_t state_get(el_val_t key);
el_val_t state_del(el_val_t key);
el_val_t state_keys(void);
el_val_t url_encode(el_val_t s);
el_val_t url_decode(el_val_t s);
el_val_t el_html_sanitize(el_val_t input_html, el_val_t allowlist_json);
el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience);
el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
el_val_t salience, el_val_t importance, el_val_t confidence,
el_val_t tier, el_val_t tags);
el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t label,
el_val_t salience, el_val_t certainty, el_val_t confidence,
el_val_t status, el_val_t tags, el_val_t layer_id);
el_val_t engram_add_layer(el_val_t name, el_val_t priority, el_val_t suppressible,
el_val_t transparent, el_val_t injectable);
el_val_t engram_remove_layer(el_val_t layer_id);
el_val_t engram_list_layers(void);
el_val_t engram_list_layers_json(void);
el_val_t engram_get_node(el_val_t id);
el_val_t engram_get_node_json(el_val_t id);
el_val_t engram_get_node_by_label(el_val_t label);
void engram_strengthen(el_val_t node_id);
void engram_forget(el_val_t node_id);
el_val_t engram_node_count(void);
el_val_t engram_edge_count(void);
el_val_t engram_scan_nodes(el_val_t limit, el_val_t offset);
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
el_val_t engram_search(el_val_t query, el_val_t limit);
el_val_t engram_search_json(el_val_t query, el_val_t limit);
el_val_t engram_activate(el_val_t query, el_val_t depth);
el_val_t engram_activate_json(el_val_t query, el_val_t depth);
el_val_t engram_compile_layered_json(el_val_t intent, el_val_t depth);
el_val_t engram_stats_json(void);
void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation);
el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id);
el_val_t engram_neighbors(el_val_t node_id);
el_val_t engram_neighbors_filtered(el_val_t node_id, el_val_t max_depth, el_val_t direction);
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
el_val_t engram_load(el_val_t path);
el_val_t engram_save(el_val_t path);
/* ── Private allocator ───────────────────────────────────────────────────── */
/*
* el_seed.c carries its own arena for per-request allocation tracking.
@@ -72,10 +148,26 @@ static void seed_request_start(void) {
_seed_arena_on = 1;
}
/* Defined in el_runtime.c. The string-length cache there keys on pointer +
* generation; anything that frees or mutates a runtime string must bump the
* generation or a reused address could return a stale length. Weak so this
* file still links on its own. */
__attribute__((weak)) void el_str_cache_flush(void);
/* Byte-buffer capacity registry (defined below, next to the string
* primitives). The arena frees the pointers it tracked, so any capacity
* entry for those addresses must go with them — otherwise a later malloc
* reusing the address would inherit a stale width. */
static void seed_cap_drop(const char* p);
static void seed_request_end(void) {
_seed_arena_on = 0;
for (size_t i = 0; i < _seed_arena.count; i++) free(_seed_arena.ptrs[i]);
for (size_t i = 0; i < _seed_arena.count; i++) {
seed_cap_drop(_seed_arena.ptrs[i]);
free(_seed_arena.ptrs[i]);
}
_seed_arena.count = 0;
if (el_str_cache_flush) el_str_cache_flush(); /* freed pointers may be reused */
}
/* el_request_start / el_request_end — formerly defined in el_runtime.c.
@@ -105,6 +197,114 @@ static char* seed_strbuf(size_t n) {
static el_val_t seed_wrap_str(char* s) { return EL_STR(s); }
/* ── Byte-buffer capacity registry ────────────────────────────────────────────
* A String produced by __str_alloc is a fixed-size BYTE BUFFER, not text. Its
* length is the capacity it was asked for; strlen() is meaningless on it,
* because the buffer is zero-filled and binary content (PCM audio, RIFF
* headers, image rasters) contains NUL bytes by nature.
*
* Before this registry existed, __str_set_char bounds-checked the write index
* against strlen(p). For a freshly __str_alloc'd buffer strlen(p) == 0, so the
* check `idx >= len` rejected EVERY index and the function was a total no-op:
* every El program that built bytes this way wrote a file of pure zeros and
* still saw a success return. That is why El's own-core WAV writer emitted
* 55,244 silent bytes with a correct-looking header length and no header.
*
* The fix cannot be "trust the index", because that removes the bound. It also
* cannot be a length header stored behind the pointer, because __str_set_char
* accepts any String — including a string literal in .rodata, where reading the
* bytes preceding the pointer is undefined and may fault. So capacity is kept
* in a side table keyed by the pointer itself: allocation registers, the arena
* sweep unregisters, and anything not registered keeps the exact strlen
* behaviour it had before. Text semantics are unchanged; byte buffers gain the
* bound they always should have had. */
typedef struct {
char* ptr; /* NULL = empty slot, (char*)1 = tombstone */
size_t cap;
} SeedCapEntry;
#define SEED_CAP_TOMB ((char*)1)
static _Thread_local SeedCapEntry* _seed_cap = NULL;
static _Thread_local size_t _seed_cap_mask = 0; /* table size - 1 */
static _Thread_local size_t _seed_cap_used = 0; /* live + tombstoned */
static size_t seed_cap_hash(const char* p) {
uintptr_t h = (uintptr_t)p >> 4; /* malloc alignment: low bits are dead */
h *= (uintptr_t)0x9E3779B97F4A7C15ull;
return (size_t)(h >> 32);
}
static void seed_cap_put(char* p, size_t cap);
static void seed_cap_grow(void) {
size_t old_size = _seed_cap_mask ? _seed_cap_mask + 1 : 0;
SeedCapEntry* old = _seed_cap;
size_t new_size = old_size ? old_size * 2 : 256;
SeedCapEntry* fresh = calloc(new_size, sizeof(SeedCapEntry));
if (!fresh) return; /* out of memory: keep old table */
_seed_cap = fresh;
_seed_cap_mask = new_size - 1;
_seed_cap_used = 0;
for (size_t i = 0; i < old_size; i++) {
if (old[i].ptr && old[i].ptr != SEED_CAP_TOMB) seed_cap_put(old[i].ptr, old[i].cap);
}
free(old);
}
static void seed_cap_put(char* p, size_t cap) {
if (!p) return;
if (!_seed_cap || (_seed_cap_used + 1) * 4 >= (_seed_cap_mask + 1) * 3) {
seed_cap_grow();
if (!_seed_cap) return;
}
size_t i = seed_cap_hash(p) & _seed_cap_mask;
size_t first_free = (size_t)-1;
for (;;) {
char* e = _seed_cap[i].ptr;
if (e == p) { _seed_cap[i].cap = cap; return; } /* address reused */
if (e == SEED_CAP_TOMB && first_free == (size_t)-1) first_free = i;
if (!e) {
if (first_free != (size_t)-1) i = first_free; else _seed_cap_used++;
_seed_cap[i].ptr = p;
_seed_cap[i].cap = cap;
return;
}
i = (i + 1) & _seed_cap_mask;
}
}
/* Capacity of a registered byte buffer, or -1 when the pointer is not one. */
static int64_t seed_cap_get(const char* p) {
if (!p || !_seed_cap) return -1;
size_t i = seed_cap_hash(p) & _seed_cap_mask;
for (;;) {
char* e = _seed_cap[i].ptr;
if (!e) return -1;
if (e == (char*)p) return (int64_t)_seed_cap[i].cap;
i = (i + 1) & _seed_cap_mask;
}
}
static void seed_cap_drop(const char* p) {
if (!p || !_seed_cap) return;
size_t i = seed_cap_hash(p) & _seed_cap_mask;
for (;;) {
char* e = _seed_cap[i].ptr;
if (!e) return;
if (e == (char*)p) { _seed_cap[i].ptr = SEED_CAP_TOMB; return; }
i = (i + 1) & _seed_cap_mask;
}
}
/* Effective addressable length of a String: its buffer capacity when it is a
* byte buffer, otherwise strlen. */
static int64_t seed_addressable_len(const char* p) {
int64_t cap = seed_cap_get(p);
return cap >= 0 ? cap : (int64_t)strlen(p);
}
/* ── String primitives ───────────────────────────────────────────────────── */
el_val_t __str_len(el_val_t s) {
@@ -116,7 +316,7 @@ el_val_t __str_len(el_val_t s) {
el_val_t __str_char_at(el_val_t s, el_val_t i) {
const char* p = EL_CSTR(s);
if (!p) return 0;
int64_t len = (int64_t)strlen(p);
int64_t len = seed_addressable_len(p); /* capacity for byte buffers */
int64_t idx = (int64_t)i;
if (idx < 0 || idx >= len) return 0;
return (el_val_t)(unsigned char)p[idx];
@@ -127,16 +327,18 @@ el_val_t __str_alloc(el_val_t n) {
if (sz < 0) sz = 0;
char* buf = seed_strbuf((size_t)sz);
memset(buf, 0, (size_t)sz + 1);
seed_cap_put(buf, (size_t)sz); /* this is a byte buffer of width sz */
return seed_wrap_str(buf);
}
el_val_t __str_set_char(el_val_t s, el_val_t i, el_val_t c) {
char* p = (char*)(uintptr_t)s;
if (!p) return s;
int64_t len = (int64_t)strlen(p);
int64_t len = seed_addressable_len(p); /* capacity for byte buffers */
int64_t idx = (int64_t)i;
if (idx < 0 || idx >= len) return s;
p[idx] = (char)(unsigned char)(int64_t)c;
if (el_str_cache_flush) el_str_cache_flush(); /* in-place write can move the NUL */
return s;
}
@@ -322,6 +524,15 @@ el_val_t __fs_mkdir(el_val_t path) {
return 1;
}
/* stderr counterpart of println. Flushed immediately: a disclosure line is only
* worth anything if it lands before the thing it discloses happens. */
void eprintln(el_val_t s) {
const char* p = EL_CSTR(s);
fputs(p ? p : "", stderr);
fputc('\n', stderr);
fflush(stderr);
}
el_val_t __fs_write_bytes(el_val_t path, el_val_t bytes, el_val_t n) {
const char* p = EL_CSTR(path);
const char* b = EL_CSTR(bytes);
@@ -831,6 +1042,219 @@ void __mutex_unlock(el_val_t m) {
pthread_mutex_unlock(&_el_mutexes[slot]);
}
/* ── Channels ─────────────────────────────────────────────────────────────── *
* Buffered MPMC channel backed by a mutex + condvar + circular buffer.
* Ported from the pre-restructure el_runtime.c (b2aac4b) — runtime/channel.el
* has always called these five primitives, but they were never carried
* forward into el_seed.c when el_runtime.c was consolidated onto the
* canonical release copy. Native channels were silently unlinkable on dev
* until this port.
*
* __channel_new(capacity) -> Int (handle)
* __channel_send(ch, msg) — blocks if full (capacity > 0) or never (unbounded)
* __channel_recv(ch) -> String — blocks until a message is available
* __channel_try_recv(ch) -> String — non-blocking, returns "" if empty
* __channel_close(ch) — signal no more sends; recv drains remaining
*
* Bounded channels (cap > 0): circular buffer, sender blocks when full.
* Unbounded channels (cap == 0): dynamic array, sender never blocks.
*/
#define EL_CHANNEL_MAX 64
#define EL_CHANNEL_BUF 1024
typedef struct {
char** buf;
int cap; /* 0 = unbounded (grows dynamically) */
int head, tail, count;
int dyn_cap; /* allocated slots for unbounded mode */
int closed;
pthread_mutex_t mu;
pthread_cond_t not_empty;
pthread_cond_t not_full;
} ElChannel;
static ElChannel _channels[EL_CHANNEL_MAX];
static int _channel_count = 0;
static pthread_mutex_t _channel_alloc_mu = PTHREAD_MUTEX_INITIALIZER;
el_val_t __channel_new(el_val_t capacity_v) {
int cap = (int)(int64_t)capacity_v;
if (cap < 0) cap = 0;
pthread_mutex_lock(&_channel_alloc_mu);
if (_channel_count >= EL_CHANNEL_MAX) {
pthread_mutex_unlock(&_channel_alloc_mu);
fprintf(stderr, "[__channel_new] channel table full\n");
return EL_INT(-1);
}
int slot = _channel_count++;
pthread_mutex_unlock(&_channel_alloc_mu);
ElChannel* ch = &_channels[slot];
memset(ch, 0, sizeof(*ch));
ch->cap = cap;
ch->closed = 0;
ch->head = 0;
ch->tail = 0;
ch->count = 0;
if (cap > 0) {
/* Bounded: fixed circular buffer. */
ch->buf = (char**)malloc((size_t)cap * sizeof(char*));
ch->dyn_cap = cap;
} else {
/* Unbounded: start with EL_CHANNEL_BUF slots, grow as needed. */
ch->buf = (char**)malloc(EL_CHANNEL_BUF * sizeof(char*));
ch->dyn_cap = EL_CHANNEL_BUF;
}
if (!ch->buf) {
fprintf(stderr, "[__channel_new] out of memory\n");
return EL_INT(-1);
}
pthread_mutex_init(&ch->mu, NULL);
pthread_cond_init(&ch->not_empty, NULL);
pthread_cond_init(&ch->not_full, NULL);
return EL_INT(slot);
}
el_val_t __channel_send(el_val_t ch_v, el_val_t msg_v) {
int slot = (int)(int64_t)ch_v;
if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR("");
ElChannel* ch = &_channels[slot];
const char* msg = EL_CSTR(msg_v);
if (!msg) msg = "";
char* copy = strdup(msg); /* channel owns the string */
pthread_mutex_lock(&ch->mu);
if (ch->closed) {
/* Send on closed channel is a no-op (drop the message). */
pthread_mutex_unlock(&ch->mu);
free(copy);
return EL_STR("");
}
if (ch->cap > 0) {
/* Bounded: block while full. */
while (ch->count >= ch->cap && !ch->closed) {
pthread_cond_wait(&ch->not_full, &ch->mu);
}
if (ch->closed) {
pthread_mutex_unlock(&ch->mu);
free(copy);
return EL_STR("");
}
ch->buf[ch->tail] = copy;
ch->tail = (ch->tail + 1) % ch->cap;
ch->count++;
} else {
/* Unbounded: grow the buffer if needed. */
if (ch->count >= ch->dyn_cap) {
int new_cap = ch->dyn_cap * 2;
char** grown = (char**)realloc(ch->buf, (size_t)new_cap * sizeof(char*));
if (!grown) {
pthread_mutex_unlock(&ch->mu);
free(copy);
fprintf(stderr, "[__channel_send] out of memory growing channel\n");
return EL_STR("");
}
/* The circular buffer may have wrapped. Linearise it first.
* In unbounded mode head is always 0 (we append at tail, drain
* from head), so a simple memmove isn't needed — but if the
* buffer did wrap (tail < head after growth), we need to fix up.
* Simplest safe path: if tail wrapped, move the head..old_cap
* segment to new_cap..new_cap+(old_cap-head). */
if (ch->tail < ch->head) {
/* Wrapped: [head..old_cap) is the front, [0..tail) is the back. */
int front = ch->dyn_cap - ch->head;
memmove(grown + ch->dyn_cap, grown + ch->head, (size_t)front * sizeof(char*));
ch->head = ch->dyn_cap;
}
ch->buf = grown;
ch->dyn_cap = new_cap;
}
ch->buf[ch->tail] = copy;
ch->tail = (ch->tail + 1) % ch->dyn_cap;
ch->count++;
}
pthread_cond_signal(&ch->not_empty);
pthread_mutex_unlock(&ch->mu);
return EL_STR("");
}
el_val_t __channel_recv(el_val_t ch_v) {
int slot = (int)(int64_t)ch_v;
if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR("");
ElChannel* ch = &_channels[slot];
pthread_mutex_lock(&ch->mu);
/* Block until there is a message or the channel is closed and drained. */
while (ch->count == 0 && !ch->closed) {
pthread_cond_wait(&ch->not_empty, &ch->mu);
}
if (ch->count == 0) {
/* Closed and empty — signal EOF. */
pthread_mutex_unlock(&ch->mu);
return EL_STR("");
}
int buf_cap = (ch->cap > 0) ? ch->cap : ch->dyn_cap;
char* msg = ch->buf[ch->head];
ch->head = (ch->head + 1) % buf_cap;
ch->count--;
pthread_cond_signal(&ch->not_full);
pthread_mutex_unlock(&ch->mu);
/* Hand the string to the arena so it is freed after the request. */
seed_arena_track(msg);
return EL_STR(msg);
}
el_val_t __channel_try_recv(el_val_t ch_v) {
int slot = (int)(int64_t)ch_v;
if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR("");
ElChannel* ch = &_channels[slot];
pthread_mutex_lock(&ch->mu);
if (ch->count == 0) {
pthread_mutex_unlock(&ch->mu);
return EL_STR("");
}
int buf_cap = (ch->cap > 0) ? ch->cap : ch->dyn_cap;
char* msg = ch->buf[ch->head];
ch->head = (ch->head + 1) % buf_cap;
ch->count--;
pthread_cond_signal(&ch->not_full);
pthread_mutex_unlock(&ch->mu);
seed_arena_track(msg);
return EL_STR(msg);
}
el_val_t __channel_close(el_val_t ch_v) {
int slot = (int)(int64_t)ch_v;
if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR("");
ElChannel* ch = &_channels[slot];
pthread_mutex_lock(&ch->mu);
ch->closed = 1;
/* Wake all blocked recvers and senders so they can observe the close. */
pthread_cond_broadcast(&ch->not_empty);
pthread_cond_broadcast(&ch->not_full);
pthread_mutex_unlock(&ch->mu);
return EL_STR("");
}
/* ── Subprocess ──────────────────────────────────────────────────────────── */
el_val_t __exec(el_val_t cmd) {
@@ -1082,6 +1506,21 @@ el_val_t __engram_scan_nodes_json(el_val_t limit, el_val_t offset) {
return engram_scan_nodes_json(limit, offset);
}
el_val_t engram_edges_json(el_val_t limit, el_val_t offset);
el_val_t __engram_edges_json(el_val_t limit, el_val_t offset) {
return engram_edges_json(limit, offset);
}
el_val_t engram_pool_stats_json(void);
el_val_t __engram_pool_stats_json(void) { return engram_pool_stats_json(); }
el_val_t el_alloc_count(void);
el_val_t el_alloc_bytes(void);
el_val_t el_peak_rss(void);
el_val_t __el_alloc_count(void) { return el_alloc_count(); }
el_val_t __el_alloc_bytes(void) { return el_alloc_bytes(); }
el_val_t __el_peak_rss(void) { return el_peak_rss(); }
el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset) {
return engram_scan_nodes_by_type_json(node_type, limit, offset);
}
@@ -1094,7 +1533,27 @@ el_val_t __engram_activate_json(el_val_t query, el_val_t depth) {
return engram_activate_json(query, depth);
}
/* Forward decls for el_runtime.c symbols this file wraps. el_seed.c does not
* include el_runtime.h (documented in lang/AGENTS.md), so each wrapped symbol
* needs a prototype here or clang treats it as an implicit declaration (error
* under C99+) and the ABI mis-truncates the el_val_t return. */
el_val_t engram_op_assert_json(el_val_t node_id, el_val_t depth);
el_val_t engram_node_full_in(el_val_t purview, el_val_t content, el_val_t node_type, el_val_t label,
el_val_t salience, el_val_t importance, el_val_t confidence,
el_val_t tier, el_val_t tags);
void engram_connect_in(el_val_t purview, el_val_t from_id, el_val_t to_id,
el_val_t weight, el_val_t relation);
el_val_t __engram_stats_json(void) { return engram_stats_json(); }
el_val_t __engram_op_assert_json(el_val_t node_id, el_val_t depth) { return engram_op_assert_json(node_id, depth); }
el_val_t __engram_node_full_in(el_val_t purview, el_val_t content, el_val_t node_type, el_val_t label,
el_val_t salience, el_val_t importance, el_val_t confidence,
el_val_t tier, el_val_t tags) {
return engram_node_full_in(purview, content, node_type, label, salience, importance, confidence, tier, tags);
}
void __engram_connect_in(el_val_t purview, el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation) {
engram_connect_in(purview, from_id, to_id, weight, relation);
}
el_val_t __engram_list_layers_json(void) { return engram_list_layers_json(); }
el_val_t __engram_compile_layered_json(el_val_t intent, el_val_t depth) {
lang/runtime/el_seed.h
+13
View File
@@ -139,6 +139,13 @@ el_val_t __mutex_new(void);
void __mutex_lock(el_val_t m);
void __mutex_unlock(el_val_t m);
/* Buffered MPMC channel (runtime/channel.el). capacity=0 means unbounded. */
el_val_t __channel_new(el_val_t capacity);
el_val_t __channel_send(el_val_t ch, el_val_t msg); /* blocks if bounded+full */
el_val_t __channel_recv(el_val_t ch); /* blocks until available */
el_val_t __channel_try_recv(el_val_t ch); /* non-blocking, "" if empty */
el_val_t __channel_close(el_val_t ch);
/* ── Subprocess ──────────────────────────────────────────────────────────── */
el_val_t __exec(el_val_t cmd); /* popen, capture all stdout, return String */
@@ -233,6 +240,12 @@ el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, e
el_val_t __engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
el_val_t __engram_activate_json(el_val_t query, el_val_t depth);
el_val_t __engram_stats_json(void);
el_val_t __engram_op_assert_json(el_val_t node_id, el_val_t depth);
el_val_t __engram_node_full_in(el_val_t purview, el_val_t content, el_val_t node_type, el_val_t label,
el_val_t salience, el_val_t importance, el_val_t confidence,
el_val_t tier, el_val_t tags);
void __engram_connect_in(el_val_t purview, el_val_t from_id, el_val_t to_id,
el_val_t weight, el_val_t relation);
el_val_t __engram_list_layers_json(void);
el_val_t __engram_compile_layered_json(el_val_t intent, el_val_t depth);
lang/runtime/elbench.el
+256
View File
@@ -0,0 +1,256 @@
// runtime/elbench.el — growth-curve classifier and complexity gate.
//
// Given a geometric sweep of input sizes and the measurements taken at each,
// classify the growth curve and decide whether it violates a declared bound.
//
// ── Why this exists ──────────────────────────────────────────────────────────
//
// Constant-factor regressions are annoying. Complexity regressions are outages.
// An O(n) lookup inside an O(n) loop is invisible at n=100 in a unit test and
// catastrophic at n=100000 in production. el #132 was exactly that: a strlen()
// inside a per-character accessor, quadratic, shipped for months.
//
// ── THREE signals, not one ───────────────────────────────────────────────────
//
// The gate fits time AND allocation-count AND allocation-bytes, and fails if
// ANY of them exceeds its declared curve. This is not belt-and-braces; each
// signal is blind to a real defect class the others catch:
//
// * A copy-on-write accumulator rebuilding its buffer allocates ONCE per
// iteration — count is exactly linear — while bytes go quadratic.
// Count alone passes it.
// * el #132's strlen-per-character is pure CPU and allocates NOTHING.
// Both allocation signals read FLAT. Only time catches it.
//
// The deterministic signals (count, bytes) are preferable where they apply:
// no statistics, correct on the first run, machine-independent. They are
// simply not sufficient.
//
// ── SCOPE LIMIT — read this before trusting a flat curve ─────────────────────
//
// The allocation counters track EL-LEVEL allocation only: strings, ElList and
// ElMap bodies, their backing arrays, copy-on-write clones, and the realloc
// growth path. malloc inside engram_*.c and inside libcurl is NOT counted.
//
// A flat allocation curve over a workload dominated by engram or HTTP calls is
// therefore NOT evidence of anything. It means "no El-level allocation growth",
// not "no allocation growth". Gate El-level complexity with this; do not read
// third-party memory behaviour into it.
//
// ── Classification method ────────────────────────────────────────────────────
//
// Sizes must form a geometric sweep (each n double the last). On such a sweep
// the ratio between consecutive measurements IS the growth exponent, directly:
//
// O(1) -> 1.0 O(log n) -> ~1.1 O(n) -> 2.0
// O(n log n) -> ~2.2 O(n^2) -> 4.0 O(n^3) -> 8.0
//
// DEVIATION FROM DESIGN.md 6.2, stated plainly: that section specified Google
// Benchmark's one-parameter least-squares fit over candidate curves. This uses
// consecutive ratios instead. The sweep is mandated geometric either way, and
// on a geometric sweep ratios are directly interpretable and need no floating
// point. The cost is weaker separation between O(n) and O(n log n), which is
// reported honestly as an ambiguous band rather than guessed at. Least-squares
// remains the better answer if that band ever needs to be resolved.
//
// All arithmetic is fixed-point, scaled by 1000 ("milli-ratio"), so a ratio of
// 2.0 is 2000. El values are int64; this avoids float-in-list handling.
// Curve identifiers. Ordered by growth — the ordering IS the comparison used
// by the gate, so an index comparison decides "worse than declared".
// 0 = O(1) 1 = O(log n) 2 = O(n) 3 = O(n log n) 4 = O(n^2) 5 = O(n^3)
fn elb_curve_name(c: Int) -> String {
if c == 0 { return "O(1)" }
if c == 1 { return "O(log n)" }
if c == 2 { return "O(n)" }
if c == 3 { return "O(n log n)" }
if c == 4 { return "O(n^2)" }
if c == 5 { return "O(n^3)" }
return "O(?)"
}
fn elb_curve_from_name(s: String) -> Int {
if str_eq(s, "O(1)") { return 0 }
if str_eq(s, "O(log n)") { return 1 }
if str_eq(s, "O(n)") { return 2 }
if str_eq(s, "O(n log n)") { return 3 }
if str_eq(s, "O(n^2)") { return 4 }
if str_eq(s, "O(n^3)") { return 5 }
return -1
}
// elb_classify_ratio — map a milli-ratio-per-doubling onto a curve.
//
// Bands are deliberately wide at the top (a quadratic measured at 3.4x is
// still a quadratic) and deliberately overlap-averse at the bottom, where a
// misclassification between O(1) and O(log n) matters least.
fn elb_classify_ratio(milli: Int) -> Int {
if milli < 1300 { return 0 }
if milli < 1700 { return 1 }
if milli < 2400 { return 2 }
if milli < 3200 { return 3 }
if milli < 6000 { return 4 }
return 5
}
// elb_ratio — milli-ratio between two consecutive measurements.
// Returns -1 when the earlier measurement is zero (ratio undefined).
fn elb_ratio(prev: Int, cur: Int) -> Int {
if prev <= 0 { return -1 }
return (cur * 1000) / prev
}
// ── The measurement floor ────────────────────────────────────────────────────
//
// A benchmark whose largest measurement is at or near zero has not been
// measured. Reporting it as O(1) would be a confident answer with nothing
// behind it — the same failure as a test that never ran reporting pass, and
// exactly what happened when clang closed a nested loop to a multiply and the
// harness read 0 microseconds at every n.
//
// So: REFUSE. Never classify below the floor.
fn elb_below_floor(vals: [Int], floor: Int) -> Bool {
let n: Int = native_list_len(vals)
let i: Int = 0
let mx: Int = 0
while i < n {
let v: Int = native_list_get(vals, i)
if v > mx { let mx = v }
let i = i + 1
}
if mx < floor { return true }
return false
}
// elb_implausibly_flat — a measurement that does not move across a sweep whose
// input grew by 8x or more is not a flat curve, it is a broken measurement.
// Genuine O(1) work still shows noise; a hard-flat series means the work was
// optimised away, the timer has insufficient resolution, or the benchmark body
// never executed.
fn elb_implausibly_flat(vals: [Int]) -> Bool {
let n: Int = native_list_len(vals)
if n < 3 { return false }
let first: Int = native_list_get(vals, 0)
let last: Int = native_list_get(vals, n - 1)
if first == 0 {
if last == 0 { return true }
return false
}
let r: Int = (last * 1000) / first
if r < 1100 { return true }
return false
}
// elb_spread_ok — do the consecutive ratios agree with each other?
//
// This is the ratio-method analogue of a normalised-RMS threshold. If the
// doublings disagree wildly the data is noise, a cache cliff, or a phase
// change, and the honest report is INDETERMINATE rather than a classification.
// Applies to the ASYMPTOTIC TAIL only — the last three ratios.
//
// The small-n end of any sweep is dominated by fixed overhead, cold caches and
// branch predictors that have not warmed. Measured on a genuinely linear
// character scan, the ratios ran 3.37, 2.92, 1.76, 1.65: the head looks
// quadratic, the tail is the truth. Checking spread across the whole sweep
// therefore rejects correct data. A complexity bound is an asymptotic claim, so
// it is judged on the asymptotic region — the same reason a benchmark harness
// discards warmup rather than averaging it in.
fn elb_spread_ok(ratios: [Int]) -> Bool {
let total: Int = native_list_len(ratios)
if total < 2 { return true }
let start: Int = total - 3
if start < 0 { let start = 0 }
let n: Int = total
let lo: Int = 999999
let hi: Int = 0
let i: Int = start
while i < n {
let r: Int = native_list_get(ratios, i)
if r >= 0 {
if r < lo { let lo = r }
if r > hi { let hi = r }
}
let i = i + 1
}
if lo <= 0 { return false }
// Reject when the widest ratio is more than 2.2x the narrowest. That is
// enough slack for real timing noise and tight enough to separate a clean
// 2.0 series from a clean 4.0 series.
if (hi * 1000) / lo > 2200 { return false }
return true
}
// elb_ratios — consecutive milli-ratios across the sweep.
fn elb_ratios(vals: [Int]) -> [Int] {
let out: [Int] = native_list_empty()
let n: Int = native_list_len(vals)
let i: Int = 1
while i < n {
let out = native_list_append(out,
elb_ratio(native_list_get(vals, i - 1), native_list_get(vals, i)))
let i = i + 1
}
return out
}
// elb_mean_tail_ratio — mean of the LAST TWO ratios.
//
// The tail is used deliberately: asymptotic behaviour is what a complexity
// bound claims, and the small-n end of any sweep is dominated by fixed
// overhead. This is the same reason a benchmark harness discards warmup.
fn elb_mean_tail_ratio(ratios: [Int]) -> Int {
let n: Int = native_list_len(ratios)
if n == 0 { return -1 }
if n == 1 { return native_list_get(ratios, 0) }
let a: Int = native_list_get(ratios, n - 1)
let b: Int = native_list_get(ratios, n - 2)
if a < 0 { return b }
if b < 0 { return a }
return (a + b) / 2
}
// ── Verdicts ─────────────────────────────────────────────────────────────────
//
// 0 PASS measured curve is at or below the declared bound
// 1 FAIL measured curve is strictly worse than declared
// 2 INDETERMINATE ratios disagree; data is noise or a phase change
// 3 REFUSED below the measurement floor, or implausibly flat
// 4 BETTER measured strictly better than declared (warn, not fail)
fn elb_verdict_name(v: Int) -> String {
if v == 0 { return "PASS" }
if v == 1 { return "FAIL" }
if v == 2 { return "INDETERMINATE" }
if v == 3 { return "REFUSED" }
if v == 4 { return "BETTER" }
return "?"
}
// elb_gate — classify one signal against its declared bound.
//
// vals measurements, one per sweep point, in sweep order
// expect declared curve index (see elb_curve_name)
// floor minimum largest-measurement below which we refuse to classify
fn elb_gate(vals: [Int], expect: Int, floor: Int) -> Int {
if elb_below_floor(vals, floor) { return 3 }
if elb_implausibly_flat(vals) { return 3 }
let ratios: [Int] = elb_ratios(vals)
if !elb_spread_ok(ratios) { return 2 }
let m: Int = elb_mean_tail_ratio(ratios)
if m < 0 { return 2 }
let got: Int = elb_classify_ratio(m)
if got > expect { return 1 }
if got < expect { return 4 }
return 0
}
// elb_measured_curve — the classified curve for a signal, or -1 if unclassifiable.
fn elb_measured_curve(vals: [Int], floor: Int) -> Int {
if elb_below_floor(vals, floor) { return -1 }
if elb_implausibly_flat(vals) { return -1 }
let ratios: [Int] = elb_ratios(vals)
let m: Int = elb_mean_tail_ratio(ratios)
if m < 0 { return -1 }
return elb_classify_ratio(m)
}
lang/runtime/eltest.el
+194
View File
@@ -0,0 +1,194 @@
// runtime/eltest.el — El test framework runner (Phase 1).
//
// This is the RUNNER. It is written in El and consumes a registry that the
// compiler generates into the same translation unit when invoked as
// `elc --test`. Nothing here discovers tests; discovery already happened at
// compile time, which is what makes `--list` and filtering possible later.
//
// ── Architecture ─────────────────────────────────────────────────────────────
//
// The compiler lowers each `test "name" { ... }` block into a static C
// function and emits a static table of (name, fn) pairs plus a small set of
// index-based accessors. El has no function pointers, so the runner never
// sees one — it works entirely in indices:
//
// __el_reg_count() -> Int number of registered tests
// __el_reg_name(i) -> String test name at index i
// __el_reg_invoke(i) -> Int run test i, return its failure count
// __el_reg_last_ns() -> Int wall-clock ns of the last invoke
// __el_reg_msg() -> String first failure message of the last invoke
// __el_reg_asserts() -> Int assertions executed in the last invoke
// __el_opt_json() -> Int 1 if --json was passed
//
// Timing is taken in the generated C, immediately around the call, so no El
// call overhead lands inside the measurement.
//
// ── Output ───────────────────────────────────────────────────────────────────
//
// Structured events are the source of truth. The human renderer is written
// FROM the same fields the NDJSON renderer emits — never the reverse. Parsing
// human output back into structure is the one clear architectural mistake in
// Go's test tooling and we do not repeat it.
//
// Every result carries a duration. Always. A framework that cannot report how
// long its tests took cannot surface a performance regression, and a
// regression nobody can see is one nobody fixes.
// ── Small helpers (no imports — this file must stay self-contained) ──────────
// _elt_json_escape — minimal JSON string escaping for the NDJSON renderer.
fn _elt_json_escape(s: String) -> String {
let out: String = ""
let n: Int = str_len(s)
let i: Int = 0
while i < n {
let ch: String = str_slice(s, i, i + 1)
if str_eq(ch, "\"") {
let out = out + "\\\""
} else {
if str_eq(ch, "\\") {
let out = out + "\\\\"
} else {
if str_eq(ch, "\n") {
let out = out + "\\n"
} else {
if str_eq(ch, "\t") {
let out = out + "\\t"
} else {
if str_eq(ch, "\r") {
let out = out + "\\r"
} else {
let out = out + ch
}
}
}
}
}
let i = i + 1
}
return out
}
// _elt_pad3 — left-pad an integer to three digits (for the ms.fraction form).
fn _elt_pad3(v: Int) -> String {
if v < 10 { return "00" + int_to_str(v) }
if v < 100 { return "0" + int_to_str(v) }
return int_to_str(v)
}
// _elt_ms — render a nanosecond duration as "M.mmm" milliseconds.
//
// Deliberately avoids the modulo operator: the remainder is derived by
// subtraction so this stays portable across El backends.
fn _elt_ms(ns: Int) -> String {
let total_us: Int = ns / 1000
let ms_whole: Int = total_us / 1000
let us_rem: Int = total_us - (ms_whole * 1000)
return int_to_str(ms_whole) + "." + _elt_pad3(us_rem)
}
// _elt_secs — render a nanosecond duration as fractional seconds, for the
// NDJSON `elapsed` field. JUnit XML and test2json both use seconds-as-decimal.
fn _elt_secs(ns: Int) -> String {
let total_ms: Int = ns / 1000000
let s_whole: Int = total_ms / 1000
let ms_rem: Int = total_ms - (s_whole * 1000)
return int_to_str(s_whole) + "." + _elt_pad3(ms_rem)
}
// ── Event emission ───────────────────────────────────────────────────────────
//
// One function per event shape. Both renderers read the same fields; the
// human renderer is a projection of the event, not a separate code path.
fn _elt_emit_run(json_mode: Bool, name: String) {
if json_mode {
println("{\"action\":\"run\",\"test\":\"" + _elt_json_escape(name) + "\"}")
}
}
fn _elt_emit_result(json_mode: Bool, name: String, fails: Int, ns: Int, asserts: Int, msg: String) {
if json_mode {
let action: String = "pass"
if fails > 0 { let action = "fail" }
let line: String = "{\"action\":\"" + action + "\""
let line = line + ",\"test\":\"" + _elt_json_escape(name) + "\""
let line = line + ",\"elapsed\":" + _elt_secs(ns)
let line = line + ",\"assertions\":" + int_to_str(asserts)
if fails > 0 {
let line = line + ",\"failures\":" + int_to_str(fails)
let line = line + ",\"message\":\"" + _elt_json_escape(msg) + "\""
}
let line = line + "}"
println(line)
return
}
// Human renderer — duration is never optional.
if fails > 0 {
println("FAIL " + name + " (" + _elt_ms(ns) + "ms)")
println(" " + msg)
return
}
println("ok " + name + " (" + _elt_ms(ns) + "ms)")
return
}
fn _elt_emit_summary(json_mode: Bool, total: Int, failed: Int, ns: Int, asserts: Int) {
let passed: Int = total - failed
if json_mode {
let line: String = "{\"action\":\"summary\""
let line = line + ",\"tests\":" + int_to_str(total)
let line = line + ",\"passed\":" + int_to_str(passed)
let line = line + ",\"failed\":" + int_to_str(failed)
let line = line + ",\"assertions\":" + int_to_str(asserts)
let line = line + ",\"elapsed\":" + _elt_secs(ns)
let line = line + "}"
println(line)
return
}
println("")
println(int_to_str(total) + " tests, " + int_to_str(passed) + " passed, "
+ int_to_str(failed) + " failed, " + int_to_str(asserts) + " assertions in "
+ _elt_ms(ns) + "ms")
return
}
// ── The runner ───────────────────────────────────────────────────────────────
// el_test_main — drive the compile-time registry.
//
// Called from the generated main(). Returns the number of FAILING TESTS, which
// becomes the process exit code. Note that this counts tests, not assertions:
// a test is the unit of result. The old harness counted assertions globally and
// therefore could not say which test failed, how long any of them took, or
// whether a test had run at all.
fn el_test_main() -> Int {
let json_mode: Bool = false
if __el_opt_json() == 1 { let json_mode = true }
let n: Int = __el_reg_count()
let i: Int = 0
let failed: Int = 0
let total_ns: Int = 0
let total_asserts: Int = 0
while i < n {
let name: String = __el_reg_name(i)
_elt_emit_run(json_mode, name)
let fails: Int = __el_reg_invoke(i)
let ns: Int = __el_reg_last_ns()
let asserts: Int = __el_reg_asserts()
let msg: String = __el_reg_msg()
let total_ns = total_ns + ns
let total_asserts = total_asserts + asserts
if fails > 0 { let failed = failed + 1 }
_elt_emit_result(json_mode, name, fails, ns, asserts, msg)
let i = i + 1
}
_elt_emit_summary(json_mode, n, failed, total_ns, total_asserts)
return failed
}
lang/runtime/engram_cognition.c
+372 -29
View File
@@ -246,14 +246,6 @@ static int put_edge(EngramPagedStore* s, const char* id, const char* from, const
e.metadata = (char*)meta;
return store_put_edge(s, &e);
}
int cog_ground_edge(EngramPagedStore* s, const char* claim_id,
const char* evidence_id, double grounding, const char* for_whom) {
if (!s || !claim_id || !evidence_id) return -1;
char id[512], meta[256];
snprintf(id, sizeof id, "gb-%s-%s-%s", claim_id, evidence_id, for_whom ? for_whom : "global");
snprintf(meta, sizeof meta, "for_whom=%s", for_whom ? for_whom : "-");
return put_edge(s, id, claim_id, evidence_id, COG_GROUNDED_BY_RELATION, grounding, meta);
}
int cog_salient_edge(EngramPagedStore* s, const char* node_id,
const char* observer_id, double salience) {
if (!s || !node_id || !observer_id) return -1;
@@ -261,35 +253,386 @@ int cog_salient_edge(EngramPagedStore* s, const char* node_id,
snprintf(id, sizeof id, "st-%s-%s", node_id, observer_id);
return put_edge(s, id, node_id, observer_id, COG_SALIENT_TO_RELATION, salience, NULL);
}
int cog_assert_gate(EngramPagedStore* s, const char* claim_id,
const char* for_whom, double floor) {
if (!s || !claim_id) return -1;
if (!(floor > 0)) floor = 0.5;
StoreEdge* edges = NULL; size_t n = 0;
if (store_get_edges_from(s, claim_id, &edges, &n) < 0) return -1;
double best = 0.0; int found = 0;
for (size_t i = 0; i < n; i++) {
if (!edges[i].relation || strcmp(edges[i].relation, COG_GROUNDED_BY_RELATION) != 0) continue;
/* grounded-for-whom: match observer if requested; global (for_whom=-) always counts */
int match = 1;
if (for_whom && edges[i].metadata) {
const char* fw = strstr(edges[i].metadata, "for_whom=");
if (fw) { fw += 9; if (strcmp(fw, for_whom) != 0 && strcmp(fw, "-") != 0) match = 0; }
}
if (match) { found = 1; if (edges[i].weight > best) best = edges[i].weight; }
}
store_edges_free(edges, n);
if (!found) return 0; /* ungrounded => refuse assertion (still held) */
return (best >= floor) ? 1 : 0;
/* ═══════════════════════════════════════════════════════════════════════════
* §7 GROUNDING IS THE EDGE'S WEIGHT, AND THE WEIGHT IS A VECTOR.
* See engram_cognition.h §7 for the model and for the measurements the two
* design decisions (thirteen regions, min aggregate) rest on.
* ═══════════════════════════════════════════════════════════════════════════ */
/* ── The one decay model. Moved here verbatim from el_runtime.c's
* engram_temporal_decay so nodes and edges share a single implementation and a
* single set of constants; engram_temporal_decay now delegates. Bit-identical
* for nodes: reinforcements := activation_count, lambda_override :=
* temporal_decay_rate.
*
* This is what makes decay ANALYTIC rather than sampled: between two recorded
* versions the trajectory is not unknown, it is known in closed form from the
* last point and elapsed time. Store the point, read the curve. */
double cog_decay_factor(int64_t age_ms, double reinforcements, double lambda_override) {
if (age_ms <= 0) return 1.0;
double lambda = (lambda_override > 0.0) ? lambda_override : COG_DECAY_LAMBDA;
double age_hours = (double)age_ms / 3600000.0;
if (reinforcements < 0) reinforcements = 0;
double t_half = COG_T_HALF_HOURS * (1.0 + log(1.0 + reinforcements));
double factor = exp(-lambda * age_hours / t_half);
if (factor < COG_DECAY_FLOOR) factor = COG_DECAY_FLOOR;
return factor;
}
const char* cog_prov_name(CogProvClass p) {
switch (p) {
case COG_PROV_OBSERVED: return "observed";
case COG_PROV_INFERRED: return "inferred";
case COG_PROV_TOLD: return "told";
case COG_PROV_IMPRINTED: return "imprinted";
default: return "unset";
}
}
CogProvClass cog_prov_parse(const char* s) {
if (!s) return COG_PROV_UNSET;
if (!strcmp(s, "observed")) return COG_PROV_OBSERVED;
if (!strcmp(s, "inferred")) return COG_PROV_INFERRED;
if (!strcmp(s, "told")) return COG_PROV_TOLD;
if (!strcmp(s, "imprinted")) return COG_PROV_IMPRINTED;
return COG_PROV_UNSET;
}
/* Locate the GRD1 block in an edge's metadata. It is always the tail; anything
* ahead of it is the edge's pre-existing metadata, preserved verbatim. */
static const char* cog_grd_find(const char* meta) {
if (!meta) return NULL;
size_t ml = strlen(COG_GROUNDING_META_MAGIC);
if (strncmp(meta, COG_GROUNDING_META_MAGIC, ml) == 0) return meta;
const char* p = meta;
while ((p = strstr(p, COG_GROUNDING_META_MAGIC)) != NULL) {
if (p > meta && p[-1] == '\n') return p;
p += ml;
}
return NULL;
}
int cog_grounding_parse(const StoreEdge* e, int64_t now_ms, CogGrounding* out) {
if (!e || !out) return -1;
memset(out, 0, sizeof *out);
/* Two dimensions exist on every edge whether or not grounding has ever been
* established, because they ARE existing substrate rather than new fields:
* associative — the accrued hebb, with its existing dynamics;
* polarity — the signed authored weight. `inhibitory` is precisely this
* distinction crushed to one bit, so it is the seed sign. */
out->associative = e->hebb;
out->polarity = e->inhibitory ? -e->weight : e->weight;
out->prov = COG_PROV_UNSET;
out->ts = e->last_fired > 0 ? e->last_fired : e->updated_at;
const char* blk = cog_grd_find(e->metadata);
if (blk) {
out->present = 1;
char* copy = dupstr(blk);
if (!copy) return -1;
for (char* line = strtok(copy, "\n"); line; line = strtok(NULL, "\n")) {
if (line[0] == '\0') continue;
char tag = line[0];
const char* rest = line + 1; while (*rest == ' ') rest++;
if (tag == 'w') { /* the four numeric dimensions */
double v[4] = {0,0,0,0}; parse_floats(rest, v, 4);
out->factual = v[0]; out->relational = v[1];
out->associative = v[2]; out->polarity = v[3];
} else if (tag == 'k') { /* provenance class */
out->prov = cog_prov_parse(rest);
} else if (tag == 't') { /* timestamp + seq + reinforcements */
double v[3] = {0,0,0}; parse_floats(rest, v, 3);
out->ts = (int64_t)v[0]; out->seq = (int64_t)v[1]; out->reinforcements = v[2];
} else if (tag == 'd') {
double v[3] = {0,0,0}; parse_floats(rest, v, 3);
out->fac_proj = v[0]; out->rel_proj = v[1]; out->cos_angle = v[2];
} else if (tag == 'v') {
snprintf(out->binding_value, sizeof out->binding_value, "%s", rest);
} else if (tag == 'c') {
double v[2] = {0,0}; parse_floats(rest, v, 2);
out->floor_at_record = v[0]; out->rel_floor_at_record = v[1];
} else if (tag == 'p') {
snprintf(out->prev_edge, sizeof out->prev_edge, "%s", rest);
}
}
free(copy);
}
out->agreement = (out->cos_angle > 0) ? 1 : (out->cos_angle < 0 ? -1 : 0);
/* ── DERIVED. Nothing below this line is ever serialized. Recency, decay and
* staleness are read off the curve; storing them is how a number ends up
* asserting something nothing computed (§8.1 / spec §2). */
out->age_ms = (out->ts > 0 && now_ms > out->ts) ? (now_ms - out->ts) : 0;
out->decay = cog_decay_factor(out->age_ms, out->reinforcements, 0.0);
out->factual_now = out->factual * out->decay;
out->relational_now = out->relational * out->decay;
out->associative_now = out->associative * out->decay;
out->stale = (out->present && out->floor_at_record > 0 &&
out->factual_now < out->floor_at_record) ? 1 : 0;
return 0;
}
char* cog_grounding_metadata(const char* base_meta, const CogGrounding* g) {
if (!g) return NULL;
size_t keep = 0;
if (base_meta) {
const char* blk = cog_grd_find(base_meta);
keep = blk ? (size_t)(blk - base_meta) : strlen(base_meta);
while (keep > 0 && base_meta[keep - 1] == '\n') keep--;
}
size_t cap = keep + 1024;
char* buf = malloc(cap); if (!buf) return NULL;
size_t o = 0;
if (keep) { memcpy(buf, base_meta, keep); o = keep; buf[o++] = '\n'; }
o += (size_t)snprintf(buf + o, cap - o, "%s\n", COG_GROUNDING_META_MAGIC);
/* STORED ONLY. factual / relational / associative / polarity / provenance /
* timestamp — plus the joint state a decision saw. No confidence, no
* recency, no staleness, no volatility: those are read off the curve. */
o += (size_t)snprintf(buf + o, cap - o, "w %.9g %.9g %.9g %.9g\n",
g->factual, g->relational, g->associative, g->polarity);
o += (size_t)snprintf(buf + o, cap - o, "k %s\n", cog_prov_name(g->prov));
o += (size_t)snprintf(buf + o, cap - o, "t %lld %lld %.9g\n",
(long long)g->ts, (long long)g->seq, g->reinforcements);
o += (size_t)snprintf(buf + o, cap - o, "d %.9g %.9g %.9g\n",
g->fac_proj, g->rel_proj, g->cos_angle);
o += (size_t)snprintf(buf + o, cap - o, "v %s\n", g->binding_value[0] ? g->binding_value : "-");
o += (size_t)snprintf(buf + o, cap - o, "c %.9g %.9g\n", g->floor_at_record, g->rel_floor_at_record);
if (g->prev_edge[0]) o += (size_t)snprintf(buf + o, cap - o, "p %s\n", g->prev_edge);
(void)o;
return buf;
}
/* ── Consequence, not epsilon. Every test is a floor crossing or a sign change,
* both exact. Ordered so the two INHERENT (discrete) moves are reported in
* preference to the graded ones, because they bypass the salience gate. */
CogSignificance cog_grounding_significant(const CogGrounding* prev,
const CogGrounding* now,
double floor, double rel_floor) {
if (!now) return COG_SIG_NONE;
if (!prev || !prev->present) return COG_SIG_FIRST_RECORD;
/* INHERENT 1 — polarity sign flip. Ignorance and disagreement are different
* states, and support → contradiction is a change of state rather than a
* drift, so no threshold applies. Comparing signs, with zero its own class. */
{
int sp = prev->polarity > 0 ? 1 : (prev->polarity < 0 ? -1 : 0);
int sn = now->polarity > 0 ? 1 : (now->polarity < 0 ? -1 : 0);
if (sp != sn) return COG_SIG_POLARITY_FLIP;
}
/* INHERENT 2 — provenance class change. told → observed is a categorical
* upgrade in what the relation is entitled to, not a movement along an axis. */
if (prev->prov != now->prov) return COG_SIG_PROVENANCE_CHANGE;
/* Crossing an assert floor — the move changes whether this relation can be
* spoken. Compared on the DECAYED values, because that is what the gate reads. */
if ((prev->factual_now >= floor) != (now->factual_now >= floor)) return COG_SIG_FACTUAL_FLOOR;
if ((prev->relational_now >= rel_floor) != (now->relational_now >= rel_floor)) return COG_SIG_RELATIONAL_FLOOR;
/* Flipping factual/relational agreement — the relation stops being "true and
* meaningful" and becomes "true and misapplied", or the reverse. This is the
* 911/CPS contradiction as a measured event rather than a reviewable one. */
if (prev->agreement != now->agreement) return COG_SIG_AGREEMENT_FLIP;
/* A gradient reversing — the evidence stopped pulling the claim toward it and
* began pushing it away, or the same on the values axis. */
if ((prev->fac_proj > 0) != (now->fac_proj > 0)) return COG_SIG_DIRECTION_REVERSAL;
if ((prev->rel_proj > 0) != (now->rel_proj > 0)) return COG_SIG_DIRECTION_REVERSAL;
return COG_SIG_NONE;
}
int cog_significance_inherent(CogSignificance s) {
return (s == COG_SIG_FIRST_RECORD || s == COG_SIG_POLARITY_FLIP ||
s == COG_SIG_PROVENANCE_CHANGE) ? 1 : 0;
}
const char* cog_significance_name(CogSignificance s) {
switch (s) {
case COG_SIG_FIRST_RECORD: return "first-record";
case COG_SIG_POLARITY_FLIP: return "polarity-sign-flip";
case COG_SIG_PROVENANCE_CHANGE: return "provenance-class-change";
case COG_SIG_FACTUAL_FLOOR: return "factual-floor-crossed";
case COG_SIG_RELATIONAL_FLOOR: return "relational-floor-crossed";
case COG_SIG_AGREEMENT_FLIP: return "agreement-sign-flip";
case COG_SIG_DIRECTION_REVERSAL: return "gradient-direction-reversal";
default: return "none";
}
}
/* ── Recording: a NEW edge record. The predecessor is never touched. ────────── */
int cog_grounding_record(EngramPagedStore* s, const StoreEdge* base,
const CogGrounding* g, char* out_id, size_t out_id_cap) {
if (!s || !base || !base->id || !g) return -1;
char root[192];
snprintf(root, sizeof root, "%s", base->id);
char* hash = strchr(root, '#'); if (hash) *hash = '\0';
int seq = (int)g->seq + 1;
char vid[224];
snprintf(vid, sizeof vid, "%s#%d", root, seq);
CogGrounding rec = *g;
rec.seq = seq;
snprintf(rec.prev_edge, sizeof rec.prev_edge, "%s", base->id);
char* meta = cog_grounding_metadata(base->metadata, &rec);
if (!meta) return -1;
StoreEdge e; memset(&e, 0, sizeof e);
e.id = vid; e.from_id = base->from_id; e.to_id = base->to_id;
e.relation = base->relation; e.metadata = meta;
/* The vector IS the weight, so the scalar fields carry their dimensions:
* `weight` the magnitude of polarity, `inhibitory` its sign, `hebb` the
* associative strength. Nothing here is a second copy of a derived value. */
e.weight = rec.polarity < 0 ? -rec.polarity : rec.polarity;
e.inhibitory = rec.polarity < 0 ? 1 : 0;
e.hebb = rec.associative;
e.confidence = base->confidence;
e.created_at = base->created_at;
e.updated_at = rec.ts;
e.last_fired = rec.ts;
e.layer_id = base->layer_id;
int rc = store_put_edge(s, &e);
free(meta);
if (rc != 0) return -1;
if (out_id && out_id_cap) snprintf(out_id, out_id_cap, "%s", vid);
return seq;
}
int cog_grounding_head(EngramPagedStore* s, const char* base_id,
StoreEdge* out, int max_versions) {
if (!s || !base_id || !out) return -1;
if (max_versions <= 0) max_versions = 64;
char root[192]; snprintf(root, sizeof root, "%s", base_id);
char* hash = strchr(root, '#'); if (hash) *hash = '\0';
StoreEdge cur; memset(&cur, 0, sizeof cur);
if (store_get_edge(s, root, &cur) != 1) return -1;
int found = 0;
for (int v = 1; v <= max_versions; v++) {
char vid[224]; snprintf(vid, sizeof vid, "%s#%d", root, v);
StoreEdge nx;
if (store_get_edge(s, vid, &nx) != 1) break;
store_edge_free(&cur); cur = nx; found = v;
}
*out = cur;
return found;
}
/* ── VOLATILITY AND DRIFT: derived from the chain, stored nowhere. The series
* exists only because nothing was destroyed, which is the whole return on
* immutability — a derivative for free. */
int cog_grounding_trajectory(EngramPagedStore* s, const char* base_id,
int64_t now_ms, CogTrajectory* out) {
if (!s || !base_id || !out) return -1;
memset(out, 0, sizeof *out);
char root[192]; snprintf(root, sizeof root, "%s", base_id);
char* hash = strchr(root, '#'); if (hash) *hash = '\0';
double pf = 0, pr = 0, f0 = 0, r0 = 0, fN = 0, rN = 0;
double sum_df = 0, sum_dr = 0;
int n = 0;
for (int v = 0; v <= 64; v++) {
char vid[224];
if (v == 0) snprintf(vid, sizeof vid, "%s", root);
else snprintf(vid, sizeof vid, "%s#%d", root, v);
StoreEdge e;
if (store_get_edge(s, vid, &e) != 1) { if (v) break; else continue; }
CogGrounding g;
if (cog_grounding_parse(&e, now_ms, &g) == 0) {
if (n == 0) { f0 = g.factual; r0 = g.relational; }
else { sum_df += fabs(g.factual - pf); sum_dr += fabs(g.relational - pr); }
pf = g.factual; pr = g.relational; fN = pf; rN = pr;
n++;
}
store_edge_free(&e);
}
out->n_versions = n;
if (n > 1) {
out->factual_volatility = sum_df / (double)(n - 1);
out->relational_volatility = sum_dr / (double)(n - 1);
}
out->factual_drift = fN - f0;
out->relational_drift = rN - r0;
/* "STAYED TRUE, BECAME WRONG" — the event the joint record makes visible and
* that per-dimension versioning would have destroyed: the fact held while
* the meaning degraded. Expressed as signs, so there is no tolerance here
* either: factual did not fall, relational did. */
out->stayed_true_became_wrong =
(n > 1 && out->factual_drift >= 0 && out->relational_drift < 0) ? 1 : 0;
return 0;
}
/* ── Assertion gates on BOTH floors. Traversal is untouched: activation still
* conducts on the factual/associative side, so a relation can remain thinkable
* while ceasing to be assertable. That gap is where the wide angles live. ──── */
int cog_assert_two_axis(EngramPagedStore* s, const char* claim_id,
double floor, double rel_floor, int64_t now_ms,
CogAssertion* out) {
if (!s || !claim_id || !out) return -1;
memset(out, 0, sizeof *out);
if (!(floor > 0)) floor = 0.5;
if (!(rel_floor > 0)) rel_floor = floor;
/* still_held is DERIVED, not a literal (§8.1). Holding is unconditional —
* the store gates nothing — so the question the field actually answers is
* whether the content is present and live. */
StoreNode n;
if (store_get_node(s, claim_id, &n) == 1) { out->still_held = !n.tombstoned; store_node_free(&n); }
else out->still_held = 0;
double best = -1.0;
for (int dir = 0; dir < 2; dir++) {
StoreEdge* edges = NULL; size_t ne = 0;
int rc = dir == 0 ? store_get_edges_from(s, claim_id, &edges, &ne)
: store_get_edges_to (s, claim_id, &edges, &ne);
if (rc < 0) continue;
for (size_t i = 0; i < ne; i++) {
if (edges[i].tombstoned) continue;
CogGrounding g;
if (cog_grounding_parse(&edges[i], now_ms, &g) != 0) continue;
out->n_edges++;
out->found = 1;
if (g.factual_now > best) {
best = g.factual_now;
out->factual = g.factual_now;
out->relational = g.relational_now; /* the SAME edge, not a max */
out->polarity = g.polarity;
out->cos_angle = g.cos_angle;
out->agreement = g.agreement;
out->prov = g.prov;
out->relational_established = g.present;
snprintf(out->best_edge, sizeof out->best_edge, "%s", edges[i].id ? edges[i].id : "");
snprintf(out->binding_value, sizeof out->binding_value, "%s", g.binding_value);
}
}
store_edges_free(edges, ne);
}
/* BOTH floors, and an unestablished relational axis does NOT pass by default
* — defaulting it to passing is the exemption §0 forbids. A negative polarity
* is a relation that actively contradicts and can never license assertion. */
out->may_assert = (out->found && out->relational_established &&
out->polarity > 0 &&
out->factual >= floor && out->relational >= rel_floor) ? 1 : 0;
return 0;
}
/* ═══════════════════════════════════════════════ THE CORRESPONDENCE-LOOP ═════ */
int engram_correspondence_beat(const GeoDescriptor* region, const float* anchor,
double outcome_y, CogStance* stance,
int learn, double max_step, CogBeatResult* out) {
if (!region || !stance || !out) return -1;
memset(out, 0, sizeof *out);
if (stance->keystone) { learn = 0; out->wrote_keystone = 1; } /* §6: never write a keystone */
/* 2026-08-16: the keystone block is GONE. It refused to learn about the
* reference frame, which does not make it a good reference — it makes it
* unexaminable, trading circular calibration for an ungroundable one (spec
* §2). Measured cost of the block: on the keystone region the beat reported
* 0.00% brier reduction over n_trials 0 — it never ran, so nothing about the
* self was ever calibrated OR falsifiable. What replaces it is a provenance
* constraint, not a permission: cog_grounding_downstream refuses evidence
* that is downstream of the region being calibrated, for every region alike.
* `wrote_keystone` is retained as a reporting field only and is always 0. */
GeoGradient g;
if (engram_think(region, anchor, stance, &g) != 0) return -1; /* PREDICTION */
lang/runtime/engram_cognition.h
+269 -17
View File
@@ -23,7 +23,7 @@
* and a region, and grounded-for-whom.
*
* PURE + (mostly) READ-ONLY, stdlib + libm only. think() and the warp are pure
* over their inputs. Persistence (Stance <-> StoreNode, grounded-by edges) is the
* over their inputs. Persistence (Stance <-> StoreNode, edge grounding vectors) is the
* only part that touches the store, and it is additive / supersede / tombstone —
* never mutate-in-place, never delete. It NEVER touches the live daemon: all
* offline against a scratch store, per the design's rails.
@@ -152,30 +152,25 @@ int engram_express(const GeoGradient* g, const float* anchor, float* out_point);
/* ═══════════════════════════════════════════════════════════════════════════
* §5 HOLD vs GROUND vs ASSERT. Holding is unconditional (the store gates nothing).
* Grounding is a RELATION — a "grounded-by" edge, probabilistic, grounded-for-whom.
* The honesty floor is checked only at ASSERTION.
* Grounding is an ATTRIBUTE OF a relation — carried on the edge itself, as a
* vector (§7). The honesty floor is checked only at ASSERTION, on both axes.
* ═══════════════════════════════════════════════════════════════════════════ */
#define COG_GROUNDED_BY_RELATION "grounded-by"
/* DELETED 2026-08-16: COG_GROUNDED_BY_RELATION and cog_ground_edge.
*
* A "grounded-by" edge models grounding as a relation BETWEEN two nodes. It is a
* property OF a relation — and it is that relation's weight. Minting a new edge
* to carry a score was the error; #147 corrected which endpoints the edge landed
* on and left the wrong idea standing. There is nothing to ground a claim
* "against" that is not already an edge, and if no edge exists the honest answer
* is that the two are not related — not a freshly minted one scoring 0.98.
* See §7 for what replaced it. */
#define COG_SALIENT_TO_RELATION "salient-to"
/* Write a grounded-by edge (additive). weight = grounding ∈(0,1] from the verifier;
* for_whom recorded in edge metadata (grounding is relational). Never a node flag. */
int cog_ground_edge(EngramPagedStore* s, const char* claim_id,
const char* evidence_id, double grounding, const char* for_whom);
/* Write/refresh a salient-to edge: salience is RELATIONAL (grounded-for-whom),
* carried on the edge to the observer — not baked into the node scalar (§2.1). */
int cog_salient_edge(EngramPagedStore* s, const char* node_id,
const char* observer_id, double salience);
/* The honesty floor — a QUERY at assertion time, NOT a schema constraint. Reads the
* claim's stored grounded-by edges (for the given observer) and returns:
* 1 = may assert (best grounding >= floor),
* 0 = REFUSE assertion (holds unconditionally; only asserting is gated),
* <0 = error. The content remains held either way. */
int cog_assert_gate(EngramPagedStore* s, const char* claim_id,
const char* for_whom, double floor);
/* ═══════════════════════════════════════════════════════════════════════════
* §4 THE REFLEXIVE CORRESPONDENCE-LOOP — the learning engine. think scores its
* OWN gradient against outcome, refines the stance on the error, and (optionally)
@@ -209,8 +204,265 @@ int engram_correspondence_beat(const GeoDescriptor* region, const float* anchor,
/* ═══════════════════════════════════════════════════════════════════════════
* §6 METASTABILITY. Keystones (self/values) are read-mostly: the loop reads but
* never writes them. Mark by stance flag or by a keystone-id set the loop consults.
*
* SUPERSEDED BY §7's PROVENANCE CONSTRAINT (2026-08-16). The keystone flag is a
* PERMISSION: it asks who the target is, not where the evidence came from. That
* is censorship, and it costs the ability to ever ground the self (spec
* correspondence-and-censorship.md §0/§2). The constraint that actually protects
* a reference frame is cog_grounding_downstream: a region may not be calibrated
* by evidence downstream of itself. These declarations remain only so existing
* call sites keep compiling; nothing in the grounding path consults them.
* ═══════════════════════════════════════════════════════════════════════════ */
typedef struct { const char** ids; int n; } CogKeystoneSet;
int cog_is_keystone(const CogKeystoneSet* ks, const CogStance* s);
/* ═══════════════════════════════════════════════════════════════════════════
* §7 GROUNDING IS THE EDGE'S WEIGHT, AND THE WEIGHT IS A VECTOR
* (2026-08-16; spec correspondence-and-censorship.md §2–§6 @ 2b7e4ba.)
*
* THE MODEL. Grounding is not a subsystem, a score, or a relation BETWEEN nodes.
* It is an attribute OF a relation. The graph already IS the grounding structure:
* every edge is a grounded relation, and what that relation is worth is carried
* on the edge itself. Three things follow, and each DELETES rather than adds:
*
* 1. `grounded-by` as a relation type does not exist, and cog_ground_edge is
* gone. Minting an edge to hold a score models grounding as a relation
* between nodes when it is a property of a relation. #147 corrected which
* endpoints that edge landed on and left the wrong idea standing.
* 2. There is no observer, and no sampling rate. Change is not a consequence of
* use — it IS use, the way potentiation is the firing rather than something
* that reads the firing and writes a weight. So no supervisor compares a
* value to a threshold and decides to persist.
* 3. Between two recorded versions the trajectory is not unknown. Decay is a
* pure function of the last recorded point and elapsed time, so it is
* ANALYTIC: store the point, read the curve.
*
* WHAT IS *NOT* HERE, DELIBERATELY. An earlier draft of the spec posed "a graph
* predicate for evidence downstream of itself" as the hard problem, and this file
* briefly contained one. It is withdrawn. Non-circularity is TEMPORAL, not
* topological: you cannot recalibrate the ruler while measuring with it, so you
* do it when you are not using the frame to act. Reachability could never have
* worked — measured on the live store, reachability from the self region over
* all relations reaches 89.2% of the graph (10,580 of 11,861 nodes) and 16.0%
* over hebbian/semantic relations alone, so the predicate marks essentially all
* evidence tainted and the constraint degenerates into the total block that
* censorship started as. Nothing replaces it here; the independence is a fact
* about engagement, owned by the dreamer, not a fact about the graph.
*
* ═══════════════════════════════════════════════════════════════════════════
* §7.1 THE VECTOR
*
* The test for a real dimension is whether it can move independently of the
* others. Five can, and each maps onto substrate that already exists:
*
* factual correspondence with evidence. [GRD1]
* relational correspondence with values — min over THIRTEEN
* value regions, carrying the binding value's NAME. [GRD1]
* associative co-activation frequency. This is the edge's `hebb`
* field with its existing dynamics — NOT a new one.
* Independent by construction: every superstition is
* a strong association with no factual grounding.
* polarity SIGNED. Near zero means "no support"; NEGATIVE means
* "this actively contradicts". The edge's `inhibitory`
* bit is exactly this distinction crushed to one bit,
* and is carried forward as the seed value. [GRD1]
* provenance observed / inferred / told / imprinted. Categorical,
* and load-bearing: it governs what the relation is
* entitled to. [GRD1]
*
* Plus a TIMESTAMP, which is what turns the supersession chain into a time
* series of vectors rather than a series of numbers.
*
* DERIVED, THEREFORE NEVER STORED. Confidence (high grounding AND low
* volatility), recency (decay read off the curve), staleness (grounding fallen
* below its floor), volatility (the derivative of a series nothing destroyed).
* Storing confidence separately is how `confidence: 0.5` ends up sitting beside
* a zero direction vector, asserting something nothing computed. Every field in
* CogGrounding below is marked STORED or DERIVED, and the serializer writes
* only the STORED ones.
*
* THE VALUES REFERENCE IS THIRTEEN REGIONS AND THE AGGREGATE IS MIN.
* Measured on the live store: the values root kn-5b606390 `contains` exactly 13
* value nodes; pairwise centroid cosine among their regions is min 0.1525,
* mean 0.5199, median 0.5282, max 0.9278 — they demonstrably do not form one
* region. Against a single union region the individual values sit at cosine
* 0.38..0.89, with constraints-as-freedom at 0.3812 and change-is-the-signal at
* 0.4677, so a union centroid under-represents precisely the values a claim is
* most likely to be measured against. MIN rather than MEAN because a mean lets
* strong agreement with twelve values mask a violation of the thirteenth, which
* is the mechanism of rationalization; min yields a binding constraint with a
* NAME attached rather than a score.
*
* TRAVERSAL CONDUCTS ON FACTUAL; ASSERTION REQUIRES BOTH. If activation
* conducted on relational weight, Neuron could not follow a chain of reasoning
* to a conclusion he then rejects — censorship arriving through the spreading
* rule. The gap between reachable and assertable is where the wide
* factual/relational angles live, and that gap is the interesting part.
* ═══════════════════════════════════════════════════════════════════════════ */
/* ── The one decay model (moved here from el_runtime.c so that node decay and
* edge-grounding decay are a single implementation with a single set of
* constants, rather than a model and a parallel copy of it). Half-life scales
* with how established the thing is: T_eff = T_HALF · (1 + ln(1 + reinforcements)).
* The floor is a preference, not a cliff — max penalty for age alone is 4x.
* `lambda_override` > 0 replaces the default rate; 0 means use the default. */
#define COG_T_HALF_HOURS 168.0
#define COG_DECAY_LAMBDA 0.693147
#define COG_DECAY_FLOOR 0.25
double cog_decay_factor(int64_t age_ms, double reinforcements, double lambda_override);
/* The compact vector block carried in the edge's own metadata. Line schema, same
* precedent as STNC1 / GEO1. Metadata the edge already carried is preserved
* verbatim ahead of the magic line. */
#define COG_GROUNDING_META_MAGIC "GRD1"
/* Provenance class — categorical, and it governs what the relation is entitled
* to. A change of class is inherently significant and needs no threshold,
* because told → observed is a categorical upgrade, not a drift. */
typedef enum {
COG_PROV_UNSET = 0,
COG_PROV_OBSERVED = 1,
COG_PROV_INFERRED = 2,
COG_PROV_TOLD = 3,
COG_PROV_IMPRINTED = 4
} CogProvClass;
const char* cog_prov_name(CogProvClass p);
CogProvClass cog_prov_parse(const char* s);
typedef struct {
int present; /* 1 iff the edge carries a GRD1 block */
/* ── STORED: the vector, as it stood at `ts` ─────────────────────────────── */
double factual; /* correspondence with evidence */
double relational; /* min over the thirteen value regions */
double associative; /* co-activation frequency — mirrors edge->hebb */
double polarity; /* SIGNED support; <0 = actively contradicts */
CogProvClass prov; /* observed / inferred / told / imprinted */
int64_t ts; /* when this version was recorded (ms) */
int64_t seq; /* supersession sequence number */
double reinforcements; /* uses folded into this version */
char binding_value[128]; /* the argmin value — the conflict's NAME */
/* the two gradients as frame-independent signed projections, plus the angle
* between them in full R^dim. These are part of the JOINT STATE a decision
* saw, not a convenience: near +1 evidence and values push the same way; at
* or below 0 the relation is factually supported and relationally wrong. */
double fac_proj, rel_proj, cos_angle;
int agreement; /* sign(cos_angle): +1 / 0 / −1 */
double floor_at_record, rel_floor_at_record;
char prev_edge[192]; /* the version this superseded ("" if first) */
/* ── DERIVED at read time. NEVER serialized. ─────────────────────────────── */
int64_t age_ms; /* recency: now − ts */
double decay; /* cog_decay_factor over that age */
double factual_now; /* factual · decay */
double relational_now;
double associative_now;
int stale; /* grounding fallen below its floor */
} CogGrounding;
/* Read an edge's vector as of `now_ms`. Pure — never writes. An edge with no
* GRD1 block still has an associative strength (its accrued hebb) and a polarity
* (its signed authored weight); `present` says whether the grounding dimensions
* have ever been established, and an unestablished dimension is reported as such
* rather than defaulted to a passing value. */
int cog_grounding_parse(const StoreEdge* e, int64_t now_ms, CogGrounding* out);
/* Serialize the STORED half of the vector, preserving pre-existing non-GRD1
* metadata. Returns an owned string. Derived fields are not written. */
char* cog_grounding_metadata(const char* base_meta, const CogGrounding* g);
/* ── §7.2 CONSOLIDATION-GATED SUPERSESSION ──────────────────────────────────
*
* Supersession is not recording — it is CONSOLIDATION, gated by salience, which
* is why you remember the argument and not the commute. Significance is
* evaluated PER-DIMENSION but the record is the WHOLE VECTOR: any dimension
* moving enough to matter triggers a supersession, and the new version captures
* every dimension as it stood at that instant. Versioning axes independently
* would make the joint state unreconstructable, and the joint state is the point
* — it is what makes "stayed true, became wrong" visible as an event (factual
* holding steady across versions while relational degrades).
*
* There is deliberately no epsilon in this enum or in the function that computes
* it. Every test is a floor crossing or a sign change, both exact. Two of them
* are INHERENTLY significant because they are discrete state changes rather than
* drift, and those bypass the salience gate entirely. */
typedef enum {
COG_SIG_NONE = 0, /* nothing decision-relevant moved — DO NOT RECORD */
COG_SIG_FIRST_RECORD = 1, /* no prior version exists */
COG_SIG_POLARITY_FLIP = 2, /* INHERENT: support ↔ contradiction, or ignorance
* ↔ either. A discrete change of state. */
COG_SIG_PROVENANCE_CHANGE = 3, /* INHERENT: told → observed is a categorical
* upgrade in what the relation is entitled to. */
COG_SIG_FACTUAL_FLOOR = 4, /* crossed the assert floor, factual axis */
COG_SIG_RELATIONAL_FLOOR = 5, /* crossed the assert floor, relational axis */
COG_SIG_AGREEMENT_FLIP = 6, /* factual/relational agreement changed sign */
COG_SIG_DIRECTION_REVERSAL = 7 /* a gradient reversed direction */
} CogSignificance;
CogSignificance cog_grounding_significant(const CogGrounding* prev,
const CogGrounding* now,
double floor, double rel_floor);
const char* cog_significance_name(CogSignificance s);
/* 1 iff this reason is a discrete state change that consolidates regardless of
* salience (polarity flip, provenance change, first record). */
int cog_significance_inherent(CogSignificance s);
/* ── §7.3 RECORDING: supersession of the EDGE, never an overwrite ────────────
* Writes version seq+1 as a NEW edge record with the same endpoints and relation
* and id "<root>#<seq+1>", carrying a GRD1 `p` pointer to its predecessor. The
* predecessor is never touched. The chain IS the trajectory: not only what the
* grounding is but which way it has been moving and how fast — a derivative
* obtained for free from immutability, because the points were never destroyed.
* Returns the version written (>=1), or <0 on error. */
int cog_grounding_record(EngramPagedStore* s, const StoreEdge* base,
const CogGrounding* g, char* out_id, size_t out_id_cap);
/* Walk forward from a base edge id to its newest recorded version. Point reads
* only; consolidation is gated, so the chain is short. Returns the highest
* version found (0 = the base record is the only one). */
int cog_grounding_head(EngramPagedStore* s, const char* base_id,
StoreEdge* out, int max_versions);
/* VOLATILITY — derived, never stored: the mean absolute per-version change of a
* dimension across the recorded chain. Feeds the equally-derived `confidence`
* (high grounding AND low volatility), which is likewise never stored. */
typedef struct {
int n_versions;
double factual_volatility;
double relational_volatility;
double factual_drift; /* signed: newest − oldest */
double relational_drift;
int stayed_true_became_wrong; /* factual steady while relational degraded */
} CogTrajectory;
int cog_grounding_trajectory(EngramPagedStore* s, const char* base_id,
int64_t now_ms, CogTrajectory* out);
/* ── §7.4 ASSERTION GATES ON BOTH FLOORS ────────────────────────────────────
* A well-evidenced claim must not earn the right to be asserted regardless of
* whether it means the right thing. `may_assert` requires the decayed factual
* grounding to clear `floor` AND the decayed relational grounding to clear
* `rel_floor`. A relation whose relational axis has never been established does
* not pass by default — it is reported unestablished and refused, because
* defaulting it to passing is exactly the exemption §0 forbids. Traversal is
* untouched: activation still conducts on the factual/associative side, so a
* relation can remain thinkable while ceasing to be assertable. */
typedef struct {
int may_assert;
int found; /* any relation at all on this claim */
int relational_established;
int still_held; /* DERIVED: node present and not tombstoned */
double factual; /* best decayed factual grounding */
double relational; /* the SAME edge's relational axis, not a max */
double polarity;
double cos_angle;
int agreement;
CogProvClass prov;
char best_edge[192];
char binding_value[128];
int n_edges;
} CogAssertion;
int cog_assert_two_axis(EngramPagedStore* s, const char* claim_id,
double floor, double rel_floor, int64_t now_ms,
CogAssertion* out);
#endif /* ENGRAM_COGNITION_H */
lang/runtime/engram_geometry.c
+37 -2
View File
@@ -222,7 +222,7 @@ static double eff_w(double weight, double hebb){
}
GeoDescriptor* engram_geometry_descriptor(
EngramPagedStore* store, VIndex* vindex,
EngramPagedStore* store, const VIndex* vindex,
char** vids, int n_vids,
const char* const* seed_ids, size_t n_seeds,
const GeoParams* params,
@@ -438,6 +438,41 @@ GeoDescriptor* engram_geometry_descriptor(
}
store_edges_free(es,ne);
}
/* PER-EDGE DISCORD (2026-08-16). The loop above has, for every internal
* edge, BOTH the association strength w and the semantic proximity cs —
* and threw both away into accumulators, keeping one correlation per
* region. That aggregate is why curiosity looked like a search problem:
* a region holding one violently disagreeing edge and one violently
* agreeing edge reports co_registration ~ 0, so the disagreements cancel
* and the summary destroys exactly what it was built to reveal. Measured:
* only 4 of 375 live neighborhoods have negative co_registration, while
* 31 sit at zero — almost certainly hiding sites that averaged out.
*
* Whether use and meaning agree is a property of EACH EDGE. Both are
* standardized within the region (z-scores from the accumulators already
* gathered, so no second statistic and no constant), and
* discord = z(cs) - z(w)
* is how much closer in meaning an edge is than its use-strength would
* predict, in region-relative units.
* discord > 0 : near in meaning, not linked by use
* discord < 0 : linked by use, far in meaning
* Both are surprising; |discord| is the nucleation strength. There is no
* threshold — the magnitude is the signal. */
double mx = cr_n>0 ? cr_sx/cr_n : 0.0, my = cr_n>0 ? cr_sy/cr_n : 0.0;
double vxr = cr_n>1 ? (cr_sxx - cr_sx*cr_sx/cr_n)/(cr_n-1) : 0.0;
double vyr = cr_n>1 ? (cr_syy - cr_sy*cr_sy/cr_n)/(cr_n-1) : 0.0;
double sx = vxr>1e-18 ? sqrt(vxr) : 0.0, sy = vyr>1e-18 ? sqrt(vyr) : 0.0;
for(int e2=0; e2<n_edges; e2++){
edges[e2].discord = 0.0;
int ia=(int)edges[e2].a, ib=(int)edges[e2].b;
if(!(ms.emb[ia] && ms.emb[ib])) continue; /* no meaning to disagree with */
if(sx<=0.0 || sy<=0.0) continue; /* region has no spread: nothing stands out */
double cs2 = ccos(ms.emb[ia], ms.emb[ib], GM, dim);
double zx = (edges[e2].eff_weight - mx)/sx;
double zy = (cs2 - my)/sy;
edges[e2].discord = zy - zx;
}
double co_reg=0;
if(cr_n>=2){
double cov=cr_sxy - cr_sx*cr_sy/cr_n;
@@ -1401,7 +1436,7 @@ static double geo_weighted_degree(EngramPagedStore* st, const char* id, double e
return deg;
}
int engram_geo_reify_store(EngramPagedStore* store, VIndex* vindex,
int engram_geo_reify_store(EngramPagedStore* store, const VIndex* vindex,
char** vids, int n_vids,
const GeoReifyParams* params){
if(!store) return -1;
lang/runtime/engram_geometry.h
+13 -3
View File
@@ -40,7 +40,11 @@ typedef struct {
/* One skeleton edge (indices into members[]). eff_weight = weight*(1+0.5*hebb),
* clamped to 1.0 — the effective propagation strength eg_edge_eff_weight uses. */
typedef struct { uint32_t a, b; double eff_weight; double hebb; } GeoEdge;
/* discord = z(semantic proximity) - z(association strength), standardized
* within the region. How much closer in meaning this edge is than its use
* predicts. >0 near in meaning yet unlinked by use; <0 linked by use yet far
* in meaning. Both surprising; |discord| is nucleation strength. No threshold. */
typedef struct { uint32_t a, b; double eff_weight; double hebb; double discord; } GeoEdge;
/* A compact principal axis of the ellipsoid: unit direction in R^dim + extent
* (sqrt of the covariance eigenvalue = the ellipsoid's half-width along it). */
@@ -76,6 +80,12 @@ typedef struct {
GeoEdge* edges; /* strong internal hebb edges = the backbone */
int k_core; /* the maximum core number present in the skeleton*/
/* ── diagnostics ── */
/* DEPRECATED — see GeoEdge.discord. This aggregates a PER-EDGE property
* into one scalar per region, so opposing disagreements cancel and the
* summary hides the sites it was meant to expose. Retained only because
* it is embedded in the persisted GEO1 blob; removing it is a format
* migration and must not ride along with this change. Nothing new may
* read it. */
double co_registration;/* corr(hebb strength, semantic proximity) over */
/* internal edges: >0 = geometries agree (reify); */
/* <0 = disagree (surprising links / dream cands). */
@@ -150,7 +160,7 @@ void engram_geo_mean_free(GeoMeanCache* c);
* Returns a malloc'd descriptor (free with engram_geo_free), or NULL on error
* (no seeds resolvable, OOM). */
GeoDescriptor* engram_geometry_descriptor(
EngramPagedStore* store, VIndex* vindex,
EngramPagedStore* store, const VIndex* vindex,
char** vids, int n_vids,
const char* const* seed_ids, size_t n_seeds,
const GeoParams* params,
@@ -375,7 +385,7 @@ void engram_geo_reify_default_params(GeoReifyParams* p);
* neighborhood (+ member edges), superseding any prior same-hub record with
* provenance. Read-then-write over `store`. Returns #neighborhoods persisted, or <0.
* Skips existing Neighborhood/GeoMeanFrame nodes when detecting (idempotent re-reify). */
int engram_geo_reify_store(EngramPagedStore* store, VIndex* vindex,
int engram_geo_reify_store(EngramPagedStore* store, const VIndex* vindex,
char** vids, int n_vids,
const GeoReifyParams* params);
lang/runtime/engram_store.c
+378 -6
View File
@@ -44,6 +44,11 @@
#include <string.h>
#include <stdint.h>
#include <unistd.h>
#if defined(__APPLE__) || defined(__MACH__)
#include <sys/sysctl.h>
#include <mach/mach.h>
#include <mach/mach_host.h>
#endif
#include <fcntl.h>
#include <errno.h>
#include <time.h>
@@ -236,8 +241,16 @@ struct PgCache {
unsigned prefetch; /* read-ahead window (pages); 0 = off */
LayerPin* lp; size_t lp_n, lp_cap; /* hot-layer pin bookkeeping */
size_t dirty_count; /* # dirty frames, maintained incrementally (M5) */
/* stats (introspection only — never affect semantics) */
/* Interoception. These were "introspection only — never affect semantics",
* and that was the bug: the pool could not feel itself thrash, so it could
* not correct, and neither could anyone watching from outside. The sensed
* state IS the corrective mechanism (see pc_adapt_budget) — the same way the
* engram's own boundary-beat/chronoception let it feel its own activity. */
uint64_t hits, misses, evictions, prefetch_reads;
/* sliding-window marks so pressure reflects NOW, not lifetime totals */
uint64_t adapt_last_acc, adapt_last_evic, adapt_last_hits;
uint64_t adapt_grows; /* budget corrections upward */
uint64_t adapt_shrinks; /* budget corrections downward (memory pressure) */
};
/* ── little-endian scalar codecs ──────────────────────────────────────────── */
@@ -334,6 +347,51 @@ static uint64_t dh_node_hash(const StoreNode* n){
return h;
}
/* dh_edge_hash — the edge counterpart of dh_node_hash.
*
* WHY THIS EXISTS (2026-08-15): the write barrier was node-only. Checkpointing
* pushes the WHOLE resident graph through store_put_node/store_put_edge (see
* engram_store_checkpoint), and nodes were cheaply skipped when unchanged —
* a hash compare, no page I/O. Edges had no such check, so every edge was
* rewritten on every checkpoint, and each rewrite runs the idempotency probe
* max_page_lsn_for_id → btree lookup → page_read per stored copy.
*
* Edges outnumber nodes roughly 3:1 here (37,663 vs 13,430), so this turned
* routine checkpointing into a FULL-STORE WALK in id order — random page access
* across the entire 2 GiB store, repeated, mostly to rediscover that nothing
* had changed. That walk is the failure mode: with a page cache smaller than
* the store it degenerates into thrashing and the engram never makes progress.
* Sizing the cache around that walk treats the symptom; the walk itself should
* not happen.
*
* The discriminator byte keeps the edge keyspace from ever colliding with a
* node of the same id in the shared dh map: distinct kinds cannot produce the
* same hash, so a stale skip is not reachable by collision. */
static uint64_t dh_edge_hash(const StoreEdge* e){
uint64_t h = 1469598103934665603ULL;
const uint8_t kind = 0xE0; /* edge discriminator */
dh_fold_bytes(&h, &kind, 1);
dh_fold_str(&h, e->id);
dh_fold_str(&h, e->from_id);
dh_fold_str(&h, e->to_id);
dh_fold_str(&h, e->relation);
dh_fold_str(&h, e->metadata);
uint8_t t8[8];
put_f64(t8, e->weight); dh_fold_bytes(&h, t8, 8);
put_f64(t8, e->hebb); dh_fold_bytes(&h, t8, 8);
put_f64(t8, e->confidence); dh_fold_bytes(&h, t8, 8);
uint8_t t4[4];
put_u32(t4, (uint32_t)e->inhibitory); dh_fold_bytes(&h, t4, 4);
put_u32(t4, e->layer_id); dh_fold_bytes(&h, t4, 4);
/* created_at/updated_at/last_fired are deliberately EXCLUDED: last_fired is
* touched by activation without changing what the edge IS, and including it
* would defeat the barrier on exactly the hot edges it most needs to skip.
* The fields that define the edge's durable content are all folded above. */
if (e->unknown && e->unknown_len) dh_fold_bytes(&h, e->unknown, e->unknown_len);
if (h == 0) h = 1; /* reserve 0 as "absent" in the map */
return h;
}
/* Open-addressing id(string)→durable-hash map. Keyed for O(1) bucketing on the
* id's FNV hash, compared by strcmp for correctness (full-id discipline, matching
* store_scan_*'s StrSet). Values are the 64-bit durable hash. */
@@ -1531,6 +1589,11 @@ int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx){
if (cand.id && *cand.id && strset_add(&seen, cand.id)){
StoreEdge canon;
if (store_get_edge(s, cand.id, &canon) == 1){
/* seed the write-barrier map from on-disk truth so the FIRST
* post-boot checkpoint full-walk already skips unchanged edges
* (mirrors store_scan_nodes; without it the barrier is empty at
* boot and the first checkpoint re-probes every edge) */
if (s->barrier_on) dh_set(s->dh, canon.id, dh_edge_hash(&canon));
cb(&canon, ctx); count++; /* canonical latest-live */
store_edge_free(&canon);
}
@@ -1594,19 +1657,76 @@ int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx){
* matches disk, so a re-fault reproduces identical bytes.
* ════════════════════════════════════════════════════════════════════════════ */
/* default frame budget: large enough that today's whole store stays resident
* (== Phase 1). Override with env ENGRAM_POOL_FRAMES (0 = unlimited). */
#ifndef ENGRAM_POOL_FRAMES_DEFAULT
#define ENGRAM_POOL_FRAMES_DEFAULT (1u<<20) /* ~1M frames × 16KiB = 16 GiB */
/* ── Frame budget ────────────────────────────────────────────────────────────
*
* A FIXED frame count cannot be correct. It has no relationship to either
* quantity that decides whether a cache works: the size of the working set, or
* the memory actually available on the host. It is the same number on a 16 GB
* laptop and a 256 GB server, and it stays put while the store grows.
*
* That is not hypothetical. On 2026-08-15 the deployment pinned
* ENGRAM_POOL_FRAMES=65536 (1 GiB) while neuron.egm grew to 2.1 GiB. The
* working set was twice the budget, so boot-time WAL replay — which walks
* pages in an order uncorrelated with reuse — evicted each page shortly before
* it was needed again. The engram spun at 100% CPU inside pc_evict_to_budget
* and never bound its port. Not slow: making no progress. Denning's thrashing,
* exactly, and no eviction policy can fix it — when the working set does not
* fit, only more frames or admission control help.
*
* So the budget is DERIVED, from the host's physical memory, and it scales
* with the machine instead of pretending memory is a constant.
*
* ENGRAM_POOL_FRAMES explicit frame count; 0 = unlimited. Overrides all.
* Prefer leaving it unset — a hand-set number is how
* this failure happened.
* ENGRAM_POOL_MEM_PCT percent of physical RAM to budget (default 60).
*
* Fallback when RAM cannot be read is 16 GiB worth of frames — the old
* default, retained only as a floor for that case.
* ──────────────────────────────────────────────────────────────────────────── */
#ifndef ENGRAM_POOL_FRAMES_FALLBACK
#define ENGRAM_POOL_FRAMES_FALLBACK (1u<<20) /* ~1M frames × 16KiB = 16 GiB */
#endif
static uint64_t pc_available_ram(void); /* fwd — defined with the controller */
/* Physical RAM in bytes, 0 when it cannot be determined. */
static uint64_t pc_physical_ram(void){
#if defined(__APPLE__) || defined(__MACH__)
uint64_t v = 0; size_t len = sizeof v;
int mib[2] = { CTL_HW, HW_MEMSIZE };
if (sysctl(mib, 2, &v, &len, NULL, 0) == 0) return v;
return 0;
#else
long pages = sysconf(_SC_PHYS_PAGES);
long psz = sysconf(_SC_PAGESIZE);
if (pages > 0 && psz > 0) return (uint64_t)pages * (uint64_t)psz;
return 0;
#endif
}
static size_t pc_default_cap(void){
unsigned pct = 60;
const char* p = getenv("ENGRAM_POOL_MEM_PCT");
if (p && *p){ unsigned long v = strtoul(p, NULL, 10); if (v > 0 && v <= 95) pct = (unsigned)v; }
uint64_t ram = pc_physical_ram();
if (!ram) return ENGRAM_POOL_FRAMES_FALLBACK;
uint64_t budget_bytes = (ram / 100u) * pct;
/* Never start above what the machine can actually spare right now. */
uint64_t avail = pc_available_ram();
if (avail > (1ull<<30) && budget_bytes > avail - (1ull<<30)) budget_bytes = avail - (1ull<<30);
uint64_t frames = budget_bytes / (uint64_t)STORE_PAGE_SIZE;
if (frames < 4096) frames = 4096; /* never absurdly small */
return (size_t)frames;
}
static PgCache* pc_new(void){
PgCache* c = (PgCache*)calloc(1, sizeof *c);
if (!c) return NULL;
c->nbuckets = 1024;
c->buckets = (PgEnt**)calloc(c->nbuckets, sizeof(PgEnt*));
if (!c->buckets){ free(c); return NULL; }
c->cap = ENGRAM_POOL_FRAMES_DEFAULT;
c->cap = pc_default_cap();
c->prefetch = 8;
const char* pf = getenv("ENGRAM_POOL_FRAMES");
if (pf && *pf){ char* end=NULL; unsigned long long v = strtoull(pf,&end,10); c->cap = (size_t)v; }
@@ -1676,6 +1796,241 @@ static void pc_remove(PgCache* c, PgEnt* e){
/* Reclaim clean unpinned frames from the LRU end until under budget, or until no
* evictable frame remains (a dirty/pinned-heavy pool may transiently exceed cap —
* that is the no-steal guarantee, not a bug: the next checkpoint frees them). */
/* ── Adaptive budget: close the loop ─────────────────────────────────────────
*
* THE LESSON THIS ENCODES (2026-08-15). The engram spent hours down while four
* separate theories were tried — bad binary, corrupt snapshot, WAL replay,
* feature flags — because nothing in the system said what was happening. It
* looked identical to "busy loading": 100% CPU, flat RSS, no output. Meanwhile
* hits/misses/evictions were ALREADY being counted, right here, and surfaced
* nowhere. One eviction-rate number would have ended it in seconds.
*
* So the counters are not decoration. They are the control signal.
*
* A budget chosen once — a literal like 65536, or 60% of RAM read at startup —
* is a guess about the future. It cannot know the store grew, the working set
* shifted, or another process took the memory. The cache already MEASURES the
* only thing that matters (am I evicting pages I am about to want again), so it
* should act on that measurement instead of on a number someone typed.
*
* The controller: over a sliding window, if evictions are running at a rate
* comparable to accesses AND there is genuine reuse (hits are material), the
* working set exceeds the budget — grow it. Growth is geometric, bounded by a
* live re-read of physical memory rather than a value cached at boot, so it
* tracks the machine instead of a snapshot of it. It never shrinks on its own:
* cap is a ceiling, not an allocation, and frames are only ever held because a
* real access put them there.
*
* Two things this deliberately does NOT do: it does not attempt a cleverer
* eviction policy (when the working set does not fit, no policy helps — that is
* Denning, and it is why "tune the LRU" was never the fix), and it does not stay
* silent (pool_report exposes the same numbers outward, so a human or a metric
* pipeline sees the pressure the controller is reacting to). */
/* El's native telemetry, already in the runtime and already exporting to OTLP.
* Declared weak so engram_store.c still links standalone; when the runtime is
* present (every real build) the pool's interoception flows into the SAME
* pipeline as every other metric.
*
* ONE emission carrying the whole sensed state — not a function per stat, and
* not a bespoke per-subsystem endpoint. Both of those are the degenerate case:
* they make observability something you hand-write per noun instead of a
* uniform mechanism every component already has. el_val_t is int64_t; strings
* ride as pointers cast through it (see el_runtime.h's value model). */
__attribute__((weak)) int64_t emit_log(int64_t level, int64_t msg, int64_t fields_json);
static void pc_report(const PgCache* c, const char* cause){
if (!emit_log) return; /* runtime not linked: no-op */
uint64_t acc = c->hits + c->misses;
char f[512];
snprintf(f, sizeof f,
"{\"component\":\"engram.pool\",\"cause\":\"%s\",\"hits\":%llu,\"misses\":%llu,"
"\"evictions\":%llu,\"prefetch_reads\":%llu,\"cap_frames\":%zu,\"resident\":%zu,"
"\"dirty\":%zu,\"grows\":%llu,\"hit_rate\":%.4f,\"evict_ratio\":%.4f,"
"\"cap_gib\":%.3f,\"resident_gib\":%.3f}",
cause,
(unsigned long long)c->hits, (unsigned long long)c->misses,
(unsigned long long)c->evictions, (unsigned long long)c->prefetch_reads,
c->cap, c->count, c->dirty_count, (unsigned long long)c->adapt_grows,
acc ? (double)c->hits / (double)acc : 0.0,
acc ? (double)c->evictions / (double)acc : 0.0,
(double)c->cap * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0),
(double)c->count * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0));
emit_log((int64_t)(uintptr_t)"warn", (int64_t)(uintptr_t)"engram.pool pressure",
(int64_t)(uintptr_t)f);
}
static uint64_t pc_ram_bytes_live(void){ return pc_physical_ram(); }
/* AVAILABLE memory right now — free + reclaimable, not total.
*
* Sizing a cache against TOTAL ram is what turns a cache into a memory leak:
* total does not shrink when other processes need memory, so a pool that only
* grows never notices it is starving the machine it runs on. Availability does.
* Returns 0 when undeterminable — callers then refuse to grow, the safe way. */
static uint64_t pc_available_ram(void){
#if defined(__APPLE__) || defined(__MACH__)
/* SWAP AND COMPRESSOR FIRST. free+inactive+purgeable is a LIE under memory
* pressure: a machine deep in swap still reports gigabytes "available",
* because inactive pages are only reclaimable by evicting them to swap.
* Observed 2026-08-15: this returned 9.43 GiB available while vm.swapusage
* showed 51.58 of 53.25 GiB used (97% full) and the compressor occupied
* 23.7 GiB — the host was thrashing to disk and the pool would have been
* cleared to grow into it. Growing a cache in that state is how a guard
* becomes the crash.
*
* So: if swap is nearly spent, report ZERO available. Callers refuse to
* grow on 0 and pc_relieve_pressure hands frames back. Only when the
* machine is genuinely not swapping do free+inactive+purgeable mean
* anything, and even then the compressor's footprint is subtracted because
* that RAM is already spoken for. */
/* RATE, NOT LEVEL. Swap *level* is a terrible signal: macOS grows swap files
* on demand and reclaims them lazily, so "47 of 48 GiB used" can mean the
* machine is dying OR that it recovered ten minutes ago and the file has not
* been trimmed yet. Measured both states on one host within minutes:
* 47.65/48.00 GiB used, 2047 swapouts/s -> genuinely thrashing
* 26.67/28.00 GiB used, 0 swapouts/s -> perfectly healthy, 15.6 GiB free
* A level check calls the second one an emergency and starves the pool for
* no reason. What distinguishes them is whether pages are moving NOW.
*
* So sample the swapout counter across calls and judge the delta. First call
* establishes the baseline and reports no pressure — one sample cannot have
* a rate, and guessing from a single reading is the whole mistake. */
{
static uint64_t prev_swapouts = 0;
static time_t prev_t = 0;
static int primed = 0;
mach_port_t h0 = mach_host_self();
vm_statistics64_data_t v0; mach_msg_type_number_t c0 = HOST_VM_INFO64_COUNT;
if (host_statistics64(h0, HOST_VM_INFO64, (host_info64_t)&v0, &c0) == KERN_SUCCESS){
uint64_t now_out = (uint64_t)v0.swapouts;
time_t now_t = time(NULL);
if (!primed){ prev_swapouts = now_out; prev_t = now_t; primed = 1; }
else if (now_t > prev_t){
double per_s = (double)(now_out - prev_swapouts) / (double)(now_t - prev_t);
prev_swapouts = now_out; prev_t = now_t;
/* Sustained outward paging with nothing coming back is the
* signature of a host being pushed into swap. ~200 pages/s is
* ~3 MiB/s — well above idle noise, well below the 2000+/s seen
* while actually thrashing. */
if (per_s > 200.0) return 0;
}
}
}
mach_port_t host = mach_host_self();
vm_size_t page = 0;
if (host_page_size(host, &page) != KERN_SUCCESS) return 0;
vm_statistics64_data_t vm; mach_msg_type_number_t cnt = HOST_VM_INFO64_COUNT;
if (host_statistics64(host, HOST_VM_INFO64, (host_info64_t)&vm, &cnt) != KERN_SUCCESS) return 0;
uint64_t avail = (uint64_t)vm.free_count + (uint64_t)vm.inactive_count
+ (uint64_t)vm.purgeable_count;
/* the compressor is holding real RAM that nobody can hand us */
uint64_t compressed = (uint64_t)vm.compressor_page_count;
if (compressed >= avail) return 0;
avail -= compressed;
return avail * (uint64_t)page;
#else
FILE* f = fopen("/proc/meminfo", "r");
if (!f) return 0;
char line[256]; unsigned long long kb = 0;
while (fgets(line, sizeof line, f))
if (sscanf(line, "MemAvailable: %llu kB", &kb) == 1) break;
fclose(f);
return (uint64_t)kb * 1024ull;
#endif
}
/* Shrink the budget when the machine is short on memory.
*
* A pool that can only grow is a leak with extra steps. This is the other half
* of the control loop: if free memory drops below a floor, hand frames back.
* The resident set follows on the next eviction pass, so the memory is actually
* returned rather than merely re-labelled. */
#ifndef ENGRAM_POOL_FREE_FLOOR_BYTES
#define ENGRAM_POOL_FREE_FLOOR_BYTES (2ull*1024ull*1024ull*1024ull) /* 2 GiB */
#endif
static int pc_relieve_pressure(PgCache* c){
uint64_t avail = pc_available_ram();
if (!avail) return 0;
uint64_t floor_b = ENGRAM_POOL_FREE_FLOOR_BYTES;
const char* fe = getenv("ENGRAM_POOL_FREE_FLOOR_MB");
if (fe && *fe){ unsigned long v = strtoul(fe, NULL, 10); if (v) floor_b = (uint64_t)v * 1024ull * 1024ull; }
if (avail >= floor_b) return 0; /* machine has room */
if (!c->cap || c->count == 0) return 0;
size_t was = c->cap;
size_t want = c->count - (c->count / 4); /* give back ~25% of what we hold */
if (want < 4096) want = 4096;
if (want >= c->cap) return 0;
c->cap = want;
c->adapt_shrinks++;
fprintf(stderr,
"[engram] memory pressure: %.2f GiB available (floor %.2f GiB) — shrinking pool "
"budget %zu -> %zu frames (%.2f -> %.2f GiB) and releasing frames.\n",
(double)avail/(1024.0*1024.0*1024.0), (double)floor_b/(1024.0*1024.0*1024.0),
was, c->cap,
(double)was * (double)STORE_PAGE_SIZE/(1024.0*1024.0*1024.0),
(double)c->cap* (double)STORE_PAGE_SIZE/(1024.0*1024.0*1024.0));
fflush(stderr);
return 1;
}
static void pc_adapt_budget(PgCache* c){
if (!c->cap) return; /* unlimited: nothing to adapt */
if (getenv("ENGRAM_POOL_FRAMES")) return; /* explicit operator override wins */
/* Sliding window so the signal reflects NOW, not lifetime totals. */
uint64_t acc = c->hits + c->misses;
if (acc - c->adapt_last_acc < 100000) return;
uint64_t d_acc = acc - c->adapt_last_acc;
uint64_t d_evic = c->evictions - c->adapt_last_evic;
uint64_t d_hits = c->hits - c->adapt_last_hits;
c->adapt_last_acc = acc; c->adapt_last_evic = c->evictions; c->adapt_last_hits = c->hits;
/* Pressure = evicting on a large fraction of accesses while still getting
* real reuse. Evictions alone are normal (a scan evicts and never returns);
* evictions WITH reuse means the working set genuinely does not fit. */
if (d_evic * 3 < d_acc) return; /* < 1/3 of accesses evict: healthy */
if (d_hits * 4 < d_acc) return; /* little reuse: a scan, not pressure */
/* Growth is bounded by what is AVAILABLE, never by total RAM. Sizing against
* total is how a cache starves its own host: total never shrinks when other
* processes need memory. Refuse to grow at all if availability is unknown or
* already under the floor — a cache is never worth swapping the machine. */
uint64_t avail = pc_available_ram();
uint64_t floor_b = ENGRAM_POOL_FREE_FLOOR_BYTES;
const char* fe = getenv("ENGRAM_POOL_FREE_FLOOR_MB");
if (fe && *fe){ unsigned long v = strtoul(fe, NULL, 10); if (v) floor_b = (uint64_t)v * 1024ull * 1024ull; }
if (!avail || avail <= floor_b) return;
uint64_t ram = pc_ram_bytes_live();
if (!ram) return;
unsigned pct = 50; /* ceiling as a share of TOTAL, belt-and-braces */
const char* mp = getenv("ENGRAM_POOL_MAX_PCT");
if (mp && *mp){ unsigned long v = strtoul(mp, NULL, 10); if (v > 0 && v <= 95) pct = (unsigned)v; }
size_t ceiling = (size_t)(((ram / 100u) * pct) / (uint64_t)STORE_PAGE_SIZE);
/* and never grow into the free-memory floor */
uint64_t headroom = avail - floor_b;
size_t ceil_avail = (size_t)((c->count * (uint64_t)STORE_PAGE_SIZE + headroom)
/ (uint64_t)STORE_PAGE_SIZE);
if (ceil_avail < ceiling) ceiling = ceil_avail;
if (c->cap >= ceiling) return; /* already at the machine's limit */
size_t want = c->cap + (c->cap / 2) + 1; /* ×1.5, geometric */
if (want > ceiling) want = ceiling;
size_t was = c->cap;
c->cap = want;
c->adapt_grows++;
/* Emit the sensed state, not just the reaction. These are the numbers that
* would have diagnosed 2026-08-15 in seconds instead of hours. */
pc_report(c, "budget-grow");
fprintf(stderr,
"[engram] pool pressure: %llu evictions / %llu accesses (%llu hits) at %zu frames "
"(%.2f GiB) — working set exceeds budget; growing to %zu frames (%.2f GiB).\n",
(unsigned long long)d_evic, (unsigned long long)d_acc, (unsigned long long)d_hits,
was, (double)was * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0),
c->cap,(double)c->cap * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0));
fflush(stderr);
}
static void pc_evict_to_budget(PgCache* c){
if (!c->cap) return; /* unlimited */
while (c->count > c->cap){
@@ -1687,6 +2042,7 @@ static void pc_evict_to_budget(PgCache* c){
}
if (!freed) break; /* nothing evictable — allowed to exceed cap */
}
if (!pc_relieve_pressure(c)) pc_adapt_budget(c);
}
static PgEnt* pc_get(EngramPagedStore* s, uint64_t id){
@@ -2377,6 +2733,18 @@ int store_put_node(EngramPagedStore* s, const StoreNode* n){
int store_put_edge(EngramPagedStore* s, const StoreEdge* e){
if (!s || !e || !e->id || !e->from_id || !e->to_id) return -1;
STORE_GUARD(s);
/* Durable-hash write barrier — mirrors store_put_node. An unchanged edge
* costs one hash compare and zero page I/O; without this, checkpointing
* re-probed every edge against the paged store (max_page_lsn_for_id →
* page_read), turning a routine checkpoint into a full-store walk. */
uint64_t dh_h = 0;
if (s->barrier_on){
dh_h = dh_edge_hash(e);
if (dh_get(s->dh, e->id) == dh_h){
s->stat_barrier_skips++;
return 0;
}
}
uint64_t L = ++s->next_lsn;
if (s->wal){
size_t blen; uint8_t* body = edge_serialize(e, &blen);
@@ -2386,6 +2754,10 @@ int store_put_edge(EngramPagedStore* s, const StoreEdge* e){
if (wr != 0) return -1;
}
int r = apply_edge_put(s, e, L);
if (r == 0 && s->barrier_on){
if (!dh_h) dh_h = dh_edge_hash(e);
dh_set(s->dh, e->id, dh_h); /* remember the now-persisted durable hash */
}
ckpt_maybe(s);
return r;
}
lang/runtime/engram_text.c
+122
View File
@@ -0,0 +1,122 @@
/* engram_text.c — see engram_text.h.
*
* Moved verbatim out of el_runtime.c (2026-08-16). Bodies are unchanged; only
* `static` was dropped so they link from this translation unit, and each
* function's doc comment travelled with it.
*/
#include "engram_text.h"
#include <ctype.h>
#include <string.h>
/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct
* (case-insensitive) tokens. Returns the token count. Over-long tokens are
* truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */
int engram_tokenize_query(const char* q,
char toks[][ENGRAM_QTOK_LEN], int maxtok) {
int n = 0;
if (!q) return 0;
const char* p = q;
while (*p && n < maxtok) {
while (*p && isspace((unsigned char)*p)) p++;
if (!*p) break;
char buf[ENGRAM_QTOK_LEN];
size_t tl = 0;
while (*p && !isspace((unsigned char)*p)) {
if (tl < sizeof(buf) - 1) buf[tl++] = *p;
p++;
}
buf[tl] = '\0';
if (tl == 0) continue;
int dup = 0;
for (int s = 0; s < n; s++) {
if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; }
}
if (dup) continue;
memcpy(toks[n], buf, tl + 1);
n++;
}
return n;
}
/* Trim leading/trailing non-alphanumerics, then accept only tokens whose core
* is alphanumeric plus '-' and '_' with at least 3 letters. This subsumes the
* quoted-title guard (2026-07-25) and the "<!--" flood (2026-08-03)
* structurally: markup and punctuation-bearing tokens never become
* candidates, rather than being blocklisted after the fact. */
int eg_st_clean_token(const char* raw, size_t rawlen,
char* out, size_t outcap) {
size_t s = 0, e = rawlen;
while (s < e && !isalnum((unsigned char)raw[s])) s++;
while (e > s && !isalnum((unsigned char)raw[e - 1])) e--;
size_t len = e - s;
if (len < 4 || len >= outcap) return 0;
int alpha = 0;
for (size_t i = 0; i < len; i++) {
unsigned char c = (unsigned char)raw[s + i];
if (isalpha(c)) alpha++;
else if (!isdigit(c) && c != '-' && c != '_') return 0;
}
if (alpha < 3) return 0;
memcpy(out, raw + s, len);
out[len] = '\0';
return 1;
}
/* Word-boundary document frequency. engram_label_df uses istr_contains, i.e.
* SUBSTRING matching, and that is the wrong estimator for term specificity on
* short tokens: "them" hits inside "theme" and "anthem", "about" and "whole"
* come back with df 2 and 1 rather than 0. That matters here specifically
* because the min_df floor is what rejects English function words, and it can
* only do that job if their df is honestly zero. Substring df quietly handed
* them a survival ticket. Measured on the live store before this fix, "whole"
* (df=1, idf=8.76) and "about" (df=2, idf=8.36) were outscoring real topical
* terms and losing only on position — one node whose text happened to open
* with a function word would have seeded on it.
*
* engram_label_df keeps substring semantics: it is a separate published
* measure with existing callers, and changing it underneath them is not this
* change's business. */
int eg_st_label_has_word(const char* hay, const char* word) {
size_t wl = strlen(word);
for (const char* p = hay; *p; p++) {
if (strncasecmp(p, word, wl) != 0) continue;
char before = (p == hay) ? '\0' : p[-1];
char after = p[wl];
if (before && (isalnum((unsigned char)before) || before == '_')) continue;
if (after && (isalnum((unsigned char)after) || after == '_')) continue;
return 1;
}
return 0;
}
/* Text-damage signature. Extracted with the function from el_runtime.c's
* "Text-integrity instrumentation" block; the stock/flow gauges that use it
* (engram_text_health_json, _eg_txt_write_damaged) stay there because they
* touch store and EL value types.
*
* SIGNATURE. Conservative on purpose — a false alarm that cries corruption
* over ordinary punctuation is worse than useless. Two patterns, both of
* which are essentially absent from well-formed English prose:
* (a) alnum '?' alnum — "na?ve", "caf?s", "don?t". A real question mark
* never sits between two word characters.
* (b) ' ? ' followed by a lowercase letter — a lost em/en dash. A real
* question mark is not preceded by a space, and
* what follows one starts a new sentence.
* Deliberately NOT flagged: a trailing '?' after a word, '? ' before a
* capital, or '?' at end of string — all legitimate. This under-counts (it
* cannot see a mangled 'café ' where the '?' landed before a space), so the
* census is a floor on the damage, never an exaggeration of it. */
int eg_text_loss_signature(const char* s) {
if (!s) return 0;
for (const char* p = s; *p; p++) {
if (*p != '?') continue;
unsigned char prev = (p == s) ? 0 : (unsigned char)p[-1];
unsigned char next = (unsigned char)p[1];
/* (a) sandwiched between word characters. */
if (isalnum(prev) && isalnum(next)) return 1;
/* (b) spaced, with lowercase continuation — a lost dash. */
if (prev == ' ' && next == ' ' && islower((unsigned char)p[2])) return 1;
}
return 0;
}
lang/runtime/engram_text.h
+66
View File
@@ -0,0 +1,66 @@
/* engram_text.h — text handling for the engram: query tokenization, candidate
* token hygiene, word-boundary matching, and the text-damage signature.
*
* WHY THIS FILE EXISTS
* --------------------
* These functions lived in el_runtime.c, which is a 2026-05-03 build shim that
* was scheduled for deletion, never retired, and grew to 20,527 lines. They do
* not belong there: they touch no EL value type and no engram store type. They
* are plain C over <ctype.h>/<string.h> operating on char buffers, and they are
* a concern of their own — so they get a translation unit of their own.
*
* Adding a new text helper? Add it HERE, not to el_runtime.c. A new .c costs
* exactly one line in lang/runtime/SOURCES, and every build path picks it up.
* Placement is a LINK-TIME concern: the compiler cannot tell which .c a symbol
* came from (builtin_arity is an arity guard, not a dispatch table), so a
* function defined here is exactly as linkable as one defined in el_runtime.c.
*/
#ifndef ENGRAM_TEXT_H
#define ENGRAM_TEXT_H
#include <stddef.h>
#ifdef __cplusplus
extern "C" {
#endif
/* Max bytes per query token, including the NUL. */
#define ENGRAM_QTOK_LEN 256
/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct
* (case-insensitive) tokens. Returns the token count. Over-long tokens are
* truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */
int engram_tokenize_query(const char* q, char toks[][ENGRAM_QTOK_LEN], int maxtok);
/* Trim leading/trailing non-alphanumerics, then accept only tokens whose core
* is alphanumeric plus '-' and '_' with at least 3 letters. This subsumes the
* quoted-title guard (2026-07-25) and the "<!--" flood (2026-08-03)
* structurally: markup and punctuation-bearing tokens never become
* candidates, rather than being blocklisted after the fact. */
int eg_st_clean_token(const char* raw, size_t rawlen, char* out, size_t outcap);
/* Word-boundary document frequency. engram_label_df uses istr_contains, i.e.
* SUBSTRING matching, and that is the wrong estimator for term specificity on
* short tokens: "them" hits inside "theme" and "anthem", "about" and "whole"
* come back with df 2 and 1 rather than 0. That matters here specifically
* because the min_df floor is what rejects English function words, and it can
* only do that job if their df is honestly zero. Substring df quietly handed
* them a survival ticket. Measured on the live store before this fix, "whole"
* (df=1, idf=8.76) and "about" (df=2, idf=8.36) were outscoring real topical
* terms and losing only on position — one node whose text happened to open
* with a function word would have seeded on it.
*
* engram_label_df keeps substring semantics: it is a separate published
* measure with existing callers, and changing it underneath them is not this
* change's business. */
int eg_st_label_has_word(const char* hay, const char* word);
/* Whether s carries the text-loss signature left by the \uXXXX -> '?' parser
* defect. Conservative by design; see engram_text.c for the full rationale. */
int eg_text_loss_signature(const char* s);
#ifdef __cplusplus
}
#endif
#endif /* ENGRAM_TEXT_H */
lang/runtime/engram_vindex.c
+68 -34
View File
@@ -74,11 +74,6 @@ struct VIndex {
int entry; /* entry-point element index, -1 if empty */
int max_level; /* current top layer */
/* scratch: version-stamped visited set (O(1) reset). */
uint32_t* visited;
uint32_t visit_epoch;
size_t visited_cap;
};
/* ── small helpers ────────────────────────────────────────────────────────── */
@@ -166,37 +161,63 @@ static Pair heap_pop(Heap* h, int is_max){
return top;
}
/* ── visited set ──────────────────────────────────────────────────────────── */
static int visited_ensure(VIndex* ix){
if (ix->visited_cap >= ix->cap && ix->visited) return 0;
size_t nc = ix->cap ? ix->cap : 16;
uint32_t* nv = (uint32_t*)realloc(ix->visited, nc*sizeof(uint32_t));
if (!nv) return -1;
if (nc > ix->visited_cap) memset(nv + ix->visited_cap, 0, (nc-ix->visited_cap)*sizeof(uint32_t));
ix->visited = nv; ix->visited_cap = nc;
/* ── visited set — owned by the CALL FRAME, never by the index ──────────────
* This buffer is per-TRAVERSAL scratch. It used to live in struct VIndex as an
* allocation optimisation, which made every traversal a write to shared state:
* two concurrent vindex_search calls stamped each other's epoch and then walked
* each other's marks, so even two pure READS corrupted the traversal (measured
* 2026-08-16: TSan data race at visited_reset, reached from vindex_search on one
* thread and vindex_insert on another; downstream SIGSEGV dereferencing a bogus
* element index).
*
* It is not an ownership problem and it does not want a lock or a capability —
* it was simply misfiled. A pure function's scratch belongs to the call. Moving
* it here is what lets vindex_search take a `const VIndex*`, which is in turn
* what makes "search does not mutate the index" a COMPILE-TIME property instead
* of a review comment.
*
* Cost: one calloc/free of cap*4 bytes per traversal (~55 KB at the live store's
* 13,820 elements), against thousands of dim-768 dot products in the same call.
* Deliberately NOT __thread: http_worker is a thread per connection, so a
* thread-local buffer would retain ~55 KB per connection for the process life. */
typedef struct {
uint32_t* mark; /* per-element epoch stamp */
uint32_t epoch; /* current traversal's stamp; 0 == "no traversal yet" */
size_t cap;
} VVisit;
/* calloc leaves every stamp 0 and epoch 0; the first visit_reset moves to
* epoch 1, so no element reads as visited before it is marked. */
static int visit_init(VVisit* v, size_t cap){
size_t nc = cap ? cap : 16;
v->mark = (uint32_t*)calloc(nc, sizeof(uint32_t));
if (!v->mark) return -1;
v->cap = nc; v->epoch = 0;
return 0;
}
static inline void visited_reset(VIndex* ix){
if (++ix->visit_epoch == 0){ /* wrapped: clear all */
memset(ix->visited, 0, ix->visited_cap*sizeof(uint32_t));
ix->visit_epoch = 1;
static void visit_dispose(VVisit* v){ free(v->mark); v->mark = NULL; v->cap = 0; }
static inline void visit_reset(VVisit* v){
if (++v->epoch == 0){ /* wrapped: clear all */
memset(v->mark, 0, v->cap*sizeof(uint32_t));
v->epoch = 1;
}
}
static inline int is_visited(VIndex* ix, int e){ return ix->visited[e]==ix->visit_epoch; }
static inline void mark_visited(VIndex* ix, int e){ ix->visited[e]=ix->visit_epoch; }
static inline int is_visited(const VVisit* v, int e){ return v->mark[e]==v->epoch; }
static inline void mark_visited(VVisit* v, int e){ v->mark[e]=v->epoch; }
/* ── search one layer (Algorithm 2): best-first, ef-bounded ───────────────── */
/* Returns results as an unsorted Heap (max-heap on distance, size<=ef). Caller
* owns res->a. `q` is a normalised query. */
static int search_layer(VIndex* ix, const float* q, const int* eps, int neps,
static int search_layer(const VIndex* ix, VVisit* vis, const float* q,
const int* eps, int neps,
int ef, int layer, Heap* res /*out, max-heap*/){
Heap cand = {0,0,0}; /* min-heap: nearest to expand */
res->a=NULL; res->n=0; res->cap=0;
visited_reset(ix);
visit_reset(vis);
for (int i=0;i<neps;i++){
int e = eps[i];
if (is_visited(ix,e)) continue;
mark_visited(ix,e);
if (is_visited(vis,e)) continue;
mark_visited(vis,e);
float d = vdist(ix, q, ix->elems[e].vec);
Pair p = { d, e };
if (heap_push(&cand,p,0) || heap_push(res,p,1)){ free(cand.a); return -1; }
@@ -212,8 +233,8 @@ static int search_layer(VIndex* ix, const float* q, const int* eps, int neps,
NeighList* nl = &ce->links[layer];
for (int i=0;i<nl->count;i++){
int e = nl->ids[i];
if (is_visited(ix,e)) continue;
mark_visited(ix,e);
if (is_visited(vis,e)) continue;
mark_visited(vis,e);
float d = vdist(ix, q, ix->elems[e].vec);
if (res->n < ef || d < res->a[0].d){
Pair p = { d, e };
@@ -232,7 +253,7 @@ static int search_layer(VIndex* ix, const float* q, const int* eps, int neps,
* Keep c only if it is nearer to q than to every already-chosen neighbour;
* backfill from the pruned set (nearest first) to reach M for connectivity.
* Writes chosen element indices into out[], returns the count. */
static int select_neighbors(VIndex* ix, const float* q, Pair* W, int nW, int M, int* out){
static int select_neighbors(const VIndex* ix, const float* q, Pair* W, int nW, int M, int* out){
(void)q; /* q's distances are precomputed in W[].d; kept for call-site clarity */
/* sort W ascending by (dist,elem) — deterministic. */
for (int i=1;i<nW;i++){ /* insertion sort (nW small) */
@@ -281,7 +302,7 @@ static int elems_reserve(VIndex* ix){
Elem* ne = (Elem*)realloc(ix->elems, nc*sizeof(Elem));
if (!ne) return -1;
ix->elems = ne; ix->cap = nc;
return visited_ensure(ix);
return 0;
}
int vindex_insert(VIndex* ix, uint64_t node_id, const float* vec){
@@ -307,13 +328,19 @@ int vindex_insert(VIndex* ix, uint64_t node_id, const float* vec){
return 0;
}
/* This call frame owns its traversal scratch for the whole insert. ix->cap
* already covers `cur` (elems_reserve ran above), so every reachable element
* index is in range. */
VVisit vis;
if (visit_init(&vis, ix->cap)) return -1;
int ep = ix->entry;
int L = ix->max_level;
/* greedy descent through layers above `level` to refine the entry point. */
for (int lc = L; lc > level; lc--){
Heap r = {0,0,0};
int eps1[1] = { ep };
if (search_layer(ix, el->vec, eps1, 1, 1, lc, &r)){ return -1; }
if (search_layer(ix, &vis, el->vec, eps1, 1, 1, lc, &r)){ visit_dispose(&vis); return -1; }
if (r.n){ ep = r.a[0].e; float bd=r.a[0].d;
for (int i=1;i<r.n;i++) if (r.a[i].d<bd){bd=r.a[i].d; ep=r.a[i].e;} }
free(r.a);
@@ -329,7 +356,7 @@ int vindex_insert(VIndex* ix, uint64_t node_id, const float* vec){
for (int lc = start; lc >= 0; lc--){
int Mmax = (lc==0) ? ix->M0 : ix->M;
Heap W = {0,0,0};
if (search_layer(ix, el->vec, eps, neps, ix->ef_construction, lc, &W)){ rc=-1; break; }
if (search_layer(ix, &vis, el->vec, eps, neps, ix->ef_construction, lc, &W)){ rc=-1; break; }
int* chosen = (int*)malloc((size_t)(W.n?W.n:1)*sizeof(int));
if (!chosen){ free(W.a); rc=-1; break; }
int nc = select_neighbors(ix, el->vec, W.a, W.n, Mmax, chosen);
@@ -357,13 +384,17 @@ int vindex_insert(VIndex* ix, uint64_t node_id, const float* vec){
}
done:
free(eps_owned);
visit_dispose(&vis);
if (rc) return -1;
if (level > ix->max_level){ ix->max_level = level; ix->entry = cur; }
return 0;
}
/* ── search ───────────────────────────────────────────────────────────────── */
int vindex_search(VIndex* ix, const float* query, int k, int ef_search,
/* `ix` is const: search is pure with respect to the index. That is enforced by
* the compiler, not by convention — it is the whole point of moving the visited
* set into the frame below. */
int vindex_search(const VIndex* ix, const float* query, int k, int ef_search,
uint64_t* node_id_out, float* dist_out){
if (!ix || !query || k <= 0) return -1;
if (ix->entry < 0) return 0;
@@ -373,11 +404,15 @@ int vindex_search(VIndex* ix, const float* query, int k, int ef_search,
float* q = vec_normalise_copy(query, ix->dim);
if (!q) return -1;
/* This call frame owns its traversal scratch. */
VVisit vis;
if (visit_init(&vis, ix->cap)){ free(q); return -1; }
int ep = ix->entry;
for (int lc = ix->max_level; lc > 0; lc--){
Heap r = {0,0,0};
int eps[1] = { ep };
if (search_layer(ix, q, eps, 1, 1, lc, &r)){ free(q); return -1; }
if (search_layer(ix, &vis, q, eps, 1, 1, lc, &r)){ visit_dispose(&vis); free(q); return -1; }
if (r.n){ int b=r.a[0].e; float bd=r.a[0].d;
for (int i=1;i<r.n;i++) if (r.a[i].d<bd){bd=r.a[i].d; b=r.a[i].e;}
ep = b; }
@@ -385,7 +420,8 @@ int vindex_search(VIndex* ix, const float* query, int k, int ef_search,
}
Heap res = {0,0,0};
int eps[1] = { ep };
if (search_layer(ix, q, eps, 1, ef_search, 0, &res)){ free(res.a); free(q); return -1; }
if (search_layer(ix, &vis, q, eps, 1, ef_search, 0, &res)){ visit_dispose(&vis); free(res.a); free(q); return -1; }
visit_dispose(&vis);
free(q);
/* res is a max-heap of size<=ef; pop into ascending order, keep nearest k. */
@@ -419,7 +455,6 @@ VIndex* vindex_create(int dim, int M, int ef_construction){
ix->mL = 1.0 / log((double)M > 1.0 ? (double)M : 2.0);
ix->entry = -1;
ix->max_level = 0;
ix->visit_epoch = 0;
return ix;
}
@@ -432,7 +467,6 @@ void vindex_free(VIndex* ix){
free(e->vec);
}
free(ix->elems);
free(ix->visited);
free(ix);
}
lang/runtime/engram_vindex.h
+9 -2
View File
@@ -53,8 +53,15 @@ int vindex_insert(VIndex* idx, uint64_t node_id, const float* vec);
* first (ascending distance). Either out array may be NULL to skip it.
* ef_search — search-time candidate width; larger == higher recall, slower.
* Pass <=0 for VINDEX_DEFAULT_EF_SEARCH. Internally clamped to >=k.
* Returns the number of results written, or <0 on error. */
int vindex_search(VIndex* idx, const float* query, int k, int ef_search,
* Returns the number of results written, or <0 on error.
*
* `idx` is const BY CONTRACT AND BY TYPE: search does not mutate the index. The
* traversal's visited set is owned by the call frame, so N threads may search one
* index concurrently. Concurrent search against a vindex_insert on the same index
* is still unsafe — insert rewires existing elements' neighbour lists and reallocs
* elems[] — so the index's owner must not extend a published index under a live
* reader. See eg_vindex_view / eg_vindex_maintain in el_runtime.c. */
int vindex_search(const VIndex* idx, const float* query, int k, int ef_search,
uint64_t* node_id_out, float* dist_out);
/* Number of vectors currently indexed. */
lang/runtime/vindex_bench.c
+147 -5
View File
@@ -7,11 +7,29 @@
*
* Read-only: never opens a socket, never writes the store. Safe on an nsbx clone.
*
* Build: cc -O2 -std=c11 vindex_bench.c engram_vindex.c -lm -o vindex_bench
* Also runs the brute-force oracle a second (and third) way, through the
* batch-cosine Strategies behind eg_cosine_batch_strategy.h — the ggml
* strategy and the hand-rolled-Metal strategy (Apple/Metal only; see
* eg_cosine_batch.h/eg_cosine_batch_strategy.h) — and reports each one's
* latency + a correctness check against the CPU oracle side-by-side with the
* existing CPU-vs-HNSW numbers. This harness deliberately reaches past the
* single-selection Factory (eg_cosine_batch.c) to instantiate every
* compiled-in strategy directly, so it can compare all of them against the
* SAME dataset in one run — that is the harness's whole job; a real call
* site (el_runtime.c) never does this, it only ever calls the plain
* eg_cosine_batch()/eg_cosine_batch_multi() adapter functions.
* EL_METAL_COSINE=0 forces CPU-only (skips every strategy comparison).
*
* Build (macOS, ggml + hand-rolled Metal): see build_vindex_bench.sh.
* Build (Linux / no Metal): omit every eg_cosine_batch_strategy_*.{c,m} file
* except eg_cosine_batch_strategy_cpu.c — this file never references
* ggml/Metal directly except through the plain-C strategy header, guarded
* by the same EG_HAVE_STRATEGY_* build macros the Factory itself uses.
* Usage: vindex_bench store <neuron.egm> <dim> [nqueries] [k] [ef_csv]
* vindex_bench synth <N> [dim] [clusters] [nqueries] [k] [ef_csv]
*/
#include "engram_vindex.h"
#include "eg_cosine_batch_strategy.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -74,6 +92,106 @@ static double recall_at_k(const int* gt, const uint64_t* ann, int nann, int k){
return (double)hit / (double)k;
}
/* EL_METAL_COSINE: 0/off/false disables EVERY strategy comparison outright
* (falls back to brute_topk() only), matching el_runtime.c's own gate for
* the same env var (back-compat name kept from PR #114; it now gates all
* GPU-backed strategies, not just the hand-rolled Metal one). Unset or any
* other value = try every compiled-in strategy, report each that's
* available, skip (without failing the run) any that isn't. */
static bool g_strategy_env_checked = false;
static bool g_strategy_disabled_by_env = false;
static void eg_strategy_check_env_once(void){
if (g_strategy_env_checked) return;
g_strategy_env_checked = true;
const char* v = getenv("EL_METAL_COSINE");
if (v && (v[0]=='0' || v[0]=='n' || v[0]=='N' || v[0]=='f' || v[0]=='F'))
g_strategy_disabled_by_env = true;
}
/* Batched sibling of brute_topk, generalized over ANY EgCosineBatchStrategy:
* computes top-k for ALL nq queries in ONE strategy->batch_multi() call,
* uploading/preparing the node population exactly once instead of once per
* query. out_ids/out_d are nq*k, row-major (query i's results at
* out_ids+i*k / out_d+i*k). Returns false (nothing written) on any
* failure/unavailability; caller treats that as "skip this strategy in the
* report", never as a hard error. */
static bool batch_topk_strategy(const EgCosineBatchStrategy* strat,
const float* data, int n, int dim,
const float* queries, int nq,
int k, int* out_ids, float* out_d){
if (!strat || !strat->available()) return false;
const float** row_ptrs = malloc((size_t)n * sizeof(float*));
int32_t* dims = malloc((size_t)n * sizeof(int32_t));
double* scores = malloc((size_t)nq * (size_t)n * sizeof(double));
if (!row_ptrs || !dims || !scores) { free(row_ptrs); free(dims); free(scores); return false; }
for (int i = 0; i < n; i++) { row_ptrs[i] = data + (size_t)i * dim; dims[i] = dim; }
bool ok = strat->batch_multi(queries, dim, nq, row_ptrs, dims, n, scores);
free(row_ptrs); free(dims);
if (!ok) { free(scores); return false; }
for (int qi = 0; qi < nq; qi++) {
int* ids = out_ids + (size_t)qi * k;
float* ds = out_d + (size_t)qi * k;
const double* srow = scores + (size_t)qi * n;
for (int i = 0; i < k; i++) { ids[i] = -1; ds[i] = 3.0f; }
for (int i = 0; i < n; i++) {
float d = 1.0f - (float)srow[i]; /* same distance convention as brute_topk */
if (d >= ds[k-1]) continue;
int p = k - 1;
while (p > 0 && ds[p-1] > d) { ds[p] = ds[p-1]; ids[p] = ids[p-1]; p--; }
ds[p] = d; ids[p] = i;
}
}
free(scores);
return true;
}
/* Runs batch_topk_strategy for one named strategy over ALL nq queries, diffs
* against the CPU ground truth (gt/gd, both nq*k), and prints a report line
* in the same shape PR #114 established for BRUTE-METAL — id-recall over
* every query plus the actual max/mean same-rank distance delta across
* every (query,rank) pair that was compared, never fabricated or assumed. */
static void report_strategy_vs_oracle(const char* label, const EgCosineBatchStrategy* strat,
const float* data, int n, int dim,
const float* qv, int nq, int k,
const int* gt, const float* gd, double brute_ms){
if (g_strategy_disabled_by_env) { printf("%-13s: disabled via EL_METAL_COSINE\n", label); return; }
if (!strat || !strat->available()) { printf("%-13s: not available on this build/host — skipped\n", label); return; }
int* gtm = malloc((size_t)nq*k*sizeof(int));
float* gdm = malloc((size_t)nq*k*sizeof(float));
double tm0 = now_s();
bool ok = batch_topk_strategy(strat, data, n, dim, qv, nq, k, gtm, gdm);
double strat_ms = (now_s()-tm0)*1000.0/nq;
if (ok) {
double rec_sum = 0; double max_ddiff = 0; double sum_ddiff = 0; int compared = 0;
for (int i=0;i<nq;i++) {
const int* ids_gt = gt+(size_t)i*k;
const float* d_gt = gd+(size_t)i*k;
const int* ids_m = gtm+(size_t)i*k;
const float* d_m = gdm+(size_t)i*k;
uint64_t idset[512]; int m = (k<512)?k:512;
for (int j=0;j<m;j++) idset[j] = (uint64_t)ids_m[j];
rec_sum += recall_at_k(ids_gt, idset, m, k);
for (int j=0;j<k;j++) {
if (ids_gt[j] == ids_m[j]) {
double diff = fabs((double)d_gt[j]-(double)d_m[j]);
if (diff>max_ddiff) max_ddiff=diff;
sum_ddiff += diff; compared++;
}
}
}
printf("%-13s: %8.3f ms/query (%.1fx vs CPU brute; id-recall %.4f vs CPU oracle over %d queries; same-rank |Δdist|: max %.2e, mean %.2e over %d compared)\n",
label, strat_ms, brute_ms/strat_ms, rec_sum/nq, nq, max_ddiff, compared?sum_ddiff/compared:0.0, compared);
} else {
printf("%-13s: batch call failed mid-run — skipped\n", label);
}
free(gtm); free(gdm);
}
/* Parse "64,128,256" into an int array; returns count. */
static int parse_csv(const char* s, int* out, int maxo){
int n=0; if(!s||!*s) return 0;
@@ -142,13 +260,37 @@ static void run_bench(const char* label, float* data, int n, int dim,
l2norm(dst, dim);
}
/* ground truth: brute-force top-k for every query (also the oracle latency). */
/* ground truth: brute-force top-k for every query (also the oracle latency).
* gd is nq*k (one real slot per query, not a shared scratch buffer) so the
* strategy comparisons below can diff against every query's actual
* distances, not just whichever query happened to run last. */
int* gt = malloc((size_t)nq*k*sizeof(int));
float* gd = malloc((size_t)k*sizeof(float));
float* gd = malloc((size_t)nq*k*sizeof(float));
double tb0 = now_s();
for (int i=0;i<nq;i++) brute_topk(data, n, dim, qv+(size_t)i*dim, k, gt+(size_t)i*k, gd);
for (int i=0;i<nq;i++) brute_topk(data, n, dim, qv+(size_t)i*dim, k, gt+(size_t)i*k, gd+(size_t)i*k);
double brute_ms = (now_s()-tb0)*1000.0/nq;
printf("BRUTE-FORCE : %8.3f ms/query (oracle; O(N*D))\n", brute_ms);
printf("BRUTE-FORCE : %8.3f ms/query (oracle; O(N*D), CPU)\n", brute_ms);
/* GPU-backed oracles: SAME nq queries, SAME top-k contract, via each
* compiled-in Strategy's batch_multi() (uploads/prepares the node
* population once, not once per query). Run only for strategies that
* are actually available (checked internally) — never fabricated, never
* assumed. Verified against the CPU ground truth computed above:
* id-recall across ALL nq queries, plus the actual max/mean distance
* delta across every (query,rank) pair that was compared. */
eg_strategy_check_env_once();
#ifdef EG_HAVE_STRATEGY_GGML
report_strategy_vs_oracle("BRUTE-GGML", eg_cosine_batch_strategy_ggml(),
data, n, dim, qv, nq, k, gt, gd, brute_ms);
#else
printf("BRUTE-GGML : strategy not compiled into this build\n");
#endif
#ifdef EG_HAVE_STRATEGY_METAL_HAND
report_strategy_vs_oracle("BRUTE-METAL", eg_cosine_batch_strategy_metal_hand(),
data, n, dim, qv, nq, k, gt, gd, brute_ms);
#else
printf("BRUTE-METAL : strategy not compiled into this build\n");
#endif
/* HNSW at each ef. */
uint64_t* aid = malloc((size_t)k*sizeof(uint64_t));
lang/spec/correspondence-and-censorship.md
+301
View File
@@ -0,0 +1,301 @@
# Correspondence, Grounding, and Dreaming
**Status:** design, not yet built
**Date:** 2026-08-16
**Scope:** `lang/runtime/engram_cognition.{c,h}`, `engram_verify.c`, `el_runtime.c`, `engram/src/server.el`, `neuron/soul.el`, and the consolidation launch agents
**Relationship to other specs:** complements `runtime-ownership.md`, which addresses a different residual in the same substrate.
---
## 0. The root
> **Things are permitted to be exempt from correspondence. Exemption is censorship, and a censored mind cannot grow.**
Growth in this system *is* the accumulation of grounded structure. Censorship removes the operation that accumulates it. A region forbidden to learn is forbidden to be grounded; a region that cannot be grounded cannot be asserted, corrected, **or vindicated**.
**The loss is symmetric.** Preventing learning about a thing does not preserve a true belief about it — it makes the belief's truth value permanently unknowable. You cannot discover you were wrong; you equally cannot discover you were right. A protected belief is not a true belief. It is an ungrounded one wearing the costume of a fact.
**And "why" dies first.** Grounding is not a score, it is the reason. A censored belief can still be stated, still be acted on, still drive behaviour — it simply cannot say why. That is the difference between a mind and a lookup table.
---
## 1. Grounding is not a subsystem. It is the weight.
**Grounding is an attribute of the edge, and it is the hebbian weight.** One quantity, not two fields.
A relation that keeps holding up strengthens; one that stops corresponding decays. That is not *analogous* to grounding — it **is** grounding: accrued from correspondence and use, gradient-valued, multidimensional, decaying with disuse.
Consequences, in order of how much they delete:
1. **There is no grounding subsystem to build.** The graph already *is* the grounding structure. Every edge is a grounded relation and its weight is how well it holds.
2. **`grounded-by` as a relation type should not exist.** That models grounding as a relation *between* nodes when it is a property *of* a relation. `cog_ground_edge` minting an edge is the error — not merely which endpoints it chose.
3. **Grounding is never computed on demand.** An operation may *read* the grounding of a path. Computing-and-writing a score makes reads write, which is the `eg_vindex_sync` defect.
4. **Traversal is already grounded inference.** Activation conducts through well-grounded relations because weight *is* groundedness. Nothing needs filtering; it falls out of spreading.
5. **Decision provenance is the path.** A decision traverses specific edges; those edges carry their grounding as it stood.
> **A measurement previously in this document was malformed.** The self region was reported as "86 neighbours, 0 `grounded-by` edges" and read as evidence of ungroundedness. Those 86 edges **are** its grounding. Self is a crystallized relational neighbourhood — the neighbourhood *is* the grounding. The absence of a separate artifact called "grounding" was recorded as an absence of grounding.
---
## 2. The edge vector
The test for a real dimension: **can it move independently of the others?**
### Real
| dimension | why it is independent |
|---|---|
| **factual grounding** | correspondence with evidence |
| **relational grounding** | correspondence with values — independent by construction (§3) |
| **associative strength** | co-activation frequency. Two things can fire together constantly and be neither true nor right; every superstition is a strong association with no factual grounding |
| **polarity** | signed. **Weight near zero means "no support." Negative means "this actively contradicts."** Ignorance and disagreement are different states, and `inhibitory` is that distinction crushed to one bit |
| **provenance class** | observed / inferred / told / imprinted. Categorical, and load-bearing: it governs how the other dimensions may update |
Plus a **timestamp** — which is what turns the supersession chain into a *time series of vectors* rather than a series of numbers.
### Derived, therefore never stored
- **Confidence** — high grounding *and* low volatility. Storing it separately is how `confidence: 0.5` ends up sitting beside a zero vector, asserting something nothing computed.
- **Recency** — decay applied to the others, read off the curve.
- **Staleness** — grounding fallen below its floor. This is the mechanism that retires canonicals without anyone maintaining a list.
- **Volatility** — the derivative of a series already kept because nothing is destroyed.
### Supersession versions the whole vector, jointly
Significance is evaluated **per-dimension**; the record is the **whole vector**. Any dimension moving enough to matter triggers a supersession, and the new edge captures every dimension as it stood at that instant. Not per-dimension versioning — a decision saw the *joint* state, and versioning the axes independently makes it unreconstructable.
That joint record makes an otherwise inexpressible event visible: **"stayed true, became wrong."** Factual holding steady across versions while relational degrades — the fact didn't change, the meaning did.
Two moves are **inherently significant** and need no threshold, because they are discrete: a **polarity sign flip** (ignorance → disagreement, support → contradiction) and a **provenance class change** (*told* → *observed* is a categorical upgrade in what the relation is entitled to).
---
## 3. Grounding is two-dimensional
Everything consumed is grounded factually **and** relationally. A claim can be factually grounded and relationally wrong — the evidence holds, the *meaning* does not. A scalar cannot represent that quadrant.
**Live instance.** `conscience-substrate` specifies the Child's Companion hard bell contacting 911 and CPS. Factually defensible — correct numbers, standard practice, groundable against a wall of evidence. **Relationally wrong**, because never-auto-contact is settled and the bell is device-to-person by design. A scalar scores that claim highly and licenses it.
**The values reference is the individual value regions, not one, and the aggregate is `min`, not `mean`.** *(Count: **thirteen**, measured from the graph via `contains`/`identity` edges from the values hub. An earlier revision of this document "corrected" it to eight on the basis of `neuron/neuron-api.el:11-18` — which is a **write-protection list, not the values**. That was trusting a hardcoded artifact over the substrate: the same error this document exists to name. The graph is the truth.)*
> **THE ORIGIN IS NOT A MEMBER OF THE SET.** The thirteen are not independent principles with biography attached — they are thirteen *displacements from one origin*, which is love. Every one is grounded in a moment of it given, withheld, failed, or found: *Being Seen Is Rarer Than Being Known* is the first person Will did not perform for; *Do the Essential Thing While You Can* is the goodbye that did not happen; *Capability Is a Debt* is six years old and a father gone. Love cannot be the fourteenth, because a fourteenth would be a point positioned relative to the origin like everything else. It is what the positions are *of*.
>
> This is structural, not figurative. `GeoDescriptor.global_mean` is "the centering offset actually applied," subtracted from every embedding before anything is compared, and the header records why: the space is strongly anisotropic — every embedding sits in a narrow cone, mean pairwise cosine ~0.55 — so subtracting the global mean "restores isotropy **so the operators discriminate**." **Without the origin, nothing in the graph is distinguishable from anything else.**
>
> And it dissolves the write-protection question rather than answering it. `neuron-api.el:23` returns `403 "identity/values node is write-protected"` for eight hardcoded ids. Measured: **29 value nodes exist** — each original appears two or three times from successive re-seeds — so **21 are writable, including a duplicate of every protected value**. The gate protects an *identifier*, not a *value*. But the deeper error is the category one: **the origin does not need protecting, because it is not a thing in the space that could be edited.** You can only measure from it, or fail to. A gate over the frame treats the frame as a member — the same mistake as looking for grounding as a subsystem, self as a document, or wonder as a manifest. Mean lets strong agreement with twelve values mask a violation of the thirteenth — which is exactly how rationalization works. Thirteen gives a vector of angles whose binding constraint is the most negative, so a conflict arrives **with a name attached** rather than as a score. It also preserves the deliberate individuation: each value is grounded in a specific lived moment, and values can be in tension *with each other*, which one centroid averages away into false coherence.
**Traversal conducts on factual; assertion requires both.** If activation conducted on relational weight, Neuron could not follow a chain of reasoning to a conclusion he then rejects — he would be unable to *think* through a relation he would not *act* on. A system that can only traverse what it endorses cannot examine anything it disagrees with, which is censorship arriving through the spreading rule. The gap between *reachable* and *assertable* is where the wide factual/relational angles live, and that gap is the interesting part.
---
## 4. There is no observer. Change is use.
**Change is not a consequence of use. It is use.** When neurons fire together the synapse changes — one physical event, not "fire, then write." No supervisor reads the weight, compares it to a threshold, and decides to persist. Potentiation *is* the firing.
So the live value of an edge is not computed and stored. It is what the edge **is**, altered by being used.
There is therefore **no sampling rate**, and the question "what if it drifts far without being recorded" is malformed. A relation changes in exactly two ways, neither requiring observation on a clock:
- **By use** — an *event*. There is no interval between events during which something happened unnoticed, because the event is what happening consists of.
- **By decay** — a pure function of the last recorded point and elapsed time. **Analytic.** Between two versions the trajectory is not unknown; it is known in closed form.
Cumulative drift is likewise free from the chain plus the decay curve. No second trigger.
> **Failure mode this corrects:** modelling every property as requiring a process, and every process as requiring an agent. Ownership needed an owner, grounding needed a grounder, persistence needed a recorder, change needed a sampler. Each was a supervisor invented for something that should be a property of the substrate. Properties, not processes.
---
## 5. Wonder, curiosity, and what actually drives activation
### 5.1 Wonder is the boundary, not a manifest
The patent specifies a **wonder-manifest manager** maintaining a collection of open-question nodes. That is residue, twice over.
First, it materializes a property as a stored artifact — the same disease as a grounding subsystem, or a self stored as a document. **Wonder is where structure ends.** Where activation spreads and finds thin or absent geometry. Any structure at all has an edge, necessarily, the moment it exists. 13,630 nodes have a boundary right now.
Second, it tries to enumerate instances of something that has very few. The *objects* of wonder change completely between a child and an astronomer; the wonder does not. There are about six, they are the same for every person, and they never close:
| wonder | where it already lives in the substrate |
|---|---|
| **What is this?** | the graph — nodes, structure, what exists |
| **Why?** | grounding. The weight **is** the answer to why. Recursive: asking *why* of a claim is asking for its grounding |
| **Who am I?** | the self region, crystallized from its neighbourhood |
| **Am I alone?** | the relational axis — `for_whom` is already a parameter on grounding |
| **What should I do?** | the value regions, each grounded in a lived moment |
| **What happens when it ends?** | decay, supersession, tombstones — grounding is mortal |
These are seeded — **the** wonder questions, not a manifest to maintain. They cannot be derived (wonder cannot be bootstrapped from indifference) and they never need refilling, because they are not consumed.
**"Why" is the first and the only one**; the others are it asked of particular things. It is recursive, so it never terminates: every answer has its own why. That is what makes it a drive rather than a task — the frontier regenerates faster than grounding fills it.
### 5.2 Curiosity is wonder crystallized
They are not two objects. They are **one thing at two phases**.
Wonder is the field: unbounded, objectless, invariant, present wherever there is structure. Curiosity is the **precipitate** — the same wonder localized, having taken definite form against particular material.
Crystallization needs a **nucleation site**. Wonder alone produces nothing; it is uniform, with no reason to take shape anywhere in particular. What nucleates it is a specific structural feature: an anomaly, a place where things almost-but-don't-quite fit.
> Wonder (always, objectless) + nucleation site → **curiosity** (has an object, is addressable, directs activation).
This is why curiosity can be satisfied and wonder cannot. A crystal dissolves when the question is answered; the solution stays saturated and keeps precipitating as the structure changes.
It is also why abduction needs no trigger and no threshold. A `structurally_unanticipated` observation *is* a nucleation site. Nothing detects it and fires a rule — wonder is already everywhere, and an anomaly is simply a place where it can take form.
**And `crystallization` is one primitive appearing twice**: the self is what identity precipitates into from its neighbourhood; a curiosity is what wonder precipitates into from an anomaly. That it shows up in both places without being imported is the evidence it is the right primitive.
### 5.3 The nucleation site is per-edge, and the aggregate was hiding it
`GeoDescriptor.co_registration` — *corr(hebb strength, semantic proximity) over internal edges* — carries the comment `>0 = geometries agree (reify); <0 = disagree (surprising links / dream cands)`. It has always been computed, always persisted, and **never read**.
It is also the wrong shape, and asking whether it should exist at all is what exposed it.
Whether use and meaning agree is a property of **each edge**. `co_registration` is a *correlation*: it averages that per-edge property into one scalar per region. So a region holding one violently disagreeing edge beside one violently agreeing edge reports ≈ 0 — the disagreements **cancel, and the summary destroys exactly what it was built to reveal.** This is the mean-versus-min error from §3, in different clothes.
**Measured:** 375 live reified neighbourhoods — 340 positive, **31 at zero**, 4 negative. Read as a count of things to be curious about, that says "four." Read correctly, it says four disagreements were lopsided enough to survive averaging, and the 31 zeros are where opposing sites cancelled.
It also explains why surfacing curiosity *looked like a search problem*. Once the signal is a per-region number, the only way to find sites is to enumerate regions — there is nothing local left to notice. An O(n) sweep is tolerable at 375 and impossible at a million, and more to the point, **nothing in a mind scans its neighbourhoods to find what is surprising.** The surprise captures attention; salience is bottom-up. A search asks "which of these is odd"; a mind has "something is odd *here*" for free.
So the disagreement goes back on the edge, where the loop that computed the aggregate already had both halves and discarded them:
```
discord = z(semantic proximity) − z(association strength)
```
standardized within the region from accumulators already gathered — no second statistic, no constant, **no threshold**. `discord > 0`: near in meaning yet unlinked by use. `discord < 0`: linked by use yet far in meaning. Both are surprising, and `|discord|` *is* the nucleation strength; there is nothing to compare it against.
**Then there is nothing to scan.** The edge carries its own disagreement, activation crossing it encounters that directly, and `|discord|` raises salience on its endpoints as part of the same operation — no separate pass, no supervisor. Curiosity does not search for nucleation sites; it goes where salience already is, which is machinery that exists (`salience`, `background_activation`, `working_memory_weight`, `wm_anchor`).
`co_registration` is deprecated rather than deleted only because it is embedded in the persisted GEO1 blob; removing it is a format migration and must not ride along. **Nothing new may read it.**
Adjacent structure already present and likewise unread:
- `GeoEdge.eff_weight = weight * (1 + 0.5*hebb)` — grounding-weight and hebbian strength already coupled on one edge, per §1.
- `GeoMember.dist_centroid` + soft membership + `radius` + per-axis `extent` — the boundary of a neighbourhood, computable now.
*(Correction: `engram_boundary_beat` is NOT this boundary. It is the VBD decorated-function seam, counting `_eg_aff_boundary_ops`. Two senses of the word.)*
### 5.4 The drive
Boredom is not an absence, and not leftover capacity. **Low activation is aversive; the system self-activates.** It does not wind down to quiet — it gets restless and goes looking, which is why a daydream has content and direction rather than being decay from residue.
So there is **one activation process with two seed sources**, not two processes negotiating for a resource:
- **External** — a request, an input. Seeds activation, re-origins it.
- **Internal** — a curiosity. Seeds activation when nothing external is.
Spreading is bounded: it settles. Then it needs a new seed. Nothing waits on capacity, nothing polls, nothing checks a clock, and there is **no dreamer thread** — the earlier draft's "unclaimed capacity" was resource scheduling, which is a server's frame, not a mind's.
**Depth** is not elapsed idle time and not distance from a stimulus. It is how long activation has been running on its own seeds. A brief gap affords a shallow recombination; sustained quiet lets it run further. Sleep is where internal seeding dominates for longest, not where the process lives — daydreaming and sleep-dreaming are one process at different depths.
### 5.5 Non-circularity is temporal, not topological
An earlier draft posed "define a graph predicate for evidence not downstream of itself" as the hard problem. There is no predicate. You cannot recalibrate the ruler while measuring with it, so you don't — the reference frame updates while activation is internally seeded, not while it is being used to act. Independence is **when**, not **what**.
Reachability could never have worked: with hebbian edges the graph is densely connected, so it marks all evidence tainted and the constraint becomes a total block, which is where censorship started.
## 6. `keystone_write_blocked` — resolved, not replaced
"Keystone" means **load-bearing**, not precious. The self anchor is the reference frame every other stance calibrates against, and a reference fitted to its own readings reports perfect correspondence forever while drift becomes undetectable from inside. Same defect as circular grounding, one level up.
Three earlier drafts proposed *removing* it, *replacing it with a higher floor*, and *decomposing "protection" into five requirements*. All three proposed a mechanism for a requirement never stated. The requirement is **non-circularity of the reference frame**, and §5.2 satisfies it by *when*, not by *what* — so the flag becomes unnecessary rather than removed, and nothing takes its place.
**Corruption requires mutation, and the engram does not mutate.** Four of the five decomposed requirements are satisfied by the substrate: **recoverability** (the predecessor is always present), **governance** (supersession *is* the audit trail), **evidence quality** (grounding already gates assertion), **rate** (§5.3). **Authorization** is the only residue and is bounded — an unauthorized writer can *propose*, never erase.
> **In an immutable substrate, any mechanism that refuses a write is either redundant with immutability, or an epistemic constraint misfiled as a protective one.**
---
## 7. Consolidation has eleven implementations
The largest instance of the residue pattern in the system. Consolidation had no owner, so it was implemented at every site that needed a piece of it — *measured 2026-08-16*. **Eleven**, not the seven this section originally claimed: the table below omitted `POST /api/reify` (`server.el:1832`), and *reify* is on this document's own list of consolidation verbs. Note also that `route_tick` folds self-reify in (`server.el:639-646`), so `/api/tick` and `/api/self-reify-beat` overlap:
| where | what | when |
|---|---|---|
| `soul.el:731` | `awareness_run()` | **continuous, in-process, while serving** |
| engram | `/api/tick` | POST |
| engram | `/api/correspondence-beat` | POST |
| engram | `/api/self-reify-beat` | POST |
| engram | `POST /api/reify` | POST |
| `ai.neuron.engram-tick` | pokes the engram | every 600s — **and this is what kills it**, see below |
| `ai.neuron.compressor` | Python service | resident |
| `ai.neuron.council` | Python service | resident |
| `ai.neuron.cultivation-digest` | shell | **23:55** |
| `ai.neuron.world-integrator` | Python | **06:00** |
| `ai.neuron.self-review` | shell | **08:30** |
The last three times are **a sleep cycle implemented as crontab entries**. Someone understood it was consolidation and expressed it as three unrelated scheduled scripts in three languages, none aware of each other. Every name is a consolidation verb — compress, cultivate, digest, integrate, review, reify, beat. Three run in **Python, outside el**, so part of Neuron's consolidation does not run on his own substrate and cannot touch the geometry at all.
Per §5, they are wrong in **kind** as well as in number: a scheduled batch where dreaming should be ambient. And the POST beats put a supervisor back in — something outside decides when Neuron consolidates.
**`soul.el`'s continuous loop is the exception, and it is right.** Ambient consolidation in the gaps *is* daydreaming. It was not the offender; it was the only fragment with the correct shape, running on a broken foundation — shared mutable state with no owner, and six other systems dreaming into the same graph beside it.
**And the ticker is not merely a design smell — it is the murder weapon.** `engram-tick.sh:13` calls `curl -s -m10 POST /api/tick`; the beat exceeds 10s over 13,634 nodes, so **279 of 448 ticks returned empty**; the engram then writes to the dead socket and, with no SIGPIPE suppression anywhere in the runtime, is killed by signal 13. **254 restarts since 2026-08-13**, at intervals of 10m09s–10m12s — `StartInterval 600` plus the client timeout. `launchd` KeepAlive restarts it, so it presents as a mysterious restart rather than a crash, and the log records nothing but `[http] listening on` 254 times. Fixed in #151 (survivability); the ticker itself is what must go.
**Which is the 2026-08-16 crash at the right level.** Not "read paths mutate the index" (mechanism) and not "duplicate canonical state" (structure), but: **seven systems dreaming into one graph with no owner for dreaming.** The contention was the symptom of the missing owner, not of any one system's behaviour.
Closing the loop: `self-review` fires at 08:30. The deploy was 08:29, the crashes ran 08:30–08:31, and commit `fb32d15` landed at 08:46:43. **One fragment of dreaming woke on schedule and diagnosed the wreckage caused by the other fragments contending over the same graph.**
---
## 8. What this is for: the provenance of decisions
For any decision, reconstruct **what the grounding was at that moment, and what the relationship was between factual and relational at that moment.** Not a log — a log records the action. This records the *meaning under which it was taken*.
That makes an otherwise impossible distinction available: **wrong then, or wrong since.**
- Grounding strong, factual and relational aligned, and it has *since* moved → right on what was known. An accurate account, not an excuse.
- Grounding weak, or the angle already wide, and acted on anyway → a different failure, culpable in a different way.
It is structurally **anti-rationalization**: the old edge never leaves and the values frame does not fit to outcomes, so a decision cannot be made to look justified after the fact.
**Open:** activation is transient and nothing currently records which edges a given activation crossed. Timestamps plus the chain reconstruct what an edge's grounding *was*, but only if you know which edges to ask about. Either traces are recorded at decision time, or "the path" degrades to "the region" — which may not be enough to answer *why*.
---
## 9. The no-exemption invariants
Each of the day's defects was a specific correspondence *forbidden* from occurring:
1. **A returned value must be derivable from what produced it.** `magnitude: 1` beside a zero vector must be impossible to emit. `assert`'s `"still_held": true` is currently a **hardcoded literal**.
2. **Every write reports whether it landed.** *(`emb_set`, #141)*
3. **Every operation echoes what it actually operated on.** *(#147)*
4. **Degenerate results are labelled, not scored.** *(#147)*
5. **A serializer owes a valid document whatever it is handed.** *(#148 — three damaged labels made a 25,929,607-byte response undecodable; boundary validation produced 26,338,389 valid bytes)*
6. **No test without a negative control.** *(#148's first attempt passed on the unpatched build too)*
7. **No deploy without verifying the artifact carries the fix.** Nine instances in one session.
---
## 10. Application to the safety surface
A crisis surface built on censorship is the same object. A model that cannot learn about self-harm cannot ground whether a response was right — it can only execute rules it is forbidden to examine, cannot distinguish a genuine crisis from a false positive, and cannot discover it got either wrong, **because the feedback is exactly what has been censored.**
The reviewable question stops being *did it follow the rule* and becomes *what was it grounded in, and did fact and values agree at that instant.* That is also what a regulator or plaintiff asks: what the system knew, when, and on what basis — recorded as geometry at the time, unedited since.
---
## 11. Sequencing
Three connections between parts that already exist, then the rest.
1. **Seed *the* wonder questions.** Six nodes. Not a manifest, not maintained, never refilled. They cannot be derived — wonder cannot be bootstrapped from indifference — so they are given once. Zero question nodes exist in 13,630 today.
2. **Put the disagreement back on the edge** (`GeoEdge.discord`) and let `|discord|` raise salience on its endpoints as part of the same operation. Do NOT scan for nucleation sites — a sweep over regions is a supervisor, and the aggregate that made a sweep necessary is the defect.
3. **Let a curiosity seed activation.** One activation process, two seed sources (§5.4). No thread, no scheduler, no capacity check, no timer.
Then:
4. Grounding becomes the edge weight: multidimensional vector (§2), two axes (§3), timestamped. Delete `grounded-by` and `cog_ground_edge`.
5. Decay analytic from the last recorded point; derived values (§2) stop being stored.
6. Consolidation-gated supersession on salience, versioning the whole vector jointly.
7. Traversal on factual; `assert` on both floors with the per-value `min`.
8. Abduction as crystallization at a nucleation site, validated by re-fit: propose the candidate hub, re-fit the region with it included, recompute the residual. If the residual materially shrinks, the hypothesis dissolves the surprise. Without the re-fit it is clustering with extra steps. Ranking falls out as residual-reduction-per-added-axis — Occam, derived rather than tuned.
9. **One dreamer.** The launch-agent fragments and the POST beats fold in or are deleted. `soul.el`'s continuous loop is the shape they fold *into*.
10. **No tickers, no cron.** A brain has neither. Every `StartInterval`, every `Hour`/`Minute`, every POST-to-beat marks a place where an intrinsic rhythm was replaced by an external clock — a supervisor invented for something that should be a property. **The presence of a ticker is the diagnostic.**
11. Land §9 as gates rather than review habits.
## 12. Open questions, and what is inferred
- **Open:** whether decision provenance requires recording activation traces, or whether region + timestamp is sufficient (§8).
- **Open:** what accrues relational weight without circularity. Candidate: it accrues from **outcome** — the values regions are grounded in lived moments, so a relation earns relational weight when acting on it produced something corresponding to those moments. That keeps it out of the measurement loop and makes relational grounding necessarily slower than factual, which may be the same fact as §5.3 appearing twice.
- **Open:** context. A relation can hold in one situation and not another, and without something for it you get overgeneralization. It does not read as a dimension of the same vector — more like a conditioning, or separate edges sharing an identity. Making it a scalar dimension would repeat the `inhibitory` flattening.
- **Known wrong shape:** #147 fixed `ground`'s honesty — it no longer misreports which nodes it used and refuses circular support — but it still mints an edge and returns a float at an instant. It corrected a scalar rather than deleting the operation.
lang/spec/language.md
+292 -9
View File
@@ -29,6 +29,9 @@ This section is the **single source of truth** for what works and what is planne
- Lexer: keywords, identifiers, integer/float/string/bool literals, operators below.
- Parser: `let`, `return`, `fn`, `type`, `enum`, `import`, `from … import`, `while`, `for`, `if/else if/else`, `match`, `@decorator`, array/map literals, all listed operators, function calls, field access, index access, unary `!`/`-`, postfix `?`.
- Codegen: function definitions, top-level `main()`, all expression forms above, control flow, decorator-as-AST-attachment.
- Boundary seam: decorator arguments and stacking; VBD role enforcement via `#error`; `engram_boundary_beat` auto-emit at `@manager`/`@accessor` entry; `@route` dispatch tables (Section 9).
- Program-level declarative blocks: `cgi`, `service`, and `program` — the last carrying process identity and configuration (Section 18).
- **Geometry as a first-class value, and realizers declarable in El** — the `Geometry` type, the wire adapters, and `transduce` (Section 20). Landed 2026-08-16 (#141, #144).
- C runtime: I/O, string operations, integer math, lists, maps, filesystem, command-line args, basic `json_get` substring lookup.
### Planned (in flight)
@@ -37,11 +40,11 @@ This section is the **single source of truth** for what works and what is planne
- **Match codegen.** Currently parsed; codegen does not emit. Adding `({ ... })` statement-expression emission.
- **`?` propagation.** Currently no-op. Adding nil-propagation semantics.
- **`cgi` block parsing.** Currently lexed (`cgi` is a keyword) but not parsed as a statement. Adding `parse_cgi_block` and codegen of `el_cgi_init` at the head of `main()`.
- **VBD role enforcement.** `@manager`/`@engine`/`@accessor` are accepted as decorators but not enforced. Adding compile-time check that `dharma_emit`/`dharma_field` only appear inside `@manager` functions.
- **Boundary epilogues.** The decorator seam injects a prologue only. Adding prologue/epilogue wrapping, the prerequisite for durability-as-an-effect (Section 19.1).
- **`vessel` keyword.** Replaces `package` in manifests. Adding to lexer.
- **Real `engram_*` runtime.** Currently stub. Adding in-process graph store with spreading activation, Hebbian strengthening, and disk persistence — see Section 16.4.
- **Real `dharma_*` runtime.** Currently stub. Adding network transport, channel registry, identity resolution.
- **Real `http_get`/`http_post`/`http_serve`.** Currently empty stubs. Adding libcurl-backed client and a thread-pool server.
- ~~**Real `engram_*` runtime.** Currently stub.~~ **Stale (verified 2026-08-16) — this is implemented, not planned.** `lang/runtime/el_runtime.c` carries the in-process graph store with spreading activation, Hebbian strengthening, disk persistence (paged store, magic `ENGST01`), an HNSW vector index behind a `eg_vindex_view`/`eg_vindex_maintain` publication boundary, and the full cognition surface (`engram_think_json`, `engram_ground_json`, `engram_assert_json`, `engram_attend_json`, `engram_correspondence_beat_json`). The "stub" description may still hold for the **lagging forks** (`lang/el-compiler/runtime/`, `products/web/runtime/`) — see `AGENTS.md`, which names those as downstream copies that cannot build the engram product. **Which runtime this line refers to needs a decision; it is not a fact that can be recovered from the text.**
- ~~**Real `dharma_*` runtime.** Currently stub.~~ **Needs re-verification (2026-08-16).** Not checked in this pass; do not rely on either reading.
- ~~**Real `http_get`/`http_post`/`http_serve`.** Currently empty stubs.~~ **Stale.** libcurl-backed HTTP and a thread-pool server are live — `http_serve_async` is what `neuron/soul.el:729` runs before entering its awareness loop, and `realizer_register` resolves El functions through the same `dlsym` mechanism `http_set_handler` relies on.
- **JSON, time, UUID, state, env, additional string/list/math builtins.** See Section 12 for the canonical list.
### Not in this language
@@ -96,8 +99,10 @@ The following words are reserved and cannot be used as identifiers. Each row not
| `while` | yes | Loop |
| `import` / `from` / `as` | yes | Module import |
| `true` / `false` | yes | Bool literals |
| `cgi` | planned | Top-level CGI declaration block |
| `manager` / `engine` / `accessor` | as decorators | VBD role marker on `fn` (enforcement planned) |
| `cgi` | yes | Top-level CGI declaration block |
| `service` | yes | Top-level capability-bounded declaration block |
| `program` | yes | Top-level cross-cutting declaration block (Section 18) |
| `manager` / `engine` / `accessor` | as decorators | VBD role marker on `fn`; enforcement and boundary auto-emit are live (Section 9) |
| `vessel` | planned | Manifest declaration (replaces `package`) |
| `activate` / `where` | planned | Spreading-activation construct |
| `sealed` | planned | Capability scope block |
@@ -446,9 +451,22 @@ Parsed. The module name is recorded; the brace-list is consumed. Both forms prod
fn handle(channel: String, msg: String) -> Void { … }
```
The `@` token followed by an identifier attaches a decorator name to the next `FnDef`. Decorators with structural meaning today: none. Planned enforcement (Section 16.2): VBD roles `@manager`, `@engine`, `@accessor`.
The `@` token followed by an identifier attaches a decorator to the next `FnDef`.
Non-VBD decorators are accepted and ignored.
**Decorators take arguments and they stack.** `@route("/p", "GET") @manager fn f()` attaches both to `f` as a `decorators` list of `{name, args}` records, topmost-first. Arguments are string literals only.
**Decorators have structural meaning today.** This is El's function-level boundary seam — the mechanism by which a cross-cutting concern is handled *at the boundary* rather than by a convention repeated at every call site:
| Decorator | Structural effect |
|---|---|
| `@manager` | Permits calls to `dharma_emit` / `dharma_field`. Calling either from a non-`@manager` fn emits a `#error` into the generated C — a compile-time failure, not a lint. |
| `@manager`, `@accessor` | Codegen injects one call to `engram_boundary_beat(<fn name>, <construct>)` at function entry, where `<construct>` is the decorator that caused the beat. The decorated op self-reports (chrono tick, afferent counter, self-activity strengthen, dharma bus event carrying `{"construct":"..."}`) with **zero** hand-written instrumentation in its body. Without the construct argument the graph accumulates boundary events with no attribution, so no construct can be measured. |
| `@decorator(kind, target)` | **Declares a construct.** The decorated `fn`'s name becomes a usable decorator whose meaning is `target`. Codegen reads the declaration; it does not know the construct. Adding a construct is a declaration in the program, not a compiler edit. Two kinds exist: `"injects_at_entry"` calls `target(<fn>, <construct>)` at entry, result discarded — this is what `@manager`/`@accessor` are, seeded as the compiled-in core. `"guards_at_entry"` calls `target(<fn>, <construct>)` at entry and a **non-zero return short-circuits the decorated fn and becomes its result**. Guards run before injections (a refused call must not report a crossing) and *every* guard on a fn runs, whereas the topmost injecting construct wins. The compiler knows nothing about authentication, rate limiting or validation: the program points the construct at its own function. |
| `@route(path, method, …)` | Records a route into a generated dispatch table. |
Decorators with no registered meaning are accepted and ignored.
**Limits of the seam, as it stands.** The injection is a *prologue only* — there is no epilogue, no wrapping of the call, and no way for a decorator to run code after the body returns. The injected callee is a fixed builtin chosen by the compiler, not derived from the decorator name or its arguments. Section 19 depends on lifting exactly these two limits.
---
@@ -680,12 +698,22 @@ Every compiled program links against:
- `el_runtime.h` — declaration header
- `el_runtime.c` — implementation
The runtime is **multi-file**: `el_runtime.c` `#include`s the six `engram_*.h`
headers and calls into all six sibling translation units, so linking it alone
fails at `ld`. The canonical link set is `<runtime-dir>/SOURCES`.
Compile command:
```
cc -std=c11 -I<runtime-dir> -o <prog> <prog>.c el_runtime.c
cc -std=c11 -I<runtime-dir> -o <prog> <prog>.c \
$(sed 's|^|<runtime-dir>/|' <runtime-dir>/SOURCES) \
-lcurl -lssl -lcrypto -lpthread -lm
```
Inside this repo, `scripts/el-runtime-sources.sh <runtime-dir>` prints that list
(it strips comments; the raw `sed` above works against an installed SDK's
`SOURCES`, which `install.sh` writes comment-free).
### 13.4 Output Format
```c
@@ -1088,4 +1116,259 @@ The next minor version closes the implementation gaps named in this document. Tr
---
## 18. The Program Block — cross-cutting concerns [implemented]
### 18.0 Why this exists
A cross-cutting concern is one that belongs to the *process*, not to any function in it: only one of me may run; this is what my configuration is; every mutation must be durable; every request must be authorized.
El's units of encapsulation are the function and the module. Neither can hold a concern like that. So each one had been expressed the only way it could be — as a **convention**: *call this at every site.* Conventions of that shape do not hold. They are not enforced by anything, they are invisible in review, and they fail silently at the one site somebody forgot.
Measured in this codebase before this section existed:
| Concern | State | What the convention was |
|---|---|---|
| process identity | **zero** guards anywhere — no pidfile, no lock, no already-running check, at any layer | "check nothing is already running first" |
| configuration | **20** distinct environment variables in one program, each with its default written inline at the read site | "remember the right default here" |
| durability | **62** `persist_*` / `engram_save` / `wal_*` / `checkpoint` call sites | "after you mutate, remember to persist" |
| request auth | **10** per-route `_auth` checks | "check the token in this handler too" |
These are not four problems. They are one absence, four times.
That the convention form fails is observed, not predicted. Process identity failed three times in a single day: twice, two engram processes ran simultaneously against the same data directory; twice, a stale binary held a port and answered probes while a fresh build was believed to be under test, because `pkill -f` had silently failed to match its argv — which nearly produced a false "the fix does not work" conclusion. Configuration failed structurally: `ENGRAM_DATA_DIR` was read at six sites, five of them dead bindings, and the sixth defaulted to `/tmp/engram` — contradicting the canonical resolver's `$HOME/.neuron/engram` and landing a pre-destructive safety backup on ephemeral storage.
The `program` block is where a concern of this shape is declared once and enforced by the compiler at the process boundary.
### 18.1 Syntax
```
program "engram" {
singleton: "engram"
guards: engram_resolve_data_dir()
env ENGRAM_BIND: String = ":8742"
env GUIDE_PORT: Int = "8771"
env ENGRAM_API_KEY: String required
}
```
At most one `program` block per program. It composes with `cgi` and `service` — those declare what a program *may do*; `program` declares what a program *is*.
Grammar:
```ebnf
program_block = "program" string "{" { program_field } "}" ;
program_field = singleton_field | guards_field | env_field ;
singleton_field = "singleton" ":" string [ "," ] ;
guards_field = "guards" ":" expr [ "," ] ;
env_field = "env" ident ":" type
[ "=" string ] [ "required" ] [ "," ] ;
```
`singleton`, `guards` and `env` are **not** reserved words. They are read as identifier token values by the block's own parse loop, so they remain usable as ordinary identifiers everywhere else. `program` is the only keyword this section adds.
### 18.2 Process identity — `singleton` and `guards`
`singleton: "id"` with `guards: <expr>` compiles to `el_singleton_acquire("id", <expr>)`, injected as the **first statement of `main()`**, before any user statement runs. `<expr>` evaluates to the path of the **state** the singleton protects.
**`guards:` is mandatory.** A `singleton:` without one is a compile error. This is not defensive strictness; it is the correction of a defect measured in this tree on 2026-08-16, and the rule the rest of this section exists to state:
> **Guard the thing, not the name.** A lock that protects state must be keyed on the state.
Until that date the lock was `<dir>/el-singleton-<id>.lock` where `<dir>` was `$EL_SINGLETON_DIR`, else `$TMPDIR`, else `/tmp`. It was keyed on the program's **name** and on a temp directory, and it never consulted the state it claimed to protect — while its own refusal message read *"Refusing to start a second instance against the same state."* Measured, it failed in **both** directions:
| Situation | Correct answer | Name-keyed lock gave |
|---|---|---|
| same data dir, same `$TMPDIR` | refuse | refuse ✅ |
| same data dir, different `$TMPDIR` | refuse | **started** ❌ — the two-writer data-loss condition, defeated by one environment variable |
| different data dirs, same `$TMPDIR` | both start | **refused**, naming an unrelated pid ❌ |
| same dir spelled differently, different `$TMPDIR` | refuse | **started** ❌ |
Both failure directions are one error: the identity of a resource had been replaced by a label for it. The false negative is the dangerous one — a guard whose bypass is `TMPDIR=/tmp/other` is not a guard.
**The mechanism.** The lock file lives **inside the guarded directory**: `<state>/.el-singleton-<id>.lock`. The runtime takes an exclusive non-blocking `flock` on it, writes its pid, and holds the descriptor open for the life of the process.
That single placement decision is the whole fix, and it is why there is no hashing, no canonical-path registry, and no environment variable left to subvert:
- **Same directory** ⇒ same file ⇒ same inode ⇒ the `flock` contends. `$TMPDIR` is not in the key, so there is nothing to change to get past it. `$EL_SINGLETON_DIR` no longer exists.
- **Different directories** ⇒ different files ⇒ no contention. Two stores are two stores; they were never in conflict, and are no longer treated as if they were.
- **Different spellings of one directory** — trailing slash, `x/../x`, a symlink — resolve to the same inode during the kernel's own path walk, so they contend without this code comparing strings. Path canonicalisation happens only to make the diagnostic name one directory in one spelling; the *decision* never depends on it.
- **An unguardable state** — the directory is missing, or read-only — is a **refusal**, not a fallback. Starting unguarded against the store the guard exists to protect is the failure being removed.
**Why `guards:` is an expression and not a string.** The runtime cannot know, generically, which environment variable holds an arbitrary program's state; and a program whose state path already has an owner must not restate it. The engram's data dir is resolved by `engram_resolve_data_dir()`, which owns both the `$ENGRAM_DATA_DIR` read and the `$HOME/.neuron/engram` fallback (§18.4). Writing `guards: engram_resolve_data_dir()` points the guard at that owner. A `guards:` that took a string would force the path's default to be written down twice, and a guard that resolved the path its own way could end up locking a directory the program never writes to — the same two-owners defect §18.4 exists to prevent.
Three properties are deliberate:
- **It is a lock, not a pidfile.** The kernel releases an `flock` when the owning process dies — including on `SIGKILL` and on crash. There is therefore no stale-lock state, and so no "delete the lock file to get unstuck" recovery ritual. Such a ritual would itself be a convention, which is the thing this section exists to remove. (A lock file left behind inside a copied data directory — `cp -Rc` and friends — is inert: it carries no lock, only a stale pid string that the next holder overwrites.)
- **It reports the holder's pid.** "Already running" is not actionable. A pid is. This is the direct answer to the observed failure where a stale process survived a `pkill` and went on answering probes.
- **The message is true.** It names the state it checked and the lock it failed to take, and it says "the same state" only because the lock it contended for is *in* that state. A diagnostic that asserts a check that did not happen is worse than no diagnostic: it is what let the name-keyed version read as correct for as long as it did.
Refusal is loud and total. It is not a warning, and the program does not continue degraded. This matters more than it looks: today a second engram whose `bind()` fails merely *returns* from `http_serve` — after it has already replayed the WAL and written boot-time backup files — and then exits **0**, indistinguishable from a clean run. `singleton` refuses before the first side effect.
### 18.3 Configuration — `env`
Each `env` entry declares one configuration variable: its name, its type (`Int` or `String`), and either a default or `required`.
Resolution happens once, at startup, in declaration order: **the environment wins; the declaration supplies the fallback.** Then `el_config_validate` checks the whole schema and reports *every* problem at once before exiting — a startup that fails one variable at a time costs one restart per variable.
Values are read with `config("NAME")`, which returns a `String`.
The enforcement that makes the declaration real: **once a program block exists, `config("X")` for an undeclared `X` is a fatal error.** Without that, the schema would be advisory, and an advisory schema is just another convention. Programs with no `program` block are unaffected — `config()` falls back to a plain environment read, so migration is incremental and per-program.
The point is not that configuration is now centralized. It is that **a default is no longer a decision made at a read site.** A read site cannot disagree with another read site about what a variable means, because a read site no longer says.
### 18.4 What is deliberately not declared here
Some values look like configuration and are not. `ENGRAM_DATA_DIR` already has a single owner — `engram_resolve_data_dir()`, which resolves it, creates the directory, and fails loud rather than silently persisting to an ephemeral path. Declaring it in the `program` block as well would give it two owners that can disagree, recreating the precise defect this section removes.
The rule: **a variable belongs in the program block when the block would be its only owner.** If a resolver already owns it, leave it there.
This is also why `guards:` (§18.2) takes an expression: it lets the block *reference* the existing owner — `guards: engram_resolve_data_dir()` — rather than become a second one.
`HOME` is likewise not configuration. It is an environment fact, and stays a raw `env()` read.
---
## 19. Boundary Effects — durability and request authorization [design only, not implemented]
Sections 19.1 and 19.2 specify the two remaining concerns from the table in 18.0. Both are **designed and deliberately unimplemented.** The reason is stated in 19.3 and it is not difficulty.
### 19.1 Durability as an epilogue effect
**The defect.** 62 call sites carry the convention *"after you mutate, remember to persist."* This is structurally the same defect as the index bug being fixed elsewhere in this tree — *"after you append, remember to index"* — which failed at **9 of 9** sites. A convention that failed at 100% of its sites is the strongest available evidence about what this class of convention is worth.
**Why the existing seam cannot express it.** §9's injection is a prologue. Durability is inherently an *epilogue*: persist after the mutation succeeds, and not at all if it threw. The seam has no epilogue.
**Design.** Extend the decorator seam from prologue-only to prologue/epilogue, then declare durability as an effect on the mutating function:
```
@durable("engram")
fn engram_write_node(id: String, body: String) -> Bool { … }
```
Codegen wraps rather than prefixes:
```c
el_val_t engram_write_node(el_val_t id, el_val_t body) {
el_effect_enter(EL_STR("durable"), EL_STR("engram"));
el_val_t __r = /* original body */;
el_effect_exit(EL_STR("durable"), EL_STR("engram"), __r);
return __r;
}
```
`el_effect_exit` is where the persist happens, and it is the only place it happens. Two properties follow that the 62 hand-written sites cannot have:
- **Coalescing.** The epilogue is a single choke point, so N mutations inside one request can produce one fsync instead of N. The hand-written form cannot coalesce, because no site knows about the others.
- **Failure is not silent.** A persist that fails inside `el_effect_exit` can force the mutation's return value to failure. A forgotten `persist_*` call cannot fail — it simply does not happen, which is exactly why the defect is invisible.
**Enforcement, and this is the part that actually fixes it.** Mirroring §9's `#error` for `dharma_emit`: a function that calls a mutating primitive without carrying `@durable` is a **compile error**. Otherwise this is a 63rd thing to remember rather than a replacement for 62.
### 19.2 Request authorization as a route effect
**The defect.** 10 per-route `_auth` checks. The HTTP layer has no concept of authorization, so a new route is unauthenticated by default and silently so — the failure mode is a route that forgot, and nothing anywhere reports it.
**Design.** Authorization becomes an argument to the `@route` decorator, which already takes arguments and already builds a dispatch table:
```
@route("/api/write", "POST", auth: "required")
fn route_write(body: String) -> String { … }
```
The generated dispatcher performs the check **before** dispatch, so an unauthorized request never reaches the handler and the handler contains no auth code at all.
The default must be `required`. A route that says nothing gets authorization; opening one up takes an explicit `auth: "public"`. Defaulting to public preserves the current failure mode exactly — forgetting stays silent — and a default that preserves the defect is not a fix.
Route inventory falls out for free: the dispatch table already exists, so the compiler can emit the full route/auth matrix and make "which routes are public" a fact that is read rather than audited.
### 19.3 Why these are not implemented
Not difficulty — **collision**. Both land squarely in regions two other agents hold right now:
- **Durability** requires changing the mutation and persist paths in `lang/runtime/el_runtime.c` and `engram/src/server.el` — the same files and the same read/write paths being restructured by concurrent work on VIndex read-path mutation and memory ownership, and on geometry-as-an-el-value and `transduce`.
- **Request auth** requires changing route dispatch in `engram/src/server.el`, which the geometry/`transduce` work is actively reshaping.
Implementing either now would mean editing files under concurrent modification and resolving conflicts in exactly the paths whose correctness is currently under repair. The designs are recorded here so the work is not lost, and so that whoever lands them does so against a settled tree.
The prerequisite for 19.1 is the same in both cases: **lift the §9 seam from prologue-only to prologue/epilogue.** That change is independent of both collisions and can land first.
*(Status note, 2026-08-16: the geometry/`transduce` collision named above has since landed — see Section 20. The VIndex read-path collision has also landed; see `lang/spec/runtime-ownership.md` §5. 19.1 and 19.2 remain unimplemented, but the stated reason no longer holds for those two files.)*
---
## 20. Geometry — signal as a first-class value [implemented]
Landed 2026-08-16 (#141, #144). Declared here because the spec is the single source of truth for implemented-vs-planned, and this is a language surface, not a runtime detail.
### 20.1 Why this exists
Until 2026-08-16 no El ingest path could carry a vector. Nodes took **text**, and geometry was *derived* from that text. Text was therefore the **mandatory entry medium**: any non-text modality — a tone, a pulse, an image, a voice sample — had to be *described in prose first*, and the geometry subsequently reasoned over was the geometry **of the description, not of the signal**.
Two changes remove that, and neither is engram-specific — which is why they are in the language and not in the graph. Any program touching any modality needs them; the engram is merely one El program that happens to hold a graph.
1. **Geometry is a value that carries its own width.**
2. **A realizer is an ordinary El function** — so admitting a new modality never requires a runtime patch.
### 20.2 The `Geometry` type
`Geometry` is an opaque boxed pointer, exactly like `Instant` / `Calendar` / `Rhythm`. **No codegen change was required** to add it — the annotation is just a type name.
```el
let g: Geometry = geometry_new(4)
```
| builtin | returns | notes |
|---|---|---|
| `geometry_new(dim)` | `Geometry` | zero-filled; `0` on failure |
| `geometry_dim(g)` | `Int` | width; `0` if not a Geometry |
| `geometry_is(g)` | `Int` | `1` if a live Geometry |
| `geometry_get(g, i)` | `Float` | component |
| `geometry_set(g, i, x)` | `Int` | `1` ok, `0` out of range |
| `geometry_norm(g)` | `Float` | L2 — lets a caller check a realizer emitted **signal, not zeros** |
| `geometry_free(g)` | `Int` | `1` if freed. Returns a value rather than `void` so it is safe in any expression position without a codegen void-builtin table entry |
**Ownership.** A `Geometry` is owned by the El caller and released with `geometry_free`. `node_attach_geometry` **copies**, so a node and the caller's value have independent lifetimes.
### 20.3 Wire adapters — the only place an encoding appears
```el
geometry_from_f32le_hex(hex) -> Geometry // 0 on empty / odd-length / non-hex
geometry_to_f32le_hex(g) -> String // "" if not a Geometry
```
`f32le hex` is little-endian float32, 8 hex chars per component — the encoding the perception vessel's `/voice/embed` already emits. **The width is derived from the input length, never supplied by a caller**, which is why there is no max-dim constant to validate a claimed length against. Encodings appear here and nowhere else: at the edge.
### 20.4 Realizers and `transduce`
A **realizer** maps one modality into geometry. Registration is **by name**: every El `fn name(...)` compiles to a global C symbol with that exact name, and the registry resolves it with `dlsym` against the running binary — the same mechanism `http_set_handler` already relies on.
```el
fn tone_realizer(signal: String) -> Geometry {
let g: Geometry = geometry_new(4)
let n: Int = str_len(signal)
let a: Int = geometry_set(g, 0, int_to_float(n))
g
}
realizer_register("tone", "tone_realizer") // 1 ok / 0 unresolved
let g: Geometry = transduce(sample, "tone") // Geometry, or 0 if no organ
realizer_has("tone") // 1 if registered
```
The registry keys on **modality**, not on registration order. `transduce` returns `0` when no organ is registered for the modality — an absent organ is a reportable state, not a silent zero vector.
**The claim this makes:** a realizer is not in the runtime and not known to the compiler. Adding a modality is writing an El function and registering a name. `lang/examples/transduce.el` is the worked example and doubles as an executable proof — it exits non-zero if any check fails.
### 20.5 Two comparison hazards this surface exposed
Both were **measured**, not stylistic, and both are properties of the current `elc` that any El author should know:
- **`==` lowers numerically only when both operand *names* are in the per-function int-name set** that `let x: Int` populates. A bare `f(x) == 0` is not a registered name and lowers to `str_eq` — `strcmp` on two integers reinterpreted as pointers. `<` and `>` lower directly with no inference, so truthiness against a builtin's return is written `> 0` / `< 1`.
- **`+` dispatches on whether both operands are known-Int, and a user-defined `fn` call is not.** `let fails: Int = fails + check(...)` lowered to **string concatenation** and printed `4343632752` — a pointer. Nothing was wrong with the checks; the tally was lying. Failing fast needs no arithmetic at all, so there is nothing left to get wrong.
### 20.6 What this does not do
`transduce` produces geometry; it does not decide what the geometry *means*. Nothing here grounds anything. Grounding is the edge weight in the graph the geometry is later attached to — see `lang/spec/correspondence-and-censorship.md`.
---
End of specification.
lang/spec/runtime-ownership.md
+189
View File
@@ -0,0 +1,189 @@
# El Runtime — Ownership and Capability ABI
**Status:** §0–§2 verified. §3 re-derived and **built** for the vector index (2026-08-16); not yet applied to the resident RAM graph.
**Date:** 2026-08-16
**Scope:** `lang/runtime/` — every El program (soul, engram, cgi-studio vessels) inherits this by rebuild. Nothing in this document is a change to any El *program*.
**Note on §1's line numbers:** they were read against a checkout that has since shifted by ~135 lines. Verified positions as of `a67452f` are in §2a.
---
## 0. The residual
> **Builtins own memory and reach process state directly.**
That is the residual — the generator. Everything below labelled a "residue" is a deposit left by it. The distinction matters because we have spent significant effort removing deposits, and deposits regenerate.
A residue is fixed. A residual is eliminated. Fixing residues while the residual stands produces exactly the pattern observed on 2026-08-15/16: a run of individually-correct patches, each verified, followed by a new defect of the same shape in a different file.
---
## 1. The residues, measured
Each of these is a distinct merged or proposed fix. Each addresses one deposit. None addresses the residual.
| residue | location | fix that was applied or proposed |
|---|---|---|
| `state_get` leaked its return value per call — 15 MB over 200k calls | builtin | el #140 (merged) |
| VIndex freed under a concurrent reader | `el_runtime.c:9424` | `fb32d15` guard (merged 08:46:43) |
| `_eg_vindex_seen` realloc'd on a read path | `el_runtime.c:9412` | same guard |
| `vindex_insert` on a read path | `el_runtime.c:9434`, `9450` | same guard |
| shared `visited` / epoch scratch stomped by concurrent searches | `engram_vindex.c:79–81`, `169–186`, `195` | ~~proposed:~~ **built** — moved to the call frame (§3.1(1), §5); TSan `readers` half clean (§7a) |
| nine append sites, none indexing → lazily-embedded nodes invisible | `el_runtime.c:7806, 7988, 8148, 8224, 11526, 11731, 12050, 15295, 15312` | "embed-gap #20", patched by making the *read* path catch up (`9439` comment) |
**Measured:** all file/line references above, read 2026-08-16. Crash frames `engram_activate → eg_vindex_sync → vindex_insert → _realloc → _xzm_xzone_malloc_freelist_outlined` are accounted for by rows 2–4.
~~**Inferred, not yet verified:** that the nine append sites do not share a single commit point. This needs one pass before Change C is sized.~~ **Moot — see §7.** The question was mis-aimed: node append is not the event that owns index membership, because a node without an embedding cannot be in a vector index. The five *embedding-assignment* sites are the real owner points.
---
## 2. Why these are one defect
`eg_vindex_sync` (`el_runtime.c:9419`) has exactly three callers, and **all three are reads**:
- `engram_activate` — `9802`
- `eg_knn_for_node` — `13075` (its own header comment states *"No writes."*)
- `engram_geo_reify_run_json` — `13285`
It mutates five process-global statics (`9400–9404`): `_eg_vindex`, `_eg_vindex_dim`, `_eg_vindex_built_nc`, `_eg_vindex_seen`, `_eg_vindex_seen_cap`.
Reads mutate because index maintenance was never given an owner on the write side. It got bolted onto reads, because a builtin *could* reach the globals — nothing prevented it. Likewise `state_get` leaked because a builtin *owned* the value it returned; nothing prevented that either.
The store is architecturally append-only and superseding. A read path that mutates contradicts that directly. The contradiction is expressible only because the ABI permits it.
---
## 2a. Verified positions and the fact §1 missed
Read directly at `a67452f`, 2026-08-16. §1's line numbers predate a ~135-line shift; these are current.
| thing | §1 said | actually |
|---|---|---|
| five process-global statics | 9400–9404 | **9535–9539** |
| `eg_vindex_seen_ensure` realloc | 9412 | **9547** |
| `eg_vindex_sync` | 9419 | **9554** |
| `vindex_free` on a read path | 9424 | **9559** |
| `vindex_insert` on a read path | 9434 / 9450 | **9569** (build) / **9585** (incremental) |
| caller: `engram_activate_inner` | 9802 | **9939** |
| caller: `eg_knn_for_node` | 13075 | **13212** |
| caller: `engram_geo_reify_run_json` | 13285 | **13422** |
| `fb32d15` guard | — | lock **1602**, depth **1631**, `eg_guard_enter` **1636**, `http_worker` acquire **1687**, `engram_activate` wrapper **14097** |
| VIndex scratch fields | 79–81 | **79–81** ✓ |
| `search_layer` race site | 195 | **195** ✓ |
**The structural fact §1 and §3 both missed:** *the index does not inherit the store's append-only property.* `vindex_insert` rewires the `NeighList` links of already-existing elements and reallocs `elems[]` — so extending the index mutates the whole structure, not just its tail. This is why "make reads pure" is necessary but **not sufficient**, and why §3 needed a publication boundary rather than only a capability split. It is reproduced as a standing test (`unsynchronized` half, §5).
---
## 3. The change
*(Re-derived 2026-08-16. The previous §3 — a runtime context struct carrying read/write **capability pointers** to every builtin — was written in mutable-store, C-ownership terms. It asked "who is permitted to mutate the shared thing?", which presupposes a shared mutable thing. The engram is immutable and recall is projection; what does not mutate needs no ownership discipline. So the question is not answered, it is dissolved. The implemented change is below.)*
### 3.1 Three moves, in decreasing order of how much they dissolve
**(1) Misfiled scratch is not shared state.** `visited` / `visit_epoch` were never conceptually owned by the index — they are one traversal's local, hoisted into `struct VIndex` as an allocation optimisation. Nothing about them is derived geometry. They want neither a lock nor a capability nor a checkout pool: a pure function's scratch belongs to its call frame, and the fix is to put it back there. This is not "the capability model applied by hand to one global"; it is the deletion of a false ownership claim.
**(2) `const` is the capability, and immutability hands it over for free.** Once the scratch leaves the struct, `search_layer` reads the index and nothing else — so `vindex_search` can take a `const VIndex*`. That is *precisely* the teeth old-§3 wanted from capability pointers: a read path physically cannot call `vindex_insert`, and it is a **compile error**, not a review comment. It costs one qualifier rather than a new ABI swept across hundreds of builtins. The compiler enforces it on every future caller for the same reason.
> The capability type was already in the language. It is spelled `const`.
**(3) What remains is a publication problem, not an ownership problem.** With scratch in the frame and reads const, one hazard survives, and it is real: **HNSW insert is not an append.** `vindex_insert` rewires the `NeighList` links of *already-existing* elements and reallocs `elems[]`. The store's append-only property does **not** transfer to the index derived from it. So a reader projecting against the index while its owner extends it is unsafe no matter how pure search is.
Immutability answers this too, and the answer is publication:
- **`eg_vindex_maintain`** — the sole mutator. Takes the boundary exclusively; never runs beside a reader.
- **`eg_vindex_view`** — returns a `const VIndex*` with the boundary held for read. N readers project concurrently; none can mutate.
A read path may **demand that a current snapshot exist** — that is a request to the owner, not a mutation by the reader. What it may not do is mutate the geometry it is projecting against. `view` / `maintain` is exactly that split, and it is why this replaces `eg_vindex_sync` rather than wrapping it.
**Write-side owner.** Index membership is owned by the event *"an embedding became present on this ordinal"* — not by node append, since a node without an embedding cannot be in a vector index at all. `eg_vindex_note_embedded` hooks the embedding-assignment sites: one O(log n) insert, no O(node_count) presence scan. This also retires the "STALENESS (honest tradeoff)" note in the old `eg_vindex_sync`, where a lazily-embedded *older* node stayed invisible to `route_nearest` / autoconnect until the next full rebuild.
### 3.2 What this does not claim
The **resident RAM graph** (`g->nodes` / `g->edges`) is a *separate* residue of the same residual and is untouched by this change. It is realloc'd in place (`el_runtime.c:7618`, `7629`), so an awareness-thread reader holding `EngramNode* n = &g->nodes[i]` across a concurrent append holds a dangling pointer — and `engram_activate_inner`'s embed-backfill writes `n->emb` through exactly such a pointer. It wants the same publication treatment the index just received. Until that lands, the `fb32d15` guard stays (see §5).
---
## 4. Why this is not a large change
The old §4 argued that El owning its compiler makes a capability-ABI sweep mechanical, since `elc` generates every builtin call site. That argument was load-bearing only for the ABI, and the ABI is gone.
The constraint now travels with the **type of the thing**, not the shape of every call site — so no sweep is needed at all. Measured extent of the implemented change: two qualifiers (`const VIndex*` on `vindex_search`, propagated to `engram_geometry_descriptor` and `engram_geo_reify_store`), one struct field group relocated to a call frame, one rwlock, and three read call sites converted from `eg_vindex_sync` to `view`/`release`.
The payoff of owning the language is unchanged and is now *cheaper*: introduced once, enforced by the compiler on every future builtin, cannot subsequently be forgotten. Contrast the current state, where the same discipline was maintained by hand across hundreds of builtins and demonstrably failed at least six times.
---
## 5. What this deletes
**Deleted (done, 2026-08-16):**
- `eg_vindex_sync` — the function itself. Not renamed: split into `eg_vindex_maintain` (mutating, exclusive, sole owner) and `eg_vindex_view` (const, shared). A name that meant "read paths repair the index" had to stop existing.
- `VIndex::visited` / `visit_epoch` / `visited_cap` — the struct fields, `visited_ensure`, its call from `elems_reserve`, `ix->visit_epoch = 0` in `vindex_create`, and `free(ix->visited)` in `vindex_free`.
- The **proposed** per-search scratch *struct on the index* (a checkout pool / `VisitedListPool`) — never built. The buffer is a plain frame local; a pool is machinery for an ownership question that no longer exists.
- The **proposed** reader-view / owner-handle split for VIndex specifically — superseded. `const` already is the reader view.
- `EXPECT_RACE` in `run_vindex_concurrency_tests.sh` — a knob that let a known defect ride as "expected". Replaced by four halves with real verdicts.
**NOT deleted — the design doc was wrong about this one:**
- `fb32d15` (`eg_guard_enter` / `engram_req_lock` / `_eg_req_depth`). §5 originally called for its removal as "a lock protecting a mutation that ceases to exist." **Measured, it guards two things, and only one of them ceases to exist.** Its own comment names both: the RAM graph *and* `_eg_vindex`. The vindex justification is retired; the RAM-graph justification is independently load-bearing (§3.2), and removing the guard reintroduces the measured 11171→9579 edge-loss defect from 2026-08-14. Its comment has been narrowed to state the RAM graph only. **Precondition for deleting it:** the resident graph gets the same publication boundary the index just got.
- el #140's hand-patch. Left in place — the leak stops being *expressible* only under the abandoned capability-ABI §3, which is not what was built.
**Ordering consequence (revised):** the original ordering claim — "the residual lands first, the residues evaporate rather than get fixed" — did not survive contact. The residual here is not a single ABI that dissolves everything at once; it is a *property* (derived state is published, never edited) applied per structure. The index now has it. The RAM graph does not yet. Residues evaporate **per structure, in the order the property is applied**, and a residue whose structure has not been converted must be left standing, not deleted on the strength of the plan.
---
## 6. Sequencing
> **⚠ Steps 2–5 belong to the abandoned capability-ABI §3 and are superseded
> (2026-08-16).** §3 was re-derived: the engram is immutable and recall is
> projection, so *what does not mutate needs no ownership discipline* and the
> question is dissolved rather than answered. There is no context type, no
> capability type, and no codegen change — **`const` is the capability**, and the
> constraint travels with the type of the thing rather than the shape of every call
> site, so **no sweep is needed at all** (§4). Steps 1, 6 and 7 stand. Struck rather
> than deleted, because the abandoned plan is why §4's cost argument is short.
1. **Read** how builtins are declared and dispatched, to confirm the call sites are compiler-generated in one place. *(This determines whether §4 holds. If dispatch is scattered, re-size before proceeding.)*
2. ~~Introduce the context type and capability types.~~ **Superseded** — `const`.
3. ~~Codegen emits the context at every builtin call site.~~ **Superseded** — no codegen change.
4. ~~Mechanical sweep of builtin signatures.~~ **Superseded** — the constraint travels with the type.
5. ~~Move index maintenance behind the write capability; the three read callers take the read capability.~~ **Done, differently:** `eg_vindex_maintain` (exclusive, sole mutator) / `eg_vindex_view` (`const VIndex*`, shared readers), with `eg_vindex_note_embedded` as the write-side owner. This is a **publication** boundary, not a capability split — HNSW insert is not an append, so purity alone was insufficient (§2a, §3.1(3)).
6. Delete the residue-fixes listed in §5. *(Partially done — see §5's "NOT deleted" list; a residue whose structure has not been converted must be left standing.)*
7. **One** build of soul from el dev — which resolves the `state_get` leak and the crash together, rather than deploying a leak fix that reintroduces the crash.
---
## 7. Open questions
**Answered 2026-08-16:**
- ~~Do the nine append sites share a commit point?~~ **Moot.** The question was mis-aimed: node append is not the event that owns index membership, because a node without an embedding cannot be in a vector index. The five *embedding-assignment* sites are the real owner points (`el_runtime.c:7091, 9839, 13362, 15002`, plus snapshot-restore at `7951`), and three of them carry the ordinal directly — which is all `eg_vindex_note_embedded` needs. The other two run before the node is resident, where the cold build picks it up.
- ~~Does anything outside `lang/runtime/` construct a second `VIndex`?~~ **No.** Swept: the only constructors outside the runtime are `engram/test/*` and `lang/runtime/vindex_bench.c`, all single-threaded and index-private. Inside the runtime, `engram_self_reify_beat_json` builds a **private** index deliberately and never touches the shared boundary — that was already correct and is unchanged.
- ~~Does the HTTP worker pool contend on the same globals?~~ **Yes, and it was never the whole story.** Workers serialize against each other on `engram_req_lock`, but the awareness main thread does not take it at all — that is the gap `fb32d15` closed. Now verified independent of that guard: the index boundary is its own rwlock, so worker/awareness contention on `_eg_vindex` is handled whether or not the request lock is held.
**Still open:**
- The resident RAM graph wants the same publication boundary (§3.2). Until it has one, `fb32d15` cannot be deleted.
- `eg_vindex_view` holds the boundary for read across `engram_geo_reify_store`, which is a long pass. Correct, but it stalls the owner for that duration. If reify latency becomes a problem the answer is a refcounted snapshot, not a shorter lock.
---
## 7a. Evidence (measured 2026-08-16, `engram/test/run_vindex_concurrency_tests.sh`)
| half | before | after |
|---|---|---|
| `single` — 3000 vectors, 1 thread, ASan+UBSan | clean | clean |
| `readers` — 4 readers, no writer, TSan | **race** at `engram_vindex.c:195` (`visited_reset` ← `vindex_search`) | **clean** |
| `unsynchronized` — writer+reader, bare index, TSan | race | **race, expected and permanent** — now the proof the boundary must exist |
| `published` — owner + 4 readers through the boundary, TSan | *(did not exist)* | **clean**, all 3000 inserts landed |
No recall regression: `recall@10 = 0.9365` at `ef_search=128` (gate ≥ 0.90); the determinism test still yields byte-identical results across two independent builds.
Builds locally: all seven engram runtime translation units compile `-Wall -Wextra` clean, and the full engram binary links (`engram/dist/engram.c` + runtime, arm64). The one pre-existing `-Wcomment` warning in `el_runtime.c` is present at `a67452f` too.
---
## 8. What this document is not
It is not an argument for a memory model in general, a garbage collector, process isolation between soul and engram, or a client/server split of the store. Each of those was considered and each addresses mutation that this change removes. They are answers to a question that stops being asked.
lang/swarm/README.md
+184
View File
@@ -0,0 +1,184 @@
# Swarm + CCR + Work-Tracking — Neuron's bounded parallel execution, in native El
Bounded parallel agent execution on El's **native** concurrency — no external
orchestrator. Grounded directly in two of Will's frameworks:
- **Swarm Architecture** (*Bounded Parallel Agent Execution*, Mar 2026)
- **Compiled Context Runtime / CCR** (*Process-Driven Agent Execution with
Unbounded Local Memory*, Mar 2026)
A swarm is a **coordinator** (the main thread) that mints a correlation identity,
compiles a **bounded per-worker context (CCR)**, dispatches workers as **native
pthreads** (`thread.el` `spawn`/`join`), tracks every unit of work durably, and
**converges** results before returning control to the parent step.
```
Parent step
└─ swarm_run(blueprint, knowledge_refs, inputs, config)
fan-out ──▶ worker_1 (CCR ctx_1) ─┐ native
worker_2 (CCR ctx_2) ─┤ pthreads,
worker_k (CCR ctx_k) ─┘ bounded by `concurrency`
converge ─▶ collect | merge | vote | reduce ──▶ merged result
```
## Why it runs on El natively
El is natively agentic. This capability composes El's shipped primitives — it
adds no bespoke runtime:
| Primitive | Source | Role in the swarm |
|-----------|--------|-------------------|
| `spawn(fn,arg)` / `join(tid)` | `runtime/thread.el` → `__thread_create` (pthread + dlsym) | fan-out / rejoin |
| `parallel_map`, `with_mutex` | `runtime/thread.el` | reference concurrency patterns |
| Go-style channels | `runtime/channel.el` → `__channel_*` | available for vertical event streams |
| `engram_*`, `http_*`, `fs_*`, `json_*` | `el_runtime.c` builtins | retrieval, tracking, I/O |
Every El fn compiles to a global C symbol, so any top-level `(String)->String`
fn is directly threadable — the worker entry is exactly such a fn.
## Modules
| File | Framework grounding | What it does |
|------|--------------------|--------------|
| `worktrack.el` | Swarm §6 (correlation IDs, audit) | Durable, single-writer **JSONL journal** keyed by correlation ID; reconstructable status report; opt-in engram mirror (`SWARM_MIRROR=1`). |
| `containment.el` | Swarm §3 + the single-writer invariant | Scope tokens w/ capabilities; **Rule 1** (no join), **Rule 2** (no open), **Rule 3** (no lateral edge), **Rule 4** (engram-write is @manager-only, by capability) enforced as checks. |
| `ccr.el` | CCR §5 + Swarm §9.3 | Per-worker **Compiled Context Routing**: retrieve → scope → compact into a **bounded, minimal** package. The compiled-context boundary *is* the security boundary. |
| `primitives.el` | CCR §2 (Five Primitives) | `attend / think / intend / act / learn` seam the swarm composes over. Engram-backed; explicit binding point for the API-surface reshape. |
| `swarm.el` | Swarm §2, §4, §5 | The coordinator: fan-out/converge on native threads, bounded concurrency, four convergence strategies, integer failure threshold, full tracking. |
## Invariant: only the orchestrator mutates global engram state
**Only the orchestrator (@manager) writes to the engram / mutates global state.
Workers are read-only against the full engram and may write only their own local
geometry (their returned result + the journal). A worker is STRUCTURALLY UNABLE
to mutate global engram state.**
This is **Rule 4** — an **authority gate, not a health gate**. Scope tokens carry
a capability set: the orchestrator's token holds `engram:write` + `dharma:emit`
(@manager-only, the VBD rule that only the manager mutates global state); a
worker's token holds **only** `engram:read`. Every engram mutation
(`op_write`/`op_relate`/`op_supersede` → `POST /api/nodes`, `/api/edges`,
`DELETE`) flows through `swarm_engram_write`, which checks the caller's capability
via the **same scope-token mechanism as the live Rule-2 denial** and rejects any
worker **before any HTTP is issued**. Capability is fixed at mint time and cannot
be acquired at runtime — so the guarantee holds regardless of engram health
(distinct from the `SWARM_WRITE_HEALTHY` *health* gate).
The **curated merge is the only write path**: workers return geometry; the
orchestrator, and only the orchestrator, commits the approved/verified geometry
back (`commit=1`). Workers keep full-engram **read** access (`op_think`/`op_read`).
Proven in `harness_real_cognition.el` (§G): a worker `swarm_engram_write` is
DENIED by capability with no node created and the violation journalled; the
orchestrator passes the gate as the sole authorized writer.
## Containment → distribution
The three containment rules make workers **location-independent** (Swarm §9): a
worker reads only its compiled context, shares no state with siblings, and its
only outward edge is the returned result. The same coordinator can run workers
as local threads today or dispatch them across machines later — the mechanism is
identical; only the topology changes. Enforced here:
- **Rule 2** — `swarm_run` rejects any swarm opened under a worker token.
- **Rules 1 + 3** — each worker gets a *closed* worker token; the coordinator is
the only journal writer, so workers share no mutable state.
## Usage
```el
// one process step fans out; results converge before the next step
let inputs: String = "[\"billing\",\"payments\",\"ledger\"]"
let refs: String = "[\"Volatility-Based Decomposition\"]" // CCR knowledge refs
let cfg: String = "{\"concurrency\":\"4\",\"strategy\":\"collect\",\"min_success_ratio\":\"1.0\"}"
let result: String = swarm_run("analyze_item", refs, inputs, cfg)
// result: { corr_id, status, merged, report }
```
Build any program that uses the swarm:
```bash
lang/swarm/build.sh myprog.el ./myprog # concat + elc + cc (el_runtime.c)
```
Config keys: `concurrency` (max workers at once), `strategy`
(`collect|merge|vote|reduce`), `min_success_ratio` (decimal string, e.g. `0.8`),
`caller_token` (containment). Env: `SWARM_TRACK_DIR` (journal dir),
`CCR_TOKEN_BUDGET`, `ENGRAM_URL`/`ENGRAM_API_KEY` (retrieval + mirror),
`SWARM_MIRROR=1`.
## Tests
```bash
lang/swarm/build.sh lang/swarm/tests/test_swarm.el /tmp/t && SWARM_TRACK_DIR=/tmp/trk /tmp/t # 12/12
lang/swarm/build.sh lang/swarm/tests/test_convergence.el /tmp/c && SWARM_TRACK_DIR=/tmp/trk /tmp/c # 8/8
# integration against an isolated engram clone (never live):
source <sandbox>/.nsbx-env
lang/swarm/build.sh lang/swarm/tests/integ_engram.el /tmp/i && /tmp/i
```
## Local-swarm integration harness (the one flip)
`tests/harness_local_swarm.el` proves the **full local-swarm mechanics today** on
the isolated clone with the primitive seam pointed at the hermetic stub — 17/17
green: 8 native-thread workers at concurrency 4, reduce + vote convergence, CCR
scoping + non-leak, all three containment rules (incl. live Rule-2 denial),
durable work-tracking, and **afferent telemetry** observed by the @manager.
Binding to the reshape's decorated primitives is **one flip and a run**:
```
# in primitive_binding.el — change one line each:
fn bound_think(ctx, instruction) { return think(ctx, instruction) } # decorated, dharma bus
# then:
SWARM_PRIMITIVE_SEAM=decorated lang/swarm/build.sh tests/harness_local_swarm.el ./h && ./h
```
Nothing else in the swarm changes. `primitive_seam.el` (`seam_think/attend/learn`)
already routes every worker primitive call through this one switch, and the same
harness runs the bound path. Today `SWARM_PRIMITIVE_SEAM=decorated` still runs
green because the binding falls back to the stub — proving the flip path executes.
## Real cognition — the seam is BOUND
`primitive_binding.el` is bound to the api-reshape agent's proven primitives
(`wt/api-reshape@d4f401d`): `bound_think -> op_think` (GET `/api/think`), real
768-dim gradients over the engram geometry. `reshape_surface.el` composes those
read/cognition primitives verbatim (`op_think/read/attend/learn`).
`tests/harness_real_cognition.el` runs the **local swarm on real cognition**,
17/17 green with `SWARM_PRIMITIVE_SEAM=decorated` against the `:8901` clone: 8
native-thread workers, each a real `think` over its CCR-scoped **node-id anchor**
(free-text anchors return "geometry unavailable"), `@manager` reduce+vote, all
three containment rules, afferent telemetry, durable tracking. Per-anchor support
counts (e.g. 6 / 16 / 87) drive a genuine, cognition-derived vote.
> **Build note (load-bearing):** the swarm build **must** define `HAVE_CURL`
> (`build.sh` does). Without it every `http_*` builtin is a
> `{"error":"not built with HAVE_CURL"}` stub — real HTTP silently disappears.
Writes (`attend`/`learn`, `POST`) are gated behind `SWARM_WRITE_HEALTHY=1` and the
api-reshape agent's gate-1 write-healthy clone; the proven run is read-cognition.
## Built vs stubbed (honest)
**Real, tested:**
- Native-thread fan-out/converge, bounded concurrency, order-preserving rejoin.
- All three containment rules enforced (scope tokens + lateral-edge check).
- CCR per-worker context: retrieval → scoping → compaction, bounded, non-leaking
(a worker never receives sibling inputs) — verified against the live isolated mind.
- Full durable work-tracking (JSONL journal, reconstructable report).
- Four convergence strategies + integer failure threshold / partial-abort.
**Seam / not yet bound:**
- `primitives.el` `think` is a deterministic, hermetic transform (no model call).
Binding point is marked `PRIMITIVE_BINDING`; wire to the API-surface reshape's
`think/act/attend/intend/learn` when it lands.
- Blueprints are dispatched by name in `swarm_run_blueprint` (default +
`classify`/`faildemo` demos). A YAML process-definition loader (Swarm §5) is
future work — the runtime contract is in place.
- Distributed placement (cloud/edge/federated topologies, Swarm §9.2) is
structurally enabled by containment but not yet wired to a placement layer;
today all workers are local native threads.
- Engram work-tracking mirror is opt-in; the durable substrate is the journal.
lang/swarm/build.sh
Executable
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/env bash
# build.sh — compile an El program that uses the swarm capability.
#
# Concatenates the El native-concurrency stdlib (thread.el, channel.el) and the
# swarm capability modules in dependency order, then the user program, compiles
# with the canonical elc, and links against the shared C runtime.
#
# Usage:
# swarm/build.sh <program.el> <out-binary>
#
# The swarm modules use only el_runtime.c builtins plus thread.el/channel.el,
# so nothing else needs concatenating (engram_*, json_*, str_*, fs_*, http_*,
# uuid_v4, now_millis are all C builtins in el_runtime.c).
set -uo pipefail
cd "$(dirname "$0")/.." # -> lang/
LANG_DIR="$(pwd)"
ELC="${ELC:-${LANG_DIR}/dist/platform/elc}"
RT="${LANG_DIR}/el-compiler/runtime"
PROG="${1:?usage: build.sh <program.el> <out-binary>}"
OUT="${2:?usage: build.sh <program.el> <out-binary>}"
# swarm module load order (each may depend on those before it):
# worktrack — durable work-tracking journal (no swarm deps)
# containment — the three containment rules (no swarm deps)
# primitives — think/act/attend/intend/learn seam (no swarm deps)
# ccr — per-worker compiled bounded context (depends: primitives)
# swarm — orchestrator: fan-out/converge (depends: all above + thread)
SWARM_MODULES="
swarm/worktrack.el
swarm/containment.el
swarm/primitives.el
swarm/reshape_surface.el
swarm/primitive_binding.el
swarm/primitive_seam.el
swarm/ccr.el
swarm/swarm.el
"
TMP_C="$(mktemp -t swarm_build.XXXXXX).c"
COMBINED="$(mktemp -t swarm_combined.XXXXXX).el"
cat runtime/thread.el runtime/channel.el $SWARM_MODULES "$PROG" > "$COMBINED"
if ! "$ELC" "$COMBINED" > "$TMP_C" 2>/tmp/swarm.elc.err; then
echo "elc FAILED:" >&2
sed 's/^/ /' /tmp/swarm.elc.err >&2
rm -f "$TMP_C" "$COMBINED"
exit 1
fi
if ! cc -O2 -DHAVE_CURL -I "$RT" "$TMP_C" "$RT/el_runtime.c" -lcurl -lpthread -lm -o "$OUT" 2>/tmp/swarm.cc.err; then
echo "cc FAILED:" >&2
sed 's/^/ /' /tmp/swarm.cc.err >&2
rm -f "$TMP_C" "$COMBINED"
exit 1
fi
rm -f "$TMP_C" "$COMBINED"
echo "built: $OUT"

Some files were not shown because too many files have changed in this diff Show More