Compare commits

..

1 Commits

Author SHA1 Message Date
Tim Lingo f6aa072d5f fix(el-runtime/win): stop truncating HTTP responses to the last fs_read length
The Windows el runtime stashes the byte count of the last fs_read in a
thread-global (_tl_fs_read_len) so binary files (PNG/WOFF2) serve without
strlen truncation at embedded NULs. But it was consumed unconditionally:
any handler that fs_reads a file and then returns a DIFFERENT, longer
response (e.g. /api/safety-contact reads the 178-byte contact file, then
wraps it in {"configured":true,"contact":...,"ok":true}) had its reply
clipped to the file size — the client received truncated, invalid JSON.

Record the buffer the length belongs to (_tl_fs_read_ptr) and only trust the
byte count when the handler returns that exact buffer; otherwise measure the
built response with strlen. Fixes every fs_read-then-transform endpoint
(safety-contact GET/POST, etc.). macOS (native runtime) was unaffected.

Verified root cause on Windows 11: GET/POST /api/safety-contact returned
Content-Length 178 with a body truncated mid-JSON; macOS returned the full
208-byte body. This is why the Neuron Windows app could not save the safety
contact (and would fail most soul calls).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-26 15:12:31 -05:00
246 changed files with 871 additions and 69453 deletions
+31 -103
View File
@@ -22,10 +22,7 @@ jobs:
- name: Install build dependencies
run: |
apt-get update -qq
apt-get install -y gcc libcurl4-openssl-dev apt-transport-https ca-certificates
echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" \
> /etc/apt/sources.list.d/google-cloud-sdk.list
apt-get update -qq && apt-get install -y google-cloud-cli
apt-get install -y gcc libcurl4-openssl-dev
# Seed: use the committed linux-amd64 binary as the bootstrap
- name: Bootstrap from committed linux binary (seed)
@@ -39,9 +36,9 @@ jobs:
run: |
dist/platform/elc-linux-amd64 elc-cli.el > dist/elc-gen2.c
gcc -O2 \
-I runtime \
-I el-compiler/runtime \
dist/elc-gen2.c \
runtime/el_runtime.c \
el-compiler/runtime/el_runtime.c \
-lcurl -lssl -lcrypto -lpthread -lm \
-o dist/platform/elc
chmod +x dist/platform/elc
@@ -54,9 +51,9 @@ jobs:
mkdir -p dist/bin
dist/platform/elc elb.el > dist/elb.c
gcc -O2 \
-I runtime \
-I el-compiler/runtime \
dist/elb.c \
runtime/el_runtime.c \
el-compiler/runtime/el_runtime.c \
-lcurl -lssl -lcrypto -lpthread -lm \
-o dist/bin/elb
chmod +x dist/bin/elb
@@ -87,22 +84,13 @@ jobs:
bash tests/html_sanitizer/run.sh
# Native El test suites (elc --test, compile-link-run)
# el_runtime.c is precompiled to .o once and reused by all 8 modules.
- name: Precompile el_runtime.o
run: |
set -euo pipefail
RUNTIME="$(pwd)/runtime"
gcc -O2 -c -I "$RUNTIME" "$RUNTIME/el_runtime.c" \
-o /tmp/el_runtime.o
echo "el_runtime.o compiled"
- name: Run tests - native (core)
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_core.el > /tmp/el_native_core.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_core
/tmp/el_native_core
@@ -110,9 +98,9 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_text.el > /tmp/el_native_text.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_text
/tmp/el_native_text
@@ -120,9 +108,9 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_string.el > /tmp/el_native_string.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_string
/tmp/el_native_string
@@ -130,9 +118,9 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_math.el > /tmp/el_native_math.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_math
/tmp/el_native_math
@@ -140,9 +128,9 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_state.el > /tmp/el_native_state.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_state
/tmp/el_native_state
@@ -150,9 +138,9 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_time.el > /tmp/el_native_time.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_time
/tmp/el_native_time
@@ -160,9 +148,9 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_json.el > /tmp/el_native_json.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_json
/tmp/el_native_json
@@ -170,9 +158,9 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_env.el > /tmp/el_native_env.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_env
/tmp/el_native_env
@@ -180,9 +168,9 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_fs.el > /tmp/el_native_fs.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c /tmp/el_runtime.o \
gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_fs
/tmp/el_native_fs
@@ -191,7 +179,7 @@ jobs:
run: |
ABS_ELB="$(pwd)/dist/bin/elb"
ABS_ELC="$(pwd)/dist/platform/elc"
ABS_RUNTIME="$(pwd)/runtime"
ABS_RUNTIME="$(pwd)/el-compiler/runtime"
ABS_OUT="$(pwd)/dist/bin"
(cd ../epm && "$ABS_ELB" --clean --elc="$ABS_ELC" --runtime="$ABS_RUNTIME" --out="$ABS_OUT")
chmod +x dist/bin/epm
@@ -202,7 +190,7 @@ jobs:
run: |
ABS_ELB="$(pwd)/dist/bin/elb"
ABS_ELC="$(pwd)/dist/platform/elc"
ABS_RUNTIME="$(pwd)/runtime"
ABS_RUNTIME="$(pwd)/el-compiler/runtime"
ABS_OUT="$(pwd)/dist/bin"
(cd tools/install && "$ABS_ELB" --clean --elc="$ABS_ELC" --runtime="$ABS_RUNTIME" --out="$ABS_OUT")
chmod +x dist/bin/el-install
@@ -214,18 +202,12 @@ jobs:
env:
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
run: |
# Fail loudly: previously this step had no `set -e`, so an auth or
# upload failure was swallowed (step exited 0 on the trailing echo)
# and the SDK silently never published. Surface failures now.
set -euo pipefail
if [ -z "${GCP_SA_KEY:-}" ]; then
echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2
exit 1
fi
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
apt-get install -y -qq apt-transport-https ca-certificates curl
echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
apt-get update -qq && apt-get install -y google-cloud-cli
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
gcloud config set project neuron-785695
echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)"
VERSION="${GITHUB_SHA:0:8}"
@@ -251,7 +233,7 @@ jobs:
--project=neuron-785695 \
--package=el-runtime-c \
--version="${VERSION}" \
--source=runtime/el_runtime.c
--source=el-compiler/runtime/el_runtime.c
gcloud artifacts generic upload \
--repository=foundation-dev \
@@ -259,7 +241,7 @@ jobs:
--project=neuron-785695 \
--package=el-runtime-h \
--version="${VERSION}" \
--source=runtime/el_runtime.h
--source=el-compiler/runtime/el_runtime.h
gcloud artifacts generic upload \
--repository=foundation-dev \
@@ -267,61 +249,7 @@ jobs:
--project=neuron-785695 \
--package=el-runtime-js \
--version="${VERSION}" \
--source=runtime/el_runtime.js
--source=el-compiler/runtime/el_runtime.js
echo "Published El SDK version=${VERSION} to foundation-dev"
# Keep key alive for the ci-base rebuild step below
# (deleted in that step after docker push)
- name: Rebuild ci-base with fresh El SDK (dev)
# Patches ci-base:dev in-place: pulls the existing image (which has all
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
#
# continue-on-error: this is a CI-cache optimization, NOT the release
# artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE
# runner where DinD/Docker availability is fragile. A failure here must
# never block or redden the job — the SDK publish above is the deliverable.
continue-on-error: true
if: github.event_name == 'push'
env:
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
run: |
set -euo pipefail
CI_BASE="us-central1-docker.pkg.dev/neuron-785695/neuron-ci/ci-base"
SHA="${GITHUB_SHA:0:8}"
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
gcloud config set project neuron-785695
gcloud auth configure-docker us-central1-docker.pkg.dev --quiet
# Pull existing ci-base:dev (or fall back to :latest on first run)
BASE_TAG="dev"
docker pull "${CI_BASE}:dev" || { docker pull "${CI_BASE}:latest" && BASE_TAG="latest"; }
# Inline Dockerfile — only replaces the El SDK layer
cat > /tmp/Dockerfile.ci-base-patch << 'EOF'
ARG BASE
FROM ${BASE}
COPY dist/platform/elc /opt/el/dist/platform/elc
COPY dist/bin/elb /opt/el/dist/bin/elb
COPY runtime/el_runtime.c /opt/el/runtime/el_runtime.c
COPY runtime/el_runtime.h /opt/el/runtime/el_runtime.h
COPY runtime/el_runtime.js /opt/el/runtime/el_runtime.js
RUN chmod +x /opt/el/dist/platform/elc /opt/el/dist/bin/elb
EOF
docker build \
--build-arg BASE="${CI_BASE}:${BASE_TAG}" \
--build-arg BUILDKIT_INLINE_CACHE=1 \
-f /tmp/Dockerfile.ci-base-patch \
-t "${CI_BASE}:dev" \
-t "${CI_BASE}:dev-${SHA}" \
.
docker push "${CI_BASE}:dev"
docker push "${CI_BASE}:dev-${SHA}"
echo "ci-base rebuilt: ${CI_BASE}:dev (${SHA})"
rm -f /tmp/gcp-key.json
+17 -79
View File
@@ -46,9 +46,9 @@ jobs:
run: |
dist/platform/elc-linux-amd64 elc-cli.el > dist/elc-gen2.c
gcc -O2 \
-I runtime \
-I el-compiler/runtime \
dist/elc-gen2.c \
runtime/el_runtime.c \
el-compiler/runtime/el_runtime.c \
-lcurl -lssl -lcrypto -lpthread -lm \
-o dist/platform/elc
chmod +x dist/platform/elc
@@ -84,7 +84,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_core.el > /tmp/el_native_core.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_core
@@ -94,7 +94,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_text.el > /tmp/el_native_text.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_text
@@ -104,7 +104,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_string.el > /tmp/el_native_string.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_string
@@ -114,7 +114,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_math.el > /tmp/el_native_math.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_math
@@ -124,7 +124,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_state.el > /tmp/el_native_state.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_state
@@ -134,7 +134,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_time.el > /tmp/el_native_time.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_time
@@ -144,7 +144,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_json.el > /tmp/el_native_json.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_json
@@ -154,7 +154,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_env.el > /tmp/el_native_env.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_env
@@ -164,7 +164,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_fs.el > /tmp/el_native_fs.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_fs
@@ -176,9 +176,9 @@ jobs:
mkdir -p dist/bin
dist/platform/elc elb.el > dist/elb.c
gcc -O2 \
-I runtime \
-I el-compiler/runtime \
dist/elb.c \
runtime/el_runtime.c \
el-compiler/runtime/el_runtime.c \
-lcurl -lssl -lcrypto -lpthread -lm \
-o dist/bin/elb
chmod +x dist/bin/elb
@@ -189,7 +189,7 @@ jobs:
run: |
ABS_ELB="$(pwd)/dist/bin/elb"
ABS_ELC="$(pwd)/dist/platform/elc"
ABS_RUNTIME="$(pwd)/runtime"
ABS_RUNTIME="$(pwd)/el-compiler/runtime"
ABS_OUT="$(pwd)/dist/bin"
(cd ../epm && "$ABS_ELB" --clean --elc="$ABS_ELC" --runtime="$ABS_RUNTIME" --out="$ABS_OUT")
chmod +x dist/bin/epm
@@ -200,7 +200,7 @@ jobs:
run: |
ABS_ELB="$(pwd)/dist/bin/elb"
ABS_ELC="$(pwd)/dist/platform/elc"
ABS_RUNTIME="$(pwd)/runtime"
ABS_RUNTIME="$(pwd)/el-compiler/runtime"
ABS_OUT="$(pwd)/dist/bin"
(cd tools/install && "$ABS_ELB" --clean --elc="$ABS_ELC" --runtime="$ABS_RUNTIME" --out="$ABS_OUT")
chmod +x dist/bin/el-install
@@ -212,21 +212,12 @@ jobs:
env:
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
run: |
# Fail loudly: previously this step had no `set -e`, so an auth or
# upload failure was swallowed (step exited 0 on the trailing echo)
# and the SDK silently never published. Surface failures now.
set -euo pipefail
if [ -z "${GCP_SA_KEY:-}" ]; then
echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2
exit 1
fi
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
apt-get install -y -qq apt-transport-https ca-certificates curl
echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
apt-get update -qq && apt-get install -y google-cloud-cli
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
gcloud config set project neuron-785695
echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)"
VERSION="${GITHUB_SHA:0:8}"
@@ -244,7 +235,7 @@ jobs:
--project=neuron-785695 \
--package=el-runtime-c \
--version="${VERSION}" \
--source=runtime/el_runtime.c
--source=el-compiler/runtime/el_runtime.c
gcloud artifacts generic upload \
--repository=foundation-stage \
@@ -252,60 +243,7 @@ jobs:
--project=neuron-785695 \
--package=el-runtime-h \
--version="${VERSION}" \
--source=runtime/el_runtime.h
--source=el-compiler/runtime/el_runtime.h
echo "Published El SDK version=${VERSION} to foundation-stage"
# Keep key alive for the ci-base rebuild step below
# (deleted in that step after docker push)
- name: Rebuild ci-base with fresh El SDK (stage)
# Patches ci-base:stage in-place: pulls the existing image (which has all
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
#
# continue-on-error: this is a CI-cache optimization, NOT the release
# artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE
# runner where DinD/Docker availability is fragile. A failure here must
# never block or redden the job — the SDK publish above is the deliverable.
continue-on-error: true
if: github.event_name == 'push'
env:
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
run: |
set -euo pipefail
CI_BASE="us-central1-docker.pkg.dev/neuron-785695/neuron-ci/ci-base"
SHA="${GITHUB_SHA:0:8}"
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
gcloud config set project neuron-785695
gcloud auth configure-docker us-central1-docker.pkg.dev --quiet
# Pull existing ci-base:stage (system deps stay cached in the base layer)
docker pull "${CI_BASE}:stage" || docker pull "${CI_BASE}:latest"
# Inline Dockerfile — only replaces the El SDK layer
cat > /tmp/Dockerfile.ci-base-patch << 'EOF'
ARG BASE
FROM ${BASE}
COPY dist/platform/elc /opt/el/dist/platform/elc
COPY dist/bin/elb /opt/el/dist/bin/elb
COPY runtime/el_runtime.c /opt/el/runtime/el_runtime.c
COPY runtime/el_runtime.h /opt/el/runtime/el_runtime.h
COPY runtime/el_runtime.js /opt/el/runtime/el_runtime.js
RUN chmod +x /opt/el/dist/platform/elc /opt/el/dist/bin/elb
EOF
docker build \
--build-arg BASE="${CI_BASE}:stage" \
--build-arg BUILDKIT_INLINE_CACHE=1 \
-f /tmp/Dockerfile.ci-base-patch \
-t "${CI_BASE}:stage" \
-t "${CI_BASE}:stage-${SHA}" \
.
docker push "${CI_BASE}:stage"
docker push "${CI_BASE}:stage-${SHA}"
echo "ci-base rebuilt: ${CI_BASE}:stage (${SHA})"
rm -f /tmp/gcp-key.json
+25 -44
View File
@@ -47,9 +47,9 @@ jobs:
mkdir -p dist/platform
dist/platform/elc-linux-amd64 elc-cli.el > dist/elc-gen2.c
gcc -O2 \
-I runtime \
-I el-compiler/runtime \
dist/elc-gen2.c \
runtime/el_runtime.c \
el-compiler/runtime/el_runtime.c \
-lcurl -lssl -lcrypto -lpthread -lm \
-o dist/platform/elc
chmod +x dist/platform/elc
@@ -62,9 +62,9 @@ jobs:
mkdir -p dist/bin
dist/platform/elc elb.el > dist/elb.c
gcc -O2 \
-I runtime \
-I el-compiler/runtime \
dist/elb.c \
runtime/el_runtime.c \
el-compiler/runtime/el_runtime.c \
-lcurl -lssl -lcrypto -lpthread -lm \
-o dist/bin/elb
chmod +x dist/bin/elb
@@ -75,7 +75,7 @@ jobs:
run: |
ABS_ELB="$(pwd)/dist/bin/elb"
ABS_ELC="$(pwd)/dist/platform/elc"
ABS_RUNTIME="$(pwd)/runtime"
ABS_RUNTIME="$(pwd)/el-compiler/runtime"
ABS_OUT="$(pwd)/dist/bin"
(cd ../epm && "$ABS_ELB" --clean --elc="$ABS_ELC" --runtime="$ABS_RUNTIME" --out="$ABS_OUT")
chmod +x dist/bin/epm
@@ -86,7 +86,7 @@ jobs:
run: |
ABS_ELB="$(pwd)/dist/bin/elb"
ABS_ELC="$(pwd)/dist/platform/elc"
ABS_RUNTIME="$(pwd)/runtime"
ABS_RUNTIME="$(pwd)/el-compiler/runtime"
ABS_OUT="$(pwd)/dist/bin"
(cd tools/install && "$ABS_ELB" --clean --elc="$ABS_ELC" --runtime="$ABS_RUNTIME" --out="$ABS_OUT")
chmod +x dist/bin/el-install
@@ -121,7 +121,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_core.el > /tmp/el_native_core.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_core
@@ -131,7 +131,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_text.el > /tmp/el_native_text.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_text
@@ -141,7 +141,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_string.el > /tmp/el_native_string.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_string
@@ -151,7 +151,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_math.el > /tmp/el_native_math.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_math
@@ -161,7 +161,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_state.el > /tmp/el_native_state.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_state
@@ -171,7 +171,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_time.el > /tmp/el_native_time.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_time
@@ -181,7 +181,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_json.el > /tmp/el_native_json.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_json
@@ -191,7 +191,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_env.el > /tmp/el_native_env.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_env
@@ -201,7 +201,7 @@ jobs:
run: |
set -euo pipefail
ELC="$(pwd)/dist/platform/elc"
RUNTIME="$(pwd)/runtime"
RUNTIME="$(pwd)/el-compiler/runtime"
"$ELC" --test tests/native/test_fs.el > /tmp/el_native_fs.c
gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c "$RUNTIME/el_runtime.c" \
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_fs
@@ -216,10 +216,8 @@ jobs:
cp lang/dist/platform/elc dist/sdk/bin/elc
cp lang/dist/bin/elb dist/sdk/bin/elb
cp lang/dist/bin/epm dist/sdk/bin/epm
cp lang/runtime/el_runtime.c dist/sdk/runtime/
cp lang/runtime/el_runtime.h dist/sdk/runtime/
cp lang/runtime/engram_store.c dist/sdk/runtime/
cp lang/runtime/engram_store.h dist/sdk/runtime/
cp lang/el-compiler/runtime/el_runtime.c dist/sdk/runtime/
cp lang/el-compiler/runtime/el_runtime.h dist/sdk/runtime/
cp lang/runtime/*.el dist/sdk/runtime/
tar -czf dist/el-sdk-latest.tar.gz -C dist/sdk .
echo "SDK tarball bundled: dist/el-sdk-latest.tar.gz"
@@ -276,10 +274,8 @@ jobs:
# Per-file assets (downstream CI needs these individually)
upload_asset lang/dist/platform/elc elc
upload_asset lang/runtime/el_runtime.c el_runtime.c
upload_asset lang/runtime/el_runtime.h el_runtime.h
upload_asset lang/runtime/engram_store.c engram_store.c
upload_asset lang/runtime/engram_store.h engram_store.h
upload_asset lang/el-compiler/runtime/el_runtime.c el_runtime.c
upload_asset lang/el-compiler/runtime/el_runtime.h el_runtime.h
# SDK bundle and installer binary
upload_asset dist/el-sdk-latest.tar.gz el-sdk-latest.tar.gz
@@ -292,21 +288,12 @@ jobs:
env:
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
run: |
# Fail loudly: previously this step had no `set -e`, so an auth or
# upload failure was swallowed (step exited 0 on the trailing echo)
# and the SDK silently never published. Surface failures now.
set -euo pipefail
if [ -z "${GCP_SA_KEY:-}" ]; then
echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2
exit 1
fi
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
apt-get install -y -qq apt-transport-https ca-certificates curl
echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
apt-get update -qq && apt-get install -y google-cloud-cli
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
gcloud config set project neuron-785695
echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)"
VERSION="${GITHUB_SHA:0:8}"
@@ -332,7 +319,7 @@ jobs:
--project=neuron-785695 \
--package=el-runtime-c \
--version="${VERSION}" \
--source=runtime/el_runtime.c
--source=el-compiler/runtime/el_runtime.c
gcloud artifacts generic upload \
--repository=foundation-prod \
@@ -340,7 +327,7 @@ jobs:
--project=neuron-785695 \
--package=el-runtime-h \
--version="${VERSION}" \
--source=runtime/el_runtime.h
--source=el-compiler/runtime/el_runtime.h
gcloud artifacts generic upload \
--repository=foundation-prod \
@@ -348,7 +335,7 @@ jobs:
--project=neuron-785695 \
--package=el-runtime-js \
--version="${VERSION}" \
--source=runtime/el_runtime.js
--source=el-compiler/runtime/el_runtime.js
echo "Published El SDK version=${VERSION} to foundation-prod"
# Keep key alive for the ci-base rebuild step below
@@ -358,12 +345,6 @@ jobs:
# Patches ci-base:latest in-place: pulls the existing image (which has all
# system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly
# built El SDK on top. Keeps the full ci-base rebuild fast and incremental.
#
# continue-on-error: this is a CI-cache optimization, NOT the release
# artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE
# runner where DinD/Docker availability is fragile. A failure here must
# never block or redden the job — the SDK publish above is the deliverable.
continue-on-error: true
if: github.event_name == 'push'
env:
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
@@ -386,9 +367,9 @@ jobs:
FROM ${BASE}
COPY dist/platform/elc /opt/el/dist/platform/elc
COPY dist/bin/elb /opt/el/dist/bin/elb
COPY runtime/el_runtime.c /opt/el/runtime/el_runtime.c
COPY runtime/el_runtime.h /opt/el/runtime/el_runtime.h
COPY runtime/el_runtime.js /opt/el/runtime/el_runtime.js
COPY el-compiler/runtime/el_runtime.c /opt/el/el-compiler/runtime/el_runtime.c
COPY el-compiler/runtime/el_runtime.h /opt/el/el-compiler/runtime/el_runtime.h
COPY el-compiler/runtime/el_runtime.js /opt/el/el-compiler/runtime/el_runtime.js
RUN chmod +x /opt/el/dist/platform/elc /opt/el/dist/bin/elb
EOF
+2 -2
View File
@@ -6,13 +6,13 @@ set -euo pipefail
ROOT="$(git rev-parse --show-toplevel)"
LANG_DIR="$ROOT/lang"
RUNTIME="$LANG_DIR/runtime"
RUNTIME="$LANG_DIR/el-compiler/runtime"
ELC="$LANG_DIR/dist/platform/elc"
# If elc isn't built yet, skip with a warning rather than blocking
if [ ! -x "$ELC" ]; then
echo "⚠ elc not found at lang/dist/platform/elc — skipping pre-commit tests"
echo " Build it first: cd lang && gcc -O2 -I runtime dist/elc-bootstrap.c runtime/el_runtime.c -lcurl -lpthread -o dist/elc-gen2 && ./dist/elc-gen2 el-compiler/src/compiler.el > /tmp/elc.c && gcc -O2 -I runtime /tmp/elc.c runtime/el_runtime.c -lcurl -lpthread -o dist/platform/elc"
echo " Build it first: cd lang && gcc -O2 -I el-compiler/runtime dist/elc-bootstrap.c el-compiler/runtime/el_runtime.c -lcurl -lpthread -o dist/elc-gen2 && ./dist/elc-gen2 el-compiler/src/compiler.el > /tmp/elc.c && gcc -O2 -I el-compiler/runtime /tmp/elc.c el-compiler/runtime/el_runtime.c -lcurl -lpthread -o dist/platform/elc"
exit 0
fi
-154
View File
@@ -1,154 +0,0 @@
# El
**A self-hosting, statically-typed language that compiles to C — built around a graph-native runtime instead of a database driver.**
El is the execution substrate for the Neuron agent runtime, the DHARMA network, and the Engram knowledge graph. This repository is the monorepo for the whole stack: the language itself, the graph memory engine it's built to talk to natively, and the tools (package manager, IDE, UI framework, diagramming) built on top of it.
---
## Why El exists
Every other language treats persistent, associative state as something you reach for through a driver — a SQL client, an ORM, a Redis library bolted on from outside. El inverts that: graph operations (`engram_*`) are runtime primitives, on the same footing as string or list operations. There is no separate database driver because the database is not separate.
El has four defining properties:
1. **Self-hosting compiler.** The compiler (`lexer.el`, `parser.el`, `codegen.el`, `compiler.el`) is written in El. It compiles El source to C, which `cc` compiles against a fixed runtime into a native binary. A Rust genesis compiler bootstrapped the first iteration; the self-hosted binary at `lang/dist/platform/elc` has been the canonical compiler ever since — every binary in `dist/platform/` was produced by an earlier version of itself compiling `el-compiler/src/`. The chain is auditable: source is the ground truth, not the binary. See [lang/BOOTSTRAP.md](lang/BOOTSTRAP.md) for the full recovery path if that binary is ever lost.
2. **C compilation target.** Every compiled program is plain C11. Every El value is `el_val_t` (`int64_t`); strings are heap pointers cast through it. Functions become C functions; top-level statements become `main()`.
3. **Graph-native runtime.** The runtime provides first-class graph operations over an in-process Engram store — no separate DB driver, no ORM.
4. **DHARMA-aware identity.** A `cgi` block declares a program's DHARMA identity at compile time. The runtime resolves identity before user code runs, so `dharma_*` calls have a stable principal and channel surface throughout.
---
## Architecture map
```
┌─────────────┐
│ lang │ El compiler + C runtime
│ (El itself) │ everything below is written in it,
└──────┬──────┘ or compiles down through it
┌─────────────┼─────────────┐
│ │ │
┌──────▼─────┐ ┌─────▼─────┐ ┌─────▼─────┐
│ engram │ │ epm │ │ ide │
│ graph/mem │ │ package │ │ editor + │
│ substrate │ │ manager │ │ LSP │
└──────┬─────┘ └───────────┘ └───────────┘
┌───────┼────────────────┬─────────────────────┐
│ │ │ │
┌─────▼───┐ ┌─▼──────────┐ ┌──▼──────────┐ ┌─────▼──────┐
│ elp │ │ ql │ │ ui │ │ arbor │
│ NLG / │ │engram-el. │ |spreading- │ |arbor │
│ 31 langs│ │studio+tests│ |activation UI│ |diagram lang│
└─────────┘ └────────────┘ └─────────────┘ └────────────┘
```
`lang` is the foundation — the compiler and C runtime everything else builds on. `engram` is the graph-native memory/state engine that gives El its identity (property 3 above). Everything else is either a tool for working with El (`epm`, `ide`) or a system built on top of Engram's graph model (`elp`, `ql`, `ui`, `arbor`).
---
## Repository layout
### [lang/](lang/) — the El language
The compiler and runtime. Self-hosting: `elc-cli.el``compiler.el``lexer.el` / `parser.el` / `codegen.el` / `codegen-js.el`, textually inlined and compiled in one pass. Compiles to C11 and links against `el-compiler/runtime/el_seed.c`, a hand-maintained OS-boundary layer (libcurl HTTP, pthreads, filesystem, arena allocation) — everything else in the runtime is native El (`runtime/*.el`).
Two layers to know: **El programs** (`.el` files — where nearly all work belongs) and **the C seed** (`el_seed.c` — edit only for genuine OS-level access; never re-implement what El can already express).
Current status (single source of truth: [lang/spec/language.md](lang/spec/language.md)): lexer/parser/codegen and the C runtime's core (I/O, strings, math, lists, maps, filesystem, args) are implemented. In flight: `%` operator, match-statement codegen, `?` nil-propagation, `cgi` block parsing + DHARMA identity resolution, VBD role enforcement (`@manager`/`@engine`/`@accessor`), the real `engram_*` and `dharma_*` runtimes (currently stubs), and libcurl-backed `http_get`/`http_post`/`http_serve`. Bitwise operators, `??`, and `as` casts are explicitly **not** in this language.
Key docs: [AGENTS.md](lang/AGENTS.md) (agent-facing orientation), [BOOTSTRAP.md](lang/BOOTSTRAP.md) (compiler recovery from scratch), [spec/language.md](lang/spec/language.md), [spec/codegen-js.md](lang/spec/codegen-js.md).
### [engram/](engram/) — graph intelligence substrate
**A local-first memory substrate for accumulating intelligence**, and the reason El's runtime doesn't need a database driver. Rust core (`engram-core`, `engram-ffi`) exposed to El and other languages (Kotlin, TypeScript/WASM, Go bindings).
The model: retrieval is **spreading activation**, not query. You name seed nodes and a query embedding; activation propagates outward through weighted edges, attenuating multiplicatively per hop (`strength = parent_strength × edge_weight × target_salience × cosine_sim`), gets pruned below a threshold, and the top-N nodes by activation strength come back. Storage and retrieval are the same structure — the way long-term potentiation works in biological memory, not the way a relational or vector database works.
Nodes live in four tiers (Working / Episodic / Semantic / Procedural, mirroring prefrontal / hippocampal / neocortical / cerebellar memory) and migrate between them based on **salience decay**`importance × recency-decay × log(activation_count)`. Forgetting is adaptive pruning, not a bug: unreinforced memories stop competing for attention without being deleted.
Backed by `sled` (embedded, local-first, no daemon) with flat cosine scan for vector search — deliberately simple until scale demands an HNSW layer. Full API and design rationale in [engram/README.md](engram/README.md).
### [elp/](elp/) — Engram Language Protocol
Bidirectional engine mapping between Engram semantic forms and natural-language surface text, across **31 languages** — from Spanish and Japanese through historical/liturgical languages (Old Norse, Sanskrit, Sumerian, Coptic, Akkadian, Ge'ez). Compilation order runs `language-profile` + `vocabulary` → per-language `morphology-*``grammar``realizer``semantics``elp`. This is what lets an Engram graph node round-trip to and from readable text in any of those languages.
### [epm/](epm/) — El Package Manager
Manages **vessels** (El's package unit): publish, install, resolve dependencies. Vessels are stored in Engram as graph nodes, not files in a registry index — `epm` reads the local `manifest.el`, talks to Engram over HTTP, and writes resolved vessels to `.epm/vessels/`. Source: `registry.el`, `install.el`, `update.el`, `manifest.el`.
### [ide/](ide/) — El IDE
Three vessels: **el-ide-server** (HTTP backend — file ops, build/run, LSP bridge, plugin host, settings), **el-lsp** (the language server — completion, hover, diagnostics, outline, format, type graph), and **el-plugin-host** (first-party plugin lifecycle: install/remove/enable/disable). `ide/projects/` and `ide/examples/` hold sample projects, including the canonical `hello-friends` first-program walkthrough.
### [ql/](ql/) — engram-el
The El-native integration layer for a *live* Engram server — not a library (no importable modules, no build artifact), a set of standalone `.el` programs run directly via `el run-file`. Three components: **Studio** (`studio/studio.el`, a full terminal graph explorer), a **Hebbian field-model** proof of concept, and El builtin / LLM-builtin smoke test suites. This is the reference for correct patterns when an El program uses Engram as its substrate. Spec: [ql/spec/elql.md](ql/spec/elql.md).
### [ui/](ui/) — el-ui
A frontend framework where **component state is an Engram graph and reactivity is spreading activation** — not virtual-DOM diffing (React), Proxy-based dependency tracking (Vue), or compile-time analysis (Svelte). Re-renders are activated and propagated the same way associative memory retrieval works in `engram/`.
~15 vessels covering the full frontend surface: `el-platform` (env/fs/network/clock abstraction), `el-config`, `el-html` (SSR emit primitives), `el-layout`, `el-style` (design tokens/themes), `el-i18n`, `el-auth` / `el-identity` (JWT, sessions, OAuth PKCE — Engram-native), `el-services` (REST/gRPC/WebSocket bindings), `el-aop` (`@authenticate`/`@authorize`/`@cache`/`@rate_limit` decorators), `el-secrets`, `el-graph` (graph rendering/editor), `el-publish` (App Store / Play Store automation), and `el-ui-compiler` (El→JS component compiler; currently a stub pending a JS backend in `elc`). Spec: [ui/spec/framework.md](ui/spec/framework.md).
### [arbor/](arbor/) — diagram language
A `.arbor` diagram language and toolchain: `arbor-core` (NodeId/shape/edge-kind types), `arbor-parse` (recursive-descent parser), `arbor-diagram` (IR + Mermaid serializer + architecture-diagram builders), `arbor-layout` (hierarchical layout — rank assignment, positioning, group bounds), `arbor-render` (SVG renderer), `arbor-cli`. (The architecture map above is the kind of diagram this is for.)
---
## Getting started
Install the El SDK from the latest release:
```bash
bash lang/install.sh
# EL_VERSION=v1.0.0 bash lang/install.sh # pin a specific release tag
# EL_PREFIX=/opt/el bash lang/install.sh # custom install prefix
```
Or build the compiler from source and verify the self-hosting chain:
```bash
cd lang
./dist/platform/elc elc-cli.el > elc-new.c
cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
-o dist/platform/elc-new \
elc-new.c el-compiler/runtime/el_seed.c
# Confirm the new binary reproduces itself exactly
./dist/platform/elc-new elc-cli.el > elc-verify.c
diff elc-new.c elc-verify.c # should be identical
mv dist/platform/elc-new dist/platform/elc
```
Run your first program:
```bash
./lang/dist/platform/elc lang/examples/hello.el > hello.c
cc -std=c11 -I lang/el-compiler/runtime -lcurl -lpthread \
-o hello hello.c lang/el-compiler/runtime/el_seed.c
./hello
```
More examples in [lang/examples/](lang/examples/), including a full starter project at `lang/examples/hello-project/`.
If the compiler binary is ever lost or corrupted, [lang/BOOTSTRAP.md](lang/BOOTSTRAP.md) is the authoritative recovery path.
---
## Development workflow
Branching follows `dev → stage → main`: work lands on `dev`, promotes to `stage` for integration testing, and is promoted to `main` for release (visible directly in the git history of this repo). CI is defined per-subproject under `.gitea/workflows/``lang`/`epm`/`ide` share the root pipeline; `engram` and `ql` carry their own (`ci-dev`, `ci-stage`, and a release workflow each).
- Language/runtime specs live at `*/spec/*.md` (`lang/spec/`, `ql/spec/`, `ui/spec/`) and are the single source of truth for implemented-vs-planned status — code and docs are expected to agree with the spec's status markers, not the other way around.
- Agent-facing orientation guides live at `*/AGENTS.md` (currently `lang/AGENTS.md`); more subprojects may grow their own as they need agent-specific conventions documented.
- Tagged releases live under `lang/releases/`, each with its own `RELEASE.md`.
---
## Status
This is an actively developed, internal monorepo — not yet published under an open license. Treat everything here as proprietary to Neuron Technologies unless told otherwise.
+3 -3
View File
@@ -1,7 +1,7 @@
// auto-generated by elc --emit-header — do not edit
extern fn sem_get(json: String, key: String) -> String
extern fn generate_frame(frame: [String]) -> String
extern fn generate_frame_lang(frame: [String], lang_code: String) -> String
extern fn build_form_from_json(semantic_form_json: String, lang_code: String) -> [String]
extern fn generate_frame(frame: Any) -> String
extern fn generate_frame_lang(frame: Any, lang_code: String) -> String
extern fn build_form_from_json(semantic_form_json: String, lang_code: String) -> Any
extern fn generate(semantic_form_json: String) -> String
extern fn generate_lang(semantic_form_json: String, lang_code: String) -> String
+28 -28
View File
@@ -1,22 +1,22 @@
// auto-generated by elc --emit-header do not edit
extern fn slots_get(slots: [String], key: String) -> String
extern fn slots_set(slots: [String], key: String, val: String) -> [String]
extern fn make_slots(k0: String, v0: String) -> [String]
extern fn make_slots2(k0: String, v0: String, k1: String, v1: String) -> [String]
extern fn make_slots3(k0: String, v0: String, k1: String, v1: String, k2: String, v2: String) -> [String]
extern fn make_slots4(k0: String, v0: String, k1: String, v1: String, k2: String, v2: String, k3: String, v3: String) -> [String]
extern fn make_slots5(k0: String, v0: String, k1: String, v1: String, k2: String, v2: String, k3: String, v3: String, k4: String, v4: String) -> [String]
extern fn rule_id(rule: [String]) -> String
extern fn rule_lhs(rule: [String]) -> String
extern fn rule_rhs_len(rule: [String]) -> Int
extern fn rule_rhs(rule: [String], idx: Int) -> String
extern fn make_rule(id: String, lhs: String, r0: String) -> [String]
extern fn make_rule2(id: String, lhs: String, r0: String, r1: String) -> [String]
extern fn make_rule3(id: String, lhs: String, r0: String, r1: String, r2: String) -> [String]
extern fn make_rule4(id: String, lhs: String, r0: String, r1: String, r2: String, r3: String) -> [String]
extern fn build_rules() -> [[String]]
extern fn get_rules() -> [[String]]
extern fn find_rule(rule_id_str: String) -> [String]
// auto-generated by elc --emit-header - do not edit
extern fn slots_get(slots: Any, key: String) -> String
extern fn slots_set(slots: Any, key: String, val: String) -> Any
extern fn make_slots(k0: String, v0: String) -> Any
extern fn make_slots2(k0: String, v0: String, k1: String, v1: String) -> Any
extern fn make_slots3(k0: String, v0: String, k1: String, v1: String, k2: String, v2: String) -> Any
extern fn make_slots4(k0: String, v0: String, k1: String, v1: String, k2: String, v2: String, k3: String, v3: String) -> Any
extern fn make_slots5(k0: String, v0: String, k1: String, v1: String, k2: String, v2: String, k3: String, v3: String, k4: String, v4: String) -> Any
extern fn rule_id(rule: Any) -> String
extern fn rule_lhs(rule: Any) -> String
extern fn rule_rhs_len(rule: Any) -> Int
extern fn rule_rhs(rule: Any, idx: Int) -> String
extern fn make_rule(id: String, lhs: String, r0: String) -> Any
extern fn make_rule2(id: String, lhs: String, r0: String, r1: String) -> Any
extern fn make_rule3(id: String, lhs: String, r0: String, r1: String, r2: String) -> Any
extern fn make_rule4(id: String, lhs: String, r0: String, r1: String, r2: String, r3: String) -> Any
extern fn build_rules() -> Any
extern fn get_rules() -> Any
extern fn find_rule(rule_id_str: String) -> Any
extern fn make_leaf(label: String, word: String) -> String
extern fn make_node1(label: String, child0: String) -> String
extern fn make_node2(label: String, child0: String, child1: String) -> String
@@ -24,15 +24,15 @@ extern fn make_node3(label: String, child0: String, child1: String, child2: Stri
extern fn make_node4(label: String, child0: String, child1: String, child2: String, child3: String) -> String
extern fn nlg_is_ws(c: String) -> Bool
extern fn skip_ws(s: String, pos: Int) -> Int
extern fn scan_token(s: String, start: Int) -> [String]
extern fn scan_token(s: String, start: Int) -> Any
extern fn render_tree(tree: String) -> String
extern fn gram_word_order(profile: [String]) -> String
extern fn gram_order_constituents(subj: String, verb: String, obj: String, profile: [String]) -> String
extern fn gram_build_vp(verb: String, aux: String, profile: [String]) -> String
extern fn gram_question_strategy(profile: [String]) -> String
extern fn gram_word_order(profile: Any) -> String
extern fn gram_order_constituents(subj: String, verb: String, obj: String, profile: Any) -> String
extern fn gram_build_vp(verb: String, aux: String, profile: Any) -> String
extern fn gram_question_strategy(profile: Any) -> String
extern fn is_pronoun(word: String) -> Bool
extern fn build_np(referent: String, slots: [String]) -> String
extern fn build_np(referent: String, slots: Any) -> String
extern fn build_pp(loc: String) -> String
extern fn build_vp_body(slots: [String]) -> String
extern fn build_vp_from_slots(slots: [String]) -> String
extern fn generate_tree(rule_id_str: String, slots: [String]) -> String
extern fn build_vp_body(slots: Any) -> String
extern fn build_vp_from_slots(slots: Any) -> String
extern fn generate_tree(rule_id_str: String, slots: Any) -> String
+46 -46
View File
@@ -1,46 +1,46 @@
// auto-generated by elc --emit-header do not edit
extern fn lang_profile(code: String, word_order: String, morph_type: String, has_case: String, has_gender: String, script_dir: String, agreement: String, null_subject: String) -> [String]
extern fn lang_get(profile: [String], key: String) -> String
extern fn lang_profile_en() -> [String]
extern fn lang_profile_ja() -> [String]
extern fn lang_profile_ar() -> [String]
extern fn lang_profile_zh() -> [String]
extern fn lang_profile_de() -> [String]
extern fn lang_profile_es() -> [String]
extern fn lang_profile_fi() -> [String]
extern fn lang_profile_sw() -> [String]
extern fn lang_profile_hi() -> [String]
extern fn lang_profile_ru() -> [String]
extern fn lang_profile_fr() -> [String]
extern fn lang_profile_la() -> [String]
extern fn lang_profile_he() -> [String]
extern fn lang_profile_sa() -> [String]
extern fn lang_profile_got() -> [String]
extern fn lang_profile_non() -> [String]
extern fn lang_profile_enm() -> [String]
extern fn lang_profile_pi() -> [String]
extern fn lang_profile_grc() -> [String]
extern fn lang_profile_ang() -> [String]
extern fn lang_profile_fro() -> [String]
extern fn lang_profile_goh() -> [String]
extern fn lang_profile_sga() -> [String]
extern fn lang_profile_txb() -> [String]
extern fn lang_profile_peo() -> [String]
extern fn lang_profile_akk() -> [String]
extern fn lang_profile_uga() -> [String]
extern fn lang_profile_egy() -> [String]
extern fn lang_profile_sux() -> [String]
extern fn lang_profile_gez() -> [String]
extern fn lang_profile_cop() -> [String]
extern fn lang_from_code(code: String) -> [String]
extern fn lang_default() -> [String]
extern fn lang_is_isolating(profile: [String]) -> Bool
extern fn lang_is_agglutinative(profile: [String]) -> Bool
extern fn lang_is_fusional(profile: [String]) -> Bool
extern fn lang_is_polysynthetic(profile: [String]) -> Bool
extern fn lang_is_rtl(profile: [String]) -> Bool
extern fn lang_has_null_subject(profile: [String]) -> Bool
extern fn lang_has_case(profile: [String]) -> Bool
extern fn lang_has_gender(profile: [String]) -> Bool
extern fn lang_word_order(profile: [String]) -> String
extern fn lang_code(profile: [String]) -> String
// auto-generated by elc --emit-header - do not edit
extern fn lang_profile(code: String, word_order: String, morph_type: String, has_case: String, has_gender: String, script_dir: String, agreement: String, null_subject: String) -> Any
extern fn lang_get(profile: Any, key: String) -> String
extern fn lang_profile_en() -> Any
extern fn lang_profile_ja() -> Any
extern fn lang_profile_ar() -> Any
extern fn lang_profile_zh() -> Any
extern fn lang_profile_de() -> Any
extern fn lang_profile_es() -> Any
extern fn lang_profile_fi() -> Any
extern fn lang_profile_sw() -> Any
extern fn lang_profile_hi() -> Any
extern fn lang_profile_ru() -> Any
extern fn lang_profile_fr() -> Any
extern fn lang_profile_la() -> Any
extern fn lang_profile_he() -> Any
extern fn lang_profile_sa() -> Any
extern fn lang_profile_got() -> Any
extern fn lang_profile_non() -> Any
extern fn lang_profile_enm() -> Any
extern fn lang_profile_pi() -> Any
extern fn lang_profile_grc() -> Any
extern fn lang_profile_ang() -> Any
extern fn lang_profile_fro() -> Any
extern fn lang_profile_goh() -> Any
extern fn lang_profile_sga() -> Any
extern fn lang_profile_txb() -> Any
extern fn lang_profile_peo() -> Any
extern fn lang_profile_akk() -> Any
extern fn lang_profile_uga() -> Any
extern fn lang_profile_egy() -> Any
extern fn lang_profile_sux() -> Any
extern fn lang_profile_gez() -> Any
extern fn lang_profile_cop() -> Any
extern fn lang_from_code(code: String) -> Any
extern fn lang_default() -> Any
extern fn lang_is_isolating(profile: Any) -> Bool
extern fn lang_is_agglutinative(profile: Any) -> Bool
extern fn lang_is_fusional(profile: Any) -> Bool
extern fn lang_is_polysynthetic(profile: Any) -> Bool
extern fn lang_is_rtl(profile: Any) -> Bool
extern fn lang_has_null_subject(profile: Any) -> Bool
extern fn lang_has_case(profile: Any) -> Bool
extern fn lang_has_gender(profile: Any) -> Bool
extern fn lang_word_order(profile: Any) -> String
extern fn lang_code(profile: Any) -> String
-1
View File
@@ -56,7 +56,6 @@
// String helpers
import "morphology.el"
fn akk_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn akk_str_ends(s: String, suf: String) -> Bool
extern fn akk_str_len(s: String) -> Int
extern fn akk_str_drop_last(s: String, n: Int) -> String
-1
View File
@@ -36,7 +36,6 @@
// String helpers
import "morphology.el"
fn ang_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn ang_str_ends(s: String, suf: String) -> Bool
extern fn ang_str_drop_last(s: String, n: Int) -> String
extern fn ang_str_last_char(s: String) -> String
-1
View File
@@ -21,7 +21,6 @@
// String helpers
import "morphology.el"
fn ar_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn ar_str_ends(s: String, suf: String) -> Bool
extern fn ar_str_len(s: String) -> Int
extern fn ar_str_drop_last(s: String, n: Int) -> String
-1
View File
@@ -54,7 +54,6 @@
// String helpers
import "morphology.el"
fn cop_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn cop_str_ends(s: String, suf: String) -> Bool
extern fn cop_str_len(s: String) -> Int
extern fn cop_drop(s: String, n: Int) -> String
-1
View File
@@ -26,7 +26,6 @@
// Dat: dem der dem den
// Gen: des der des der
import "morphology.el"
fn de_article_def(gender: String, gram_case: String, number: String) -> String {
if str_eq(number, "pl") {
if str_eq(gram_case, "nom") { return "die" }
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn de_article_def(gender: String, gram_case: String, number: String) -> String
extern fn de_article_indef(gender: String, gram_case: String, number: String) -> String
extern fn de_article(gender: String, gram_case: String, number: String, definite: String) -> String
-1
View File
@@ -52,7 +52,6 @@
// String helpers
import "morphology.el"
fn egy_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn egy_str_ends(s: String, suf: String) -> Bool
extern fn egy_str_len(s: String) -> Int
extern fn egy_drop(s: String, n: Int) -> String
-1
View File
@@ -31,7 +31,6 @@
// String helpers
import "morphology.el"
fn enm_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn enm_str_ends(s: String, suf: String) -> Bool
extern fn enm_drop(s: String, n: Int) -> String
extern fn enm_first_char(s: String) -> String
-1
View File
@@ -12,7 +12,6 @@
// String helpers (local, matching morphology.el conventions)
import "morphology.el"
fn es_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn es_str_ends(s: String, suf: String) -> Bool
extern fn es_str_drop_last(s: String, n: Int) -> String
extern fn es_str_last_char(s: String) -> String
-1
View File
@@ -25,7 +25,6 @@
// If only neutral vowels are found, default to "front" (the conservative choice
// for borrowed words and those without clear back vowels).
import "morphology.el"
fn fi_harmony(word: String) -> String {
let n: Int = str_len(word)
let i: Int = n - 1
+3 -3
View File
@@ -1,11 +1,11 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn fi_harmony(word: String) -> String
extern fn fi_suffix(base: String, harmony: String) -> String
extern fn fi_noun_case(stem: String, gram_case: String, number: String, harmony: String) -> String
extern fn fi_str_last_char(s: String) -> String
extern fn fi_apply_case(noun: String, gram_case: String, number: String) -> String
extern fn fi_verb_stem(dict_form: String) -> String
extern fn fi_irregular_verb(dict_form: String) -> [String]
extern fn fi_irregular_verb(dict_form: String) -> Any
extern fn fi_present_ending(stem: String, person: String, number: String, harmony: String) -> String
extern fn fi_past_stem(stem: String) -> String
extern fn fi_past_ending(stem: String, person: String, number: String, harmony: String) -> String
@@ -14,4 +14,4 @@ extern fn fi_negative(verb: String, person: String, number: String) -> String
extern fn fi_conjugate(verb: String, tense: String, person: String, number: String) -> String
extern fn fi_question_suffix(harmony: String) -> String
extern fn fi_make_question(verb_form: String, harmony: String) -> String
extern fn fi_full_paradigm(noun: String) -> [String]
extern fn fi_full_paradigm(noun: String) -> Any
-1
View File
@@ -19,7 +19,6 @@
// String helpers (local, matching morphology.el conventions)
import "morphology.el"
fn fr_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn fr_str_ends(s: String, suf: String) -> Bool
extern fn fr_str_drop_last(s: String, n: Int) -> String
extern fn fr_str_last_char(s: String) -> String
-1
View File
@@ -53,7 +53,6 @@
// String helpers
import "morphology.el"
fn fro_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn fro_str_ends(s: String, suf: String) -> Bool
extern fn fro_drop(s: String, n: Int) -> String
extern fn fro_slot(person: String, number: String) -> Int
-1
View File
@@ -64,7 +64,6 @@
// String helpers
import "morphology.el"
fn gez_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn gez_str_ends(s: String, suf: String) -> Bool
extern fn gez_str_len(s: String) -> Int
extern fn gez_str_drop_last(s: String, n: Int) -> String
-1
View File
@@ -48,7 +48,6 @@
// String helpers
import "morphology.el"
fn goh_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn goh_str_ends(s: String, suf: String) -> Bool
extern fn goh_drop(s: String, n: Int) -> String
extern fn goh_slot(person: String, number: String) -> Int
-1
View File
@@ -49,7 +49,6 @@
// String helpers
import "morphology.el"
fn got_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn got_str_ends(s: String, suf: String) -> Bool
extern fn got_str_drop_last(s: String, n: Int) -> String
extern fn got_slot(person: String, number: String) -> Int
-1
View File
@@ -31,7 +31,6 @@
// String helpers
import "morphology.el"
fn grc_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn grc_str_ends(s: String, suf: String) -> Bool
extern fn grc_str_drop_last(s: String, n: Int) -> String
extern fn grc_str_last_char(s: String) -> String
-1
View File
@@ -51,7 +51,6 @@
// String helpers
import "morphology.el"
fn he_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn he_str_ends(s: String, suf: String) -> Bool
extern fn he_str_len(s: String) -> Int
extern fn he_str_drop_last(s: String, n: Int) -> String
-1
View File
@@ -24,7 +24,6 @@
// String helpers
import "morphology.el"
fn hi_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn hi_str_ends(s: String, suf: String) -> Bool
extern fn hi_str_drop_last(s: String, n: Int) -> String
extern fn hi_str_last_char(s: String) -> String
-1
View File
@@ -23,7 +23,6 @@
// Note: this is a heuristic classifier for romanized input. For production use
// with native kana/kanji forms, the dictionary form (辞書形) must be consulted.
import "morphology.el"
fn ja_verb_group(dict_form: String) -> String {
// Irregular verbs (exact match on dictionary form)
if str_eq(dict_form, "する") { return "irregular" }
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn ja_verb_group(dict_form: String) -> String
extern fn ja_ichidan_stem(dict_form: String) -> String
extern fn ja_godan_stem_change(dict_form: String, row: String) -> String
-1
View File
@@ -25,7 +25,6 @@
// String helpers
import "morphology.el"
fn la_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn la_str_ends(s: String, suf: String) -> Bool
extern fn la_str_drop_last(s: String, n: Int) -> String
extern fn la_str_last_char(s: String) -> String
-1
View File
@@ -27,7 +27,6 @@
// String helpers
import "morphology.el"
fn non_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn non_str_ends(s: String, suf: String) -> Bool
extern fn non_drop(s: String, n: Int) -> String
extern fn non_last(s: String) -> String
-1
View File
@@ -31,7 +31,6 @@
// String helpers
import "morphology.el"
fn peo_drop(s: String, n: Int) -> String {
let len: Int = str_len(s)
if n >= len { return "" }
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn peo_drop(s: String, n: Int) -> String
extern fn peo_ends(s: String, suf: String) -> Bool
extern fn peo_slot(person: String, number: String) -> Int
-1
View File
@@ -30,7 +30,6 @@
// String helpers
import "morphology.el"
fn pi_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn pi_str_ends(s: String, suf: String) -> Bool
extern fn pi_drop(s: String, n: Int) -> String
extern fn pi_last_char(s: String) -> String
-1
View File
@@ -35,7 +35,6 @@
// The heuristic returns the most probable gender. Caller should override
// for known exceptions (путь, рубль are masc despite ).
import "morphology.el"
fn ru_gender(noun: String) -> String {
let n: Int = str_len(noun)
if n == 0 { return "m" }
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn ru_gender(noun: String) -> String
extern fn ru_stem_type(noun: String, gender: String) -> String
extern fn ru_noun_case(noun: String, gender: String, gram_case: String, number: String) -> String
-1
View File
@@ -42,7 +42,6 @@
// String helpers
import "morphology.el"
fn sa_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn sa_str_ends(s: String, suf: String) -> Bool
extern fn sa_str_drop_last(s: String, n: Int) -> String
extern fn sa_slot(person: String, number: String) -> Int
-1
View File
@@ -31,7 +31,6 @@
// String helpers
import "morphology.el"
fn sga_drop(s: String, n: Int) -> String {
let len: Int = str_len(s)
if n >= len { return "" }
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn sga_drop(s: String, n: Int) -> String
extern fn sga_first(s: String) -> String
extern fn sga_rest(s: String) -> String
-1
View File
@@ -53,7 +53,6 @@
// String helpers
import "morphology.el"
fn sux_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn sux_str_ends(s: String, suf: String) -> Bool
extern fn sux_str_drop_last(s: String, n: Int) -> String
extern fn sux_str_last_char(s: String) -> String
-1
View File
@@ -24,7 +24,6 @@
// String helpers
import "morphology.el"
fn sw_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn sw_str_ends(s: String, suf: String) -> Bool
extern fn sw_str_drop_last(s: String, n: Int) -> String
extern fn sw_str_first_char(s: String) -> String
-1
View File
@@ -30,7 +30,6 @@
// String helpers
import "morphology.el"
fn txb_drop(s: String, n: Int) -> String {
let len: Int = str_len(s)
if n >= len { return "" }
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn txb_drop(s: String, n: Int) -> String
extern fn txb_ends(s: String, suf: String) -> Bool
extern fn txb_slot(person: String, number: String) -> Int
-1
View File
@@ -48,7 +48,6 @@
// String helpers
import "morphology.el"
fn uga_str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn uga_str_ends(s: String, suf: String) -> Bool
extern fn uga_str_len(s: String) -> Int
extern fn uga_str_drop_last(s: String, n: Int) -> String
-11
View File
@@ -33,17 +33,6 @@
// String helpers
import "language-profile.el"
import "morphology-es.el"
import "morphology-fr.el"
import "morphology-de.el"
import "morphology-ru.el"
import "morphology-fi.el"
import "morphology-ar.el"
import "morphology-hi.el"
import "morphology-sw.el"
import "morphology-la.el"
import "morphology-ja.el"
fn str_ends(s: String, suf: String) -> Bool {
return str_ends_with(s, suf)
}
+5 -5
View File
@@ -1,4 +1,4 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn str_ends(s: String, suf: String) -> Bool
extern fn str_last_char(s: String) -> String
extern fn str_last2(s: String) -> String
@@ -8,7 +8,7 @@ extern fn is_vowel(c: String) -> Bool
extern fn morph_apply_suffix(base: String, suffix: String) -> String
extern fn en_irregular_plural(word: String) -> String
extern fn en_irregular_singular(word: String) -> String
extern fn en_irregular_verb(base: String) -> [String]
extern fn en_irregular_verb(base: String) -> Any
extern fn en_verb_3sg(base: String) -> String
extern fn en_should_double_final(base: String) -> Bool
extern fn en_verb_past(base: String) -> String
@@ -16,10 +16,10 @@ extern fn en_verb_gerund(base: String) -> String
extern fn en_pluralize_regular(singular: String) -> String
extern fn en_verb_form(base: String, tense: String, person: String, number: String) -> String
extern fn agree_determiner(det: String, noun: String) -> String
extern fn morph_pluralize(noun: String, profile: [String]) -> String
extern fn morph_pluralize(noun: String, profile: Any) -> String
extern fn morph_map_canonical(verb: String, code: String) -> String
extern fn morph_conjugate(verb: String, tense: String, person: String, number: String, profile: [String]) -> String
extern fn morph_inflect(word: String, features: String, profile: [String]) -> String
extern fn morph_conjugate(verb: String, tense: String, person: String, number: String, profile: Any) -> String
extern fn morph_inflect(word: String, features: String, profile: Any) -> String
extern fn pluralize(singular: String) -> String
extern fn singularize(plural: String) -> String
extern fn verb_form(base: String, tense: String, person: String, number: String) -> String
+5 -5
View File
@@ -1,10 +1,10 @@
// auto-generated by elc --emit-header do not edit
// auto-generated by elc --emit-header - do not edit
extern fn agent_person(agent: String) -> String
extern fn agent_number(agent: String) -> String
extern fn realize_np(referent: String, number: String) -> String
extern fn realize_vp_lang(base_verb: String, tense: String, aspect: String, person: String, number: String, profile: [String]) -> [String]
extern fn realize_question_lang(predicate: String, tense: String, aspect: String, person: String, number: String, agent: String, patient: String, location: String, profile: [String]) -> String
extern fn realize_vp_lang(base_verb: String, tense: String, aspect: String, person: String, number: String, profile: Any) -> Any
extern fn realize_question_lang(predicate: String, tense: String, aspect: String, person: String, number: String, agent: String, patient: String, location: String, profile: Any) -> String
extern fn capitalize_first(s: String) -> String
extern fn add_punct(s: String, intent: String) -> String
extern fn realize_lang(form: [String], profile: [String]) -> String
extern fn realize(form: [String]) -> String
extern fn realize_lang(form: Any, profile: Any) -> String
extern fn realize(form: Any) -> String
+15 -15
View File
@@ -1,18 +1,18 @@
// auto-generated by elc --emit-header do not edit
extern fn sem_frame(intent: String, subject: String, obj: String, modifiers: String) -> [String]
extern fn sem_frame_lang(intent: String, subject: String, obj: String, modifiers: String, lang_code: String) -> [String]
extern fn sem_frame_simple(intent: String, subject: String) -> [String]
extern fn sem_frame_obj(intent: String, subject: String, obj: String) -> [String]
extern fn sem_intent(frame: [String]) -> String
extern fn sem_subject(frame: [String]) -> String
extern fn sem_object(frame: [String]) -> String
extern fn sem_modifiers(frame: [String]) -> String
extern fn sem_lang(frame: [String]) -> String
// auto-generated by elc --emit-header - do not edit
extern fn sem_frame(intent: String, subject: String, obj: String, modifiers: String) -> Any
extern fn sem_frame_lang(intent: String, subject: String, obj: String, modifiers: String, lang_code: String) -> Any
extern fn sem_frame_simple(intent: String, subject: String) -> Any
extern fn sem_frame_obj(intent: String, subject: String, obj: String) -> Any
extern fn sem_intent(frame: Any) -> String
extern fn sem_subject(frame: Any) -> String
extern fn sem_object(frame: Any) -> String
extern fn sem_modifiers(frame: Any) -> String
extern fn sem_lang(frame: Any) -> String
extern fn sem_first_modifier(mods: String) -> String
extern fn sem_intent_to_realize(intent: String) -> String
extern fn sem_to_spec(frame: [String]) -> [String]
extern fn sem_to_spec_full(frame: [String], verb: String, tense: String, aspect: String) -> [String]
extern fn sem_to_spec(frame: Any) -> Any
extern fn sem_to_spec_full(frame: Any, verb: String, tense: String, aspect: String) -> Any
extern fn sem_realize_greet(subject: String) -> String
extern fn sem_realize(frame: [String]) -> String
extern fn sem_realize_full(frame: [String], verb: String, tense: String, aspect: String) -> String
extern fn sem_realize_lang(frame: [String], lang_code: String) -> String
extern fn sem_realize(frame: Any) -> String
extern fn sem_realize_full(frame: Any, verb: String, tense: String, aspect: String) -> String
extern fn sem_realize_lang(frame: Any, lang_code: String) -> String
+19 -19
View File
@@ -1,20 +1,20 @@
// auto-generated by elc --emit-header do not edit
extern fn lex_word(entry: [String]) -> String
extern fn lex_pos(entry: [String]) -> String
extern fn lex_form(entry: [String], idx: Int) -> String
extern fn lex_class(entry: [String]) -> String
extern fn make_entry(word: String, pos: String, f0: String, f1: String, f2: String, f3: String, f4: String, cls: String) -> [String]
extern fn make_entry2(word: String, pos: String, f0: String, f1: String, cls: String) -> [String]
extern fn make_entry3(word: String, pos: String, f0: String, f1: String, f2: String, cls: String) -> [String]
extern fn make_entry1(word: String, pos: String, f0: String, cls: String) -> [String]
extern fn build_vocab() -> [[String]]
extern fn get_vocab() -> [[String]]
extern fn vocab_lookup(word: String, lang_code: String) -> [String]
extern fn vocab_lookup_en(word: String) -> [String]
// auto-generated by elc --emit-header - do not edit
extern fn lex_word(entry: Any) -> String
extern fn lex_pos(entry: Any) -> String
extern fn lex_form(entry: Any, idx: Int) -> String
extern fn lex_class(entry: Any) -> String
extern fn make_entry(word: String, pos: String, f0: String, f1: String, f2: String, f3: String, f4: String, cls: String) -> Any
extern fn make_entry2(word: String, pos: String, f0: String, f1: String, cls: String) -> Any
extern fn make_entry3(word: String, pos: String, f0: String, f1: String, f2: String, cls: String) -> Any
extern fn make_entry1(word: String, pos: String, f0: String, cls: String) -> Any
extern fn build_vocab() -> Any
extern fn get_vocab() -> Any
extern fn vocab_lookup(word: String, lang_code: String) -> Any
extern fn vocab_lookup_en(word: String) -> Any
extern fn vocab_synonym(word: String, lang_register: String, lang_code: String) -> String
extern fn vocab_by_pos(pos: String) -> [[String]]
extern fn vocab_by_class(cls: String) -> [[String]]
extern fn entry_found(entry: [String]) -> Bool
extern fn entry_word(entry: [String]) -> String
extern fn entry_pos(entry: [String]) -> String
extern fn entry_form(entry: [String], n: Int) -> String
extern fn vocab_by_pos(pos: String) -> Any
extern fn vocab_by_class(cls: String) -> Any
extern fn entry_found(entry: Any) -> Bool
extern fn entry_word(entry: Any) -> String
extern fn entry_pos(entry: Any) -> String
extern fn entry_form(entry: Any, n: Int) -> String
+1 -1
View File
@@ -22,7 +22,7 @@ cd "$(dirname "$0")"
EL_HOME="${EL_HOME:-$(cd ../.. && pwd)/el}"
ELC="${ELC:-${EL_HOME}/dist/platform/elc}"
RUNTIME_DIR="${EL_HOME}/runtime"
RUNTIME_DIR="${EL_HOME}/el-compiler/runtime"
SRC_DIR="$(cd .. && pwd)/src"
if [ ! -x "${ELC}" ]; then
+1 -1
View File
@@ -81,7 +81,7 @@ jobs:
# Link to produce the engram binary
- name: Link engram binary
run: |
cc -std=c11 -O2 -DHAVE_CURL \
cc -std=c11 -O2 \
-I /usr/local/lib/el \
-o dist/engram \
dist/engram.c \
+1 -1
View File
@@ -88,7 +88,7 @@ jobs:
# Link to produce the engram binary
- name: Link engram binary
run: |
cc -std=c11 -O2 -DHAVE_CURL \
cc -std=c11 -O2 \
-I /usr/local/lib/el \
-o dist/engram \
dist/engram.c \
+1 -8
View File
@@ -49,12 +49,6 @@ jobs:
echo "Downloading el_runtime.h..."
curl -fsSL "${RELEASE_BASE}/el_runtime.h" -o /usr/local/lib/el/el_runtime.h
echo "Downloading engram_store.c..."
curl -fsSL "${RELEASE_BASE}/engram_store.c" -o /usr/local/lib/el/engram_store.c
echo "Downloading engram_store.h..."
curl -fsSL "${RELEASE_BASE}/engram_store.h" -o /usr/local/lib/el/engram_store.h
echo "El SDK installed:"
elc --version || true
@@ -68,12 +62,11 @@ jobs:
# Link to produce the engram binary
- name: Link engram binary
run: |
cc -std=c11 -O2 -DHAVE_CURL \
cc -std=c11 -O2 \
-I /usr/local/lib/el \
-o dist/engram \
dist/engram.c \
/usr/local/lib/el/el_runtime.c \
/usr/local/lib/el/engram_store.c \
-lcurl -lpthread
echo "Linked dist/engram"
ls -lh dist/engram
+2 -5
View File
@@ -1,6 +1,3 @@
.DS_Store
*.db
*.elc
*.elh
dist/
target/
*.db
.DS_Store
BIN
View File
Binary file not shown.
+75 -260
View File
@@ -10,9 +10,6 @@ el_val_t query_param(el_val_t path, el_val_t key);
el_val_t query_int(el_val_t path, el_val_t key, el_val_t default_val);
el_val_t extract_id(el_val_t path, el_val_t prefix);
el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_act_stats(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_text_health(el_val_t method, el_val_t path, el_val_t body);
el_val_t persist_canonical(void);
el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_get_node(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_scan_nodes(el_val_t method, el_val_t path, el_val_t body);
@@ -20,29 +17,19 @@ el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_search(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_create_edges_batch(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_save(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_load(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_health(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_embed_backfill(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_load_merge(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_capture_knowledge(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_similarity(el_val_t method, el_val_t path, el_val_t body);
el_val_t check_auth_ok(el_val_t method, el_val_t body);
el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body);
el_val_t bind_raw;
el_val_t bind_str;
el_val_t port;
el_val_t data_dir_raw;
el_val_t data_dir;
el_val_t snapshot_path;
el_val_t boot_snap;
el_val_t parse_port(el_val_t bind) {
el_val_t colon = str_index_of(bind, EL_STR(":"));
@@ -121,40 +108,17 @@ el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body) {
return 0;
}
el_val_t route_act_stats(el_val_t method, el_val_t path, el_val_t body) {
return engram_act_stats_json();
return 0;
}
el_val_t route_text_health(el_val_t method, el_val_t path, el_val_t body) {
return engram_text_health_json();
return 0;
}
el_val_t persist_canonical(void) {
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
el_val_t dir = ({ el_val_t _if_result_1 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_1 = (EL_STR("/tmp/engram")); } else { _if_result_1 = (dir_raw); } _if_result_1; });
return engram_save(el_str_concat(dir, EL_STR("/snapshot.json")));
return 0;
}
el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body) {
el_val_t content = json_get_string(body, EL_STR("content"));
el_val_t nt_raw = json_get_string(body, EL_STR("node_type"));
el_val_t node_type = ({ el_val_t _if_result_2 = 0; if (str_eq(nt_raw, EL_STR(""))) { _if_result_2 = (EL_STR("Memory")); } else { _if_result_2 = (nt_raw); } _if_result_2; });
el_val_t sal_present = json_get_raw(body, EL_STR("salience"));
el_val_t salience = ({ el_val_t _if_result_3 = 0; if (str_eq(sal_present, EL_STR(""))) { _if_result_3 = (el_from_float(0.5)); } else { _if_result_3 = (json_get_float(body, EL_STR("salience"))); } _if_result_3; });
el_val_t label_raw = json_get_string(body, EL_STR("label"));
el_val_t label = ({ el_val_t _if_result_4 = 0; if (str_eq(label_raw, EL_STR(""))) { _if_result_4 = (content); } else { _if_result_4 = (label_raw); } _if_result_4; });
el_val_t imp_present = json_get_raw(body, EL_STR("importance"));
el_val_t importance = ({ el_val_t _if_result_5 = 0; if (str_eq(imp_present, EL_STR(""))) { _if_result_5 = (el_from_float(0.5)); } else { _if_result_5 = (json_get_float(body, EL_STR("importance"))); } _if_result_5; });
el_val_t conf_present = json_get_raw(body, EL_STR("confidence"));
el_val_t confidence = ({ el_val_t _if_result_6 = 0; if (str_eq(conf_present, EL_STR(""))) { _if_result_6 = (el_from_float(1.0)); } else { _if_result_6 = (json_get_float(body, EL_STR("confidence"))); } _if_result_6; });
el_val_t tier_raw = json_get_string(body, EL_STR("tier"));
el_val_t tier = ({ el_val_t _if_result_7 = 0; if (str_eq(tier_raw, EL_STR(""))) { _if_result_7 = (EL_STR("Working")); } else { _if_result_7 = (tier_raw); } _if_result_7; });
el_val_t tags = json_get_string(body, EL_STR("tags"));
el_val_t id = engram_node_full(content, node_type, label, salience, importance, confidence, tier, tags);
el_val_t saved = persist_canonical();
el_val_t node_type = json_get_string(body, EL_STR("node_type"));
if (str_eq(node_type, EL_STR(""))) {
node_type = EL_STR("Memory");
}
el_val_t salience = json_get_float(body, EL_STR("salience"));
if (str_eq(salience, el_from_float(0.0))) {
salience = el_from_float(0.5);
}
el_val_t id = engram_node(content, node_type, salience);
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"id\":\""), id), EL_STR("\",\"content\":\"")), content), EL_STR("\",\"node_type\":\"")), node_type), EL_STR("\"}"));
return 0;
}
@@ -180,9 +144,11 @@ el_val_t route_scan_nodes(el_val_t method, el_val_t path, el_val_t body) {
}
el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body) {
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
el_val_t dir = ({ el_val_t _if_result_8 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_8 = (EL_STR("/tmp/engram")); } else { _if_result_8 = (dir_raw); } _if_result_8; });
el_val_t snap_path = el_str_concat(dir, EL_STR("/.scan-export.json"));
el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
if (str_eq(dir, EL_STR(""))) {
dir = EL_STR("/tmp/engram");
}
el_val_t snap_path = el_str_concat(dir, EL_STR("/snapshot.json"));
engram_save(snap_path);
el_val_t snap = fs_read(snap_path);
if (str_eq(snap, EL_STR(""))) {
@@ -197,22 +163,36 @@ el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body) {
}
el_val_t route_search(el_val_t method, el_val_t path, el_val_t body) {
el_val_t q = ({ el_val_t _if_result_9 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_9 = (query_param(path, EL_STR("q"))); } else { _if_result_9 = (json_get_string(body, EL_STR("query"))); } _if_result_9; });
el_val_t lim_url = query_int(path, EL_STR("limit"), 0);
el_val_t lim_body = json_get_int(body, EL_STR("limit"));
el_val_t lim_either = ({ el_val_t _if_result_10 = 0; if ((lim_url > 0)) { _if_result_10 = (lim_url); } else { _if_result_10 = (lim_body); } _if_result_10; });
el_val_t limit = ({ el_val_t _if_result_11 = 0; if ((lim_either > 0)) { _if_result_11 = (lim_either); } else { _if_result_11 = (20); } _if_result_11; });
el_val_t q = EL_STR("");
if (str_eq(method, EL_STR("GET"))) {
q = query_param(path, EL_STR("q"));
} else {
q = json_get_string(body, EL_STR("query"));
}
el_val_t limit = query_int(path, EL_STR("limit"), 20);
if (limit == 0) {
limit = json_get_int(body, EL_STR("limit"));
}
if (limit == 0) {
limit = 20;
}
return engram_search_json(q, limit);
return 0;
}
el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body) {
el_val_t q = ({ el_val_t _if_result_12 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_12 = (query_param(path, EL_STR("q"))); } else { _if_result_12 = (json_get_string(body, EL_STR("query"))); } _if_result_12; });
if (str_eq(q, EL_STR(""))) {
return err_json(EL_STR("missing query"));
el_val_t q = EL_STR("");
el_val_t depth = 3;
if (str_eq(method, EL_STR("GET"))) {
q = query_param(path, EL_STR("q"));
depth = query_int(path, EL_STR("depth"), 3);
} else {
q = json_get_string(body, EL_STR("query"));
el_val_t bd = json_get_int(body, EL_STR("depth"));
if (bd > 0) {
depth = bd;
}
}
el_val_t d_raw = ({ el_val_t _if_result_13 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_13 = (query_int(path, EL_STR("depth"), 3)); } else { _if_result_13 = (json_get_int(body, EL_STR("depth"))); } _if_result_13; });
el_val_t depth = ({ el_val_t _if_result_14 = 0; if ((d_raw > 0)) { _if_result_14 = (d_raw); } else { _if_result_14 = (3); } _if_result_14; });
return el_str_concat(el_str_concat(EL_STR("{\"results\":"), engram_activate_json(q, depth)), EL_STR("}"));
return 0;
}
@@ -220,51 +200,19 @@ el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body) {
el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body) {
el_val_t from_id = json_get_string(body, EL_STR("from_id"));
el_val_t to_id = json_get_string(body, EL_STR("to_id"));
el_val_t rel_raw = json_get_string(body, EL_STR("relation"));
el_val_t relation = ({ el_val_t _if_result_15 = 0; if (str_eq(rel_raw, EL_STR(""))) { _if_result_15 = (EL_STR("associates")); } else { _if_result_15 = (rel_raw); } _if_result_15; });
el_val_t w_present = json_get_raw(body, EL_STR("weight"));
el_val_t weight = ({ el_val_t _if_result_16 = 0; if (str_eq(w_present, EL_STR(""))) { _if_result_16 = (el_from_float(0.5)); } else { _if_result_16 = (json_get_float(body, EL_STR("weight"))); } _if_result_16; });
el_val_t relation = json_get_string(body, EL_STR("relation"));
if (str_eq(relation, EL_STR(""))) {
relation = EL_STR("associates");
}
el_val_t weight = json_get_float(body, EL_STR("weight"));
if (str_eq(weight, el_from_float(0.0))) {
weight = el_from_float(0.5);
}
engram_connect(from_id, to_id, weight, relation);
el_val_t saved = persist_canonical();
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"from_id\":\""), from_id), EL_STR("\",\"to_id\":\"")), to_id), EL_STR("\",\"relation\":\"")), relation), EL_STR("\"}"));
return 0;
}
el_val_t route_create_edges_batch(el_val_t method, el_val_t path, el_val_t body) {
el_val_t arr = json_get_raw(body, EL_STR("edges"));
if (str_eq(arr, EL_STR(""))) {
return err_json(EL_STR("missing edges array"));
}
el_val_t n = json_array_len(arr);
if (n == 0) {
return EL_STR("{\"ok\":true,\"accepted\":0,\"skipped\":0}");
}
el_val_t i = 0;
el_val_t accepted = 0;
el_val_t skipped = 0;
while (i < n) {
el_val_t item = json_array_get(arr, i);
el_val_t from_id = json_get_string(item, EL_STR("from_id"));
el_val_t to_id = json_get_string(item, EL_STR("to_id"));
if (str_eq(from_id, EL_STR("")) || str_eq(to_id, EL_STR(""))) {
skipped = (skipped + 1);
} else {
el_val_t rel_raw = json_get_string(item, EL_STR("relation"));
el_val_t relation = ({ el_val_t _if_result_17 = 0; if (str_eq(rel_raw, EL_STR(""))) { _if_result_17 = (EL_STR("associates")); } else { _if_result_17 = (rel_raw); } _if_result_17; });
el_val_t w_present = json_get_raw(item, EL_STR("weight"));
el_val_t weight = ({ el_val_t _if_result_18 = 0; if (str_eq(w_present, EL_STR(""))) { _if_result_18 = (el_from_float(0.5)); } else { _if_result_18 = (json_get_float(item, EL_STR("weight"))); } _if_result_18; });
engram_connect(from_id, to_id, weight, relation);
accepted = (accepted + 1);
}
i = (i + 1);
}
if (accepted > 0) {
el_val_t saved = persist_canonical();
}
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"accepted\":"), int_to_str(accepted)), EL_STR(",\"skipped\":")), int_to_str(skipped)), EL_STR("}"));
return 0;
}
el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body) {
el_val_t id = extract_id(path, EL_STR("/api/neighbors/"));
if (str_eq(id, EL_STR(""))) {
@@ -281,7 +229,6 @@ el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body) {
return err_json(EL_STR("missing node_id"));
}
engram_strengthen(id);
el_val_t saved = persist_canonical();
return ok_json();
return 0;
}
@@ -292,140 +239,40 @@ el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body) {
return err_json(EL_STR("missing id"));
}
engram_forget(id);
el_val_t saved = persist_canonical();
return ok_json();
return 0;
}
el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) {
el_val_t p_raw = json_get_string(body, EL_STR("path"));
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
el_val_t dir = ({ el_val_t _if_result_19 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_19 = (EL_STR("/tmp/engram")); } else { _if_result_19 = (dir_raw); } _if_result_19; });
el_val_t p = ({ el_val_t _if_result_20 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_20 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_20 = (p_raw); } _if_result_20; });
el_val_t sv = engram_save(p);
el_val_t sv_ok = ({ el_val_t _if_result_21 = 0; if ((sv == 0)) { _if_result_21 = (EL_STR("false")); } else { _if_result_21 = (EL_STR("true")); } _if_result_21; });
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":"), sv_ok), EL_STR(",\"path\":\"")), p), EL_STR("\",\"node_count\":")), int_to_str(engram_node_count())), EL_STR(",\"edge_count\":")), int_to_str(engram_edge_count())), EL_STR("}"));
el_val_t p = json_get_string(body, EL_STR("path"));
if (str_eq(p, EL_STR(""))) {
el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
if (str_eq(dir, EL_STR(""))) {
dir = EL_STR("/tmp/engram");
}
p = el_str_concat(dir, EL_STR("/snapshot.json"));
}
engram_save(p);
return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"path\":\""), p), EL_STR("\"}"));
return 0;
}
el_val_t route_load(el_val_t method, el_val_t path, el_val_t body) {
el_val_t p_raw = json_get_string(body, EL_STR("path"));
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
el_val_t dir = ({ el_val_t _if_result_22 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_22 = (EL_STR("/tmp/engram")); } else { _if_result_22 = (dir_raw); } _if_result_22; });
el_val_t p = ({ el_val_t _if_result_23 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_23 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_23 = (p_raw); } _if_result_23; });
el_val_t ld = engram_load(p);
el_val_t ld_ok = ({ el_val_t _if_result_24 = 0; if ((ld == 0)) { _if_result_24 = (EL_STR("false")); } else { _if_result_24 = (EL_STR("true")); } _if_result_24; });
el_val_t nc_after = engram_node_count();
el_val_t hollow = ({ el_val_t _if_result_25 = 0; if ((nc_after == 0)) { _if_result_25 = (EL_STR("true")); } else { _if_result_25 = (EL_STR("false")); } _if_result_25; });
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":"), ld_ok), EL_STR(",\"path\":\"")), p), EL_STR("\",\"node_count\":")), int_to_str(nc_after)), EL_STR(",\"edge_count\":")), int_to_str(engram_edge_count())), EL_STR(",\"hollow\":")), hollow), EL_STR("}"));
el_val_t p = json_get_string(body, EL_STR("path"));
if (str_eq(p, EL_STR(""))) {
el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
if (str_eq(dir, EL_STR(""))) {
dir = EL_STR("/tmp/engram");
}
p = el_str_concat(dir, EL_STR("/snapshot.json"));
}
engram_load(p);
return ok_json();
return 0;
}
el_val_t route_health(el_val_t method, el_val_t path, el_val_t body) {
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"status\":\"ok\",\"engine\":\"engram-runtime-native\",\"node_count\":"), int_to_str(engram_node_count())), EL_STR(",\"edge_count\":")), int_to_str(engram_edge_count())), EL_STR("}"));
return 0;
}
el_val_t route_embed_backfill(el_val_t method, el_val_t path, el_val_t body) {
el_val_t n = query_int(path, EL_STR("n"), 32);
el_val_t result = engram_embed_backfill(n);
el_val_t done = json_get_float(result, EL_STR("embedded"));
if (done > el_from_float(0.0)) {
el_val_t saved = persist_canonical();
}
return result;
return 0;
}
el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) {
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
el_val_t dir = ({ el_val_t _if_result_26 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_26 = (EL_STR("/tmp/engram")); } else { _if_result_26 = (dir_raw); } _if_result_26; });
el_val_t snap_path = el_str_concat(dir, EL_STR("/.sync-export.json"));
engram_save(snap_path);
el_val_t snap = fs_read(snap_path);
if (str_eq(snap, EL_STR(""))) {
return err_json(EL_STR("sync export failed: snapshot unreadable"));
}
return snap;
return 0;
}
el_val_t route_load_merge(el_val_t method, el_val_t path, el_val_t body) {
el_val_t p = json_get_string(body, EL_STR("path"));
if (str_eq(p, EL_STR(""))) {
return err_json(EL_STR("path is required"));
}
if (str_eq(fs_read(p), EL_STR(""))) {
return err_json(EL_STR("file missing or empty"));
}
el_val_t before_n = engram_node_count();
el_val_t before_e = engram_edge_count();
engram_load_merge(p);
el_val_t added_n = (engram_node_count() - before_n);
el_val_t added_e = (engram_edge_count() - before_e);
el_val_t saved = persist_canonical();
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"nodes_added\":"), int_to_str(added_n)), EL_STR(",\"edges_added\":")), int_to_str(added_e)), EL_STR(",\"node_count\":")), int_to_str(engram_node_count())), EL_STR("}"));
return 0;
}
el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body) {
el_val_t content = json_get_string(body, EL_STR("content"));
if (str_eq(content, EL_STR(""))) {
return err_json(EL_STR("missing content"));
}
el_val_t sal = el_from_float(0.3);
el_val_t imp = el_from_float(0.3);
el_val_t conf = el_from_float(0.8);
el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), EL_STR("state-event"), sal, imp, conf, EL_STR("Episodic"), EL_STR("[\"internal-state\",\"InternalStateEvent\"]"));
el_val_t ret_raw = env(EL_STR("ENGRAM_ISE_RETENTION_MS"));
el_val_t ret_ms = ({ el_val_t _if_result_27 = 0; if (str_eq(ret_raw, EL_STR(""))) { _if_result_27 = (172800000); } else { _if_result_27 = (str_to_int(ret_raw)); } _if_result_27; });
el_val_t pruned = engram_prune_telemetry(ret_ms);
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\",\"pruned\":")), int_to_str(pruned)), EL_STR("}"));
return 0;
}
el_val_t route_capture_knowledge(el_val_t method, el_val_t path, el_val_t body) {
el_val_t content = json_get_string(body, EL_STR("content"));
if (str_eq(content, EL_STR(""))) {
return err_json(EL_STR("missing content"));
}
el_val_t title = json_get_string(body, EL_STR("title"));
el_val_t label = ({ el_val_t _if_result_28 = 0; if (str_eq(title, EL_STR(""))) { _if_result_28 = (str_slice(content, 0, 60)); } else { _if_result_28 = (title); } _if_result_28; });
el_val_t category_raw = json_get_string(body, EL_STR("category"));
el_val_t category = ({ el_val_t _if_result_29 = 0; if (str_eq(category_raw, EL_STR(""))) { _if_result_29 = (EL_STR("other")); } else { _if_result_29 = (category_raw); } _if_result_29; });
el_val_t ktier_raw = json_get_string(body, EL_STR("tier"));
el_val_t ktier = ({ el_val_t _if_result_30 = 0; if (str_eq(ktier_raw, EL_STR(""))) { _if_result_30 = (EL_STR("note")); } else { _if_result_30 = (ktier_raw); } _if_result_30; });
el_val_t project = json_get_string(body, EL_STR("project"));
el_val_t tags_raw = json_get_raw(body, EL_STR("tags"));
el_val_t tags_base = ({ el_val_t _if_result_31 = 0; if (str_eq(tags_raw, EL_STR(""))) { _if_result_31 = (EL_STR("[]")); } else { _if_result_31 = (tags_raw); } _if_result_31; });
el_val_t base_len = str_len(tags_base);
el_val_t head = str_slice(tags_base, 0, (base_len - 1));
el_val_t sep = ({ el_val_t _if_result_32 = 0; if (str_eq(head, EL_STR("["))) { _if_result_32 = (EL_STR("")); } else { _if_result_32 = (EL_STR(",")); } _if_result_32; });
el_val_t safe_cat = str_replace(category, EL_STR("\""), EL_STR("'"));
el_val_t safe_tier = str_replace(ktier, EL_STR("\""), EL_STR("'"));
el_val_t safe_proj = str_replace(project, EL_STR("\""), EL_STR("'"));
el_val_t proj_tag = ({ el_val_t _if_result_33 = 0; if (str_eq(safe_proj, EL_STR(""))) { _if_result_33 = (EL_STR("")); } else { _if_result_33 = (el_str_concat(el_str_concat(EL_STR(",\"project:"), safe_proj), EL_STR("\""))); } _if_result_33; });
el_val_t tags = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(head, sep), EL_STR("\"category:")), safe_cat), EL_STR("\",\"tier:")), safe_tier), EL_STR("\"")), proj_tag), EL_STR("]"));
el_val_t sal = el_from_float(0.5);
el_val_t imp = el_from_float(0.5);
el_val_t conf = el_from_float(0.9);
el_val_t id = engram_node_full(content, EL_STR("Knowledge"), label, sal, imp, conf, EL_STR("Semantic"), tags);
el_val_t saved = persist_canonical();
return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\"}"));
return 0;
}
el_val_t route_similarity(el_val_t method, el_val_t path, el_val_t body) {
el_val_t a = query_param(path, EL_STR("a"));
el_val_t b = query_param(path, EL_STR("b"));
if (str_eq(a, EL_STR(""))) {
return err_json(EL_STR("missing a"));
}
if (str_eq(b, EL_STR(""))) {
return err_json(EL_STR("missing b"));
}
el_val_t sim = engram_cosine_sim(a, b);
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"a\":\""), a), EL_STR("\",\"b\":\"")), b), EL_STR("\",\"cosine\":")), float_to_str(sim)), EL_STR("}"));
return EL_STR("{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}");
return 0;
}
@@ -452,24 +299,12 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
return route_health(method, path, body);
}
}
if (str_eq(method, EL_STR("POST")) && str_eq(clean, EL_STR("/api/neuron/state-events"))) {
return route_emit_ise(method, path, body);
}
if (!check_auth_ok(method, body)) {
return err_json(EL_STR("unauthorized"));
}
if (str_eq(method, EL_STR("POST")) && str_eq(clean, EL_STR("/api/neuron/knowledge/capture"))) {
return route_capture_knowledge(method, path, body);
}
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/stats")) || str_eq(clean, EL_STR("/stats")))) {
return route_stats(method, path, body);
}
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/act-stats")) || str_eq(clean, EL_STR("/act-stats")))) {
return route_act_stats(method, path, body);
}
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/text-health")) || str_eq(clean, EL_STR("/text-health")))) {
return route_text_health(method, path, body);
}
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/nodes")) || str_eq(clean, EL_STR("/nodes")))) {
return route_create_node(method, path, body);
}
@@ -488,9 +323,6 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/edges")) || str_eq(clean, EL_STR("/edges")))) {
return route_create_edge(method, path, body);
}
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/edges/batch")) || str_eq(clean, EL_STR("/edges/batch")))) {
return route_create_edges_batch(method, path, body);
}
if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/neighbors/"))) {
return route_neighbors(method, path, body);
}
@@ -515,40 +347,23 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/load")) || str_eq(clean, EL_STR("/load")))) {
return route_load(method, path, body);
}
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/load-merge")) || str_eq(clean, EL_STR("/load-merge")))) {
return route_load_merge(method, path, body);
}
if (str_eq(method, EL_STR("GET")) && str_eq(clean, EL_STR("/api/sync"))) {
return route_sync(method, path, body);
}
if (str_eq(clean, EL_STR("/api/embed-backfill"))) {
return route_embed_backfill(method, path, body);
}
if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/similarity"))) {
return route_similarity(method, path, body);
}
return el_str_concat(el_str_concat(EL_STR("{\"error\":\"not found\",\"path\":\""), clean), EL_STR("\"}"));
return 0;
}
int main(int _argc, char** _argv) {
el_runtime_init_args(_argc, _argv);
bind_raw = env(EL_STR("ENGRAM_BIND"));
bind_str = ({ el_val_t _if_result_34 = 0; if (str_eq(bind_raw, EL_STR(""))) { _if_result_34 = (EL_STR(":8742")); } else { _if_result_34 = (bind_raw); } _if_result_34; });
bind_str = env(EL_STR("ENGRAM_BIND"));
if (str_eq(bind_str, EL_STR(""))) {
bind_str = EL_STR(":8742");
}
port = parse_port(bind_str);
data_dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
data_dir = ({ el_val_t _if_result_35 = 0; if (str_eq(data_dir_raw, EL_STR(""))) { _if_result_35 = (EL_STR("/tmp/engram")); } else { _if_result_35 = (data_dir_raw); } _if_result_35; });
data_dir = env(EL_STR("ENGRAM_DATA_DIR"));
if (str_eq(data_dir, EL_STR(""))) {
data_dir = EL_STR("/tmp/engram");
}
snapshot_path = el_str_concat(data_dir, EL_STR("/snapshot.json"));
engram_load(snapshot_path);
boot_snap = fs_read(snapshot_path);
if (!str_eq(boot_snap, EL_STR(""))) {
if (engram_node_count() == 0) {
println(EL_STR("[engram] WARNING: snapshot.json is non-empty but load produced 0 nodes \xe2\x80\x94 preserving copy at snapshot.failed-load.json"));
fs_write(el_str_concat(data_dir, EL_STR("/snapshot.failed-load.json")), boot_snap);
} else {
fs_write(el_str_concat(data_dir, EL_STR("/snapshot.boot-backup.json")), boot_snap);
}
}
println(EL_STR("[engram] runtime-native graph engine"));
println(el_str_concat(EL_STR("[engram] data_dir="), data_dir));
println(el_str_concat(EL_STR("[engram] node_count="), int_to_str(engram_node_count())));
+58 -603
View File
@@ -76,182 +76,13 @@ fn route_stats(method: String, path: String, body: String) -> String {
engram_stats_json()
}
// route_act_stats GET /api/act-stats
// (2026-08-04 self-review) engram_act_stats_json() has existed since the
// 2026-07-27 review but was reachable ONLY through the soul daemon's heartbeat
// binding. Every activation-layer gauge WM evictions, breakthroughs, embedder
// breaker state, context drift, and now the Hebbian counters was therefore
// invisible unless the soul happened to be running and its ISEs were read back
// out of the store. Diagnosing the activation layer required a working soul,
// which is exactly backwards: the lower layer should be observable on its own.
// This review needed it to verify link formation and could not get at it. One
// line of plumbing, and the whole activation layer becomes directly diagnosable.
fn route_act_stats(method: String, path: String, body: String) -> String {
engram_act_stats_json()
}
// route_text_health GET /api/text-health
// (2026-08-08 self-review) The daily census half of the text-integrity gauge.
// Today's review found that the JSON parser had been replacing every \uXXXX
// escape with a literal '?' for at least two months: 3,119 of 4,081
// non-telemetry nodes (76%) were damaged, including the self traversal root
// and every values node, and NOTHING detected it because every gauge in the
// system measured whether the machinery was running, and none measured whether
// the text it carried was intact. No snapshot on disk predates the damage, so
// it cannot be undone; it can only be made impossible to repeat quietly.
//
// The parser is fixed. This route is the standing check: `damaged` should now
// hold flat at its historical floor and never climb. `write_damaged` (also on
// the heartbeat as txt_damaged) is the live regression signal non-zero means
// a write path is mangling text right now.
fn route_text_health(method: String, path: String, body: String) -> String {
engram_text_health_json()
}
// (2026-07-18 self-review) Scoping sweep: `let` inside an if-block creates an
// inner scope only it does NOT mutate the outer binding (documented with
// evidence in awareness.el, 2026-05-25). Every default/reassignment below used
// that broken pattern, so defaults never applied: nodes were created with
// node_type="" and salience=0.0, /api/search and /api/activate ALWAYS ran with
// q="" regardless of input, edges defaulted to relation=""/weight=0.0, and
// save/load with no "path" hit engram_save(""). Rewritten to the
// `let x = if cond { a } else { b }` expression form (the pattern the newer
// routes route_emit_ise/route_capture_knowledge already use correctly).
// store_on ENGRAM_STORE flag (tiered paged store as the durable owner). Matches
// engram_store_enabled() in el_runtime.c EXACTLY (1 / on / true). Default off
// every persistence path below is byte-for-byte the historical snapshot behavior.
fn store_on() -> Bool {
let v: String = env("ENGRAM_STORE")
if str_eq(v, "1") { return true }
if str_eq(v, "on") { return true }
if str_eq(v, "true") { return true }
return false
}
// persist_canonical save the canonical snapshot after a durable write.
//
// WHY (2026-07-22 self-review): the 2026-07-21 fix correctly stopped READ
// routes from writing the canonical snapshot.json but nothing was left
// that saved it on WRITE. Every mutation (node create, edge create,
// knowledge capture, forget, merge) lived only in RAM until someone POSTed
// /api/save manually; a process restart silently discarded everything since
// the last manual save. Observed live: two engram restarts during the
// 2026-07-22 review reverted the store to a ~17h-old snapshot, destroying
// same-day writes. Reads must never write the canonical; writes must always
// persist it. ISE telemetry is deliberately excluded (48h-pruned, loss-
// tolerant, ~2/min snapshotting the whole store per heartbeat is waste;
// any durable write that follows persists the pruning too).
fn persist_canonical() -> Int {
// ENGRAM_STORE: the paged store is the durable owner a checkpoint flushes
// dirty pages behind a WAL-durable record (durable the moment the WAL fsyncs).
// This is the fix for the "restart reverted to a 17h-old snapshot" data loss:
// durable writes no longer depend on a full snapshot.json rewrite. Returns 1
// on a successful checkpoint, 0 otherwise. Flag-off: unchanged (writes JSON).
if store_on() {
return engram_store_checkpoint()
}
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = engram_resolve_data_dir()
// (2026-08-10 self-review) This returned a hardcoded 1, which made every
// caller's `let saved: Int = persist_canonical()` a dead variable six
// durable write paths each believed they had confirmation of a successful
// canonical persist and none of them had any. Propagate the real result.
return engram_save(dir + "/snapshot.json")
}
// WAL persistence (design doc §§3-14; gated behind ENGRAM_WAL=on) ──────────
// Default OFF every persist path below is byte-identical to the historical
// per-write full-snapshot behavior. When ON, structural mutations append O(1)
// WAL records instead of rewriting the whole graph, with threshold compaction.
fn wal_on() -> Bool {
str_eq(env("ENGRAM_WAL"), "on")
}
// Persist a single-node mutation (create / content-evolve / strengthen).
fn persist_node(id: String) -> Int {
if wal_on() {
let d: String = engram_resolve_data_dir()
let a: Int = engram_wal_node_put(d, id)
let c: Int = engram_wal_maybe_compact(d)
return a
}
return persist_canonical()
}
// Persist edges appended at index >= start (covers single-edge and batch).
fn persist_edges_since(start: Int) -> Int {
if wal_on() {
let d: String = engram_resolve_data_dir()
let a: Int = engram_wal_edges_since(d, start)
let c: Int = engram_wal_maybe_compact(d)
return a
}
return persist_canonical()
}
// Persist a Hebbian consolidation batch as ONE WAL record (single fsync, §5-B).
fn persist_hebb_batch(start: Int) -> Int {
if wal_on() {
let d: String = engram_resolve_data_dir()
let a: Int = engram_wal_hebb_batch(d, start)
let c: Int = engram_wal_maybe_compact(d)
return a
}
return persist_canonical()
}
// Bulk mutation (embedding backfill, load-merge): write a fresh compaction base
// so the many-node change is durable in one atomic snapshot; WAL is truncated.
fn persist_bulk() -> Int {
if wal_on() {
let d: String = engram_resolve_data_dir()
return engram_wal_compact(d)
}
return persist_canonical()
}
// INCOMPLETE-ROUTE FIX (2026-07-24 self-review): this route silently dropped
// label, importance, tier, and tags engram_node() defaults label to content
// and importance to 0.5, so every node created over HTTP lost its metadata.
// Observed live: the soul's boot-counter write-back landed with
// label="soul:boot_count:99" (content), importance 0.5, no tags. Honor the
// full field set via engram_node_full when any of them is supplied.
// PRESENCE-AWARE DEFAULTS (2026-08-01 self-review): the old pattern
// `if x == 0.0 { default }` made a legitimate 0.0 unrepresentable a caller
// setting salience/importance/weight to zero silently got 0.5. json_get_raw
// returns "" when the key is ABSENT and the raw token when present, so
// absence and zero are now distinguishable. Also: confidence was hardcoded
// to 1.0 regardless of input every HTTP-created node claimed full
// epistemic confidence. Now honored from the payload (default 1.0).
fn route_create_node(method: String, path: String, body: String) -> String {
let content: String = json_get_string(body, "content")
let nt_raw: String = json_get_string(body, "node_type")
let node_type: String = if str_eq(nt_raw, "") { "Memory" } else { nt_raw }
let sal_present: String = json_get_raw(body, "salience")
let salience: Float = if str_eq(sal_present, "") { 0.5 } else { json_get_float(body, "salience") }
let label_raw: String = json_get_string(body, "label")
let label: String = if str_eq(label_raw, "") { content } else { label_raw }
let imp_present: String = json_get_raw(body, "importance")
let importance: Float = if str_eq(imp_present, "") { 0.5 } else { json_get_float(body, "importance") }
let conf_present: String = json_get_raw(body, "confidence")
let confidence: Float = if str_eq(conf_present, "") { 1.0 } else { json_get_float(body, "confidence") }
let tier_raw: String = json_get_string(body, "tier")
let tier: String = if str_eq(tier_raw, "") { "Working" } else { tier_raw }
let tags: String = json_get_string(body, "tags")
// NO el_from_float WRAPPER (2026-08-01 self-review): salience/importance/
// confidence are already Float (el_val_t) values json_get_float and
// Float literals both encode. Wrapping them in el_from_float AGAIN
// reinterpreted the boxed bits as a raw double, producing garbage that
// failed engram_decode_score's range check and clamped every HTTP-created
// node to defaults (salience 0.9 in 0.5 stored; confidence 0.6 in → 1.0
// stored verified live). route_emit_ise always passed Floats bare and
// its 0.3/0.3/0.8 stored correctly; this call now does the same.
let id: String = engram_node_full(
content, node_type, label,
salience, importance, confidence,
tier, tags
)
let saved: Int = persist_node(id)
let node_type: String = json_get_string(body, "node_type")
if str_eq(node_type, "") { let node_type = "Memory" }
let salience: Float = json_get_float(body, "salience")
if salience == 0.0 { let salience = 0.5 }
let id: String = engram_node(content, node_type, salience)
"{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\"}"
}
@@ -272,14 +103,13 @@ fn route_scan_nodes(method: String, path: String, body: String) -> String {
}
// route_scan_edges bulk export of all edges as a JSON array. Implemented
// via engram_save fs_read of a SCRATCH export path. (2026-07-21 self-review:
// previously this saved over the canonical snapshot.json on every GET if the
// process ever booted with a partial/empty store, the first read request
// clobbered the good snapshot. Read routes must never write the canonical path.)
// via engram_save fs_read of the canonical on-disk snapshot, which the
// runtime keeps in lockstep with the in-memory graph. Live against the
// running graph, not a stale export.
fn route_scan_edges(method: String, path: String, body: String) -> String {
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = engram_resolve_data_dir()
let snap_path: String = dir + "/.scan-export.json"
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let snap_path: String = dir + "/snapshot.json"
engram_save(snap_path)
let snap: String = fs_read(snap_path)
if str_eq(snap, "") { return "[]" }
@@ -292,90 +122,43 @@ fn route_scan_edges(method: String, path: String, body: String) -> String {
}
fn route_search(method: String, path: String, body: String) -> String {
let q: String = if str_eq(method, "GET") { query_param(path, "q") } else { json_get_string(body, "query") }
let lim_url: Int = query_int(path, "limit", 0)
let lim_body: Int = json_get_int(body, "limit")
let lim_either: Int = if lim_url > 0 { lim_url } else { lim_body }
let limit: Int = if lim_either > 0 { lim_either } else { 20 }
let q: String = ""
if str_eq(method, "GET") {
let q = query_param(path, "q")
} else {
let q = json_get_string(body, "query")
}
let limit: Int = query_int(path, "limit", 20)
if limit == 0 { let limit = json_get_int(body, "limit") }
if limit == 0 { let limit = 20 }
return engram_search_json(q, limit)
}
fn route_activate(method: String, path: String, body: String) -> String {
let q: String = if str_eq(method, "GET") { query_param(path, "q") } else { json_get_string(body, "query") }
// Guard: engram_activate with an empty query matches zero seeds, which
// zeroes ALL carried working-memory weights (documented in awareness.el
// perceive()). Never let an empty activation through to wipe WM.
if str_eq(q, "") { return err_json("missing query") }
let d_raw: Int = if str_eq(method, "GET") { query_int(path, "depth", 3) } else { json_get_int(body, "depth") }
let depth: Int = if d_raw > 0 { d_raw } else { 3 }
let q: String = ""
let depth: Int = 3
if str_eq(method, "GET") {
let q = query_param(path, "q")
let depth = query_int(path, "depth", 3)
} else {
let q = json_get_string(body, "query")
let bd: Int = json_get_int(body, "depth")
if bd > 0 { let depth = bd }
}
return "{\"results\":" + engram_activate_json(q, depth) + "}"
}
fn route_create_edge(method: String, path: String, body: String) -> String {
let from_id: String = json_get_string(body, "from_id")
let to_id: String = json_get_string(body, "to_id")
let rel_raw: String = json_get_string(body, "relation")
let relation: String = if str_eq(rel_raw, "") { "associates" } else { rel_raw }
// Presence-aware (2026-08-01): weight 0.0 is a legitimate edge weight
// (dormant association); only default when the key is absent.
let w_present: String = json_get_raw(body, "weight")
let weight: Float = if str_eq(w_present, "") { 0.5 } else { json_get_float(body, "weight") }
let ec0: Int = engram_edge_count()
let relation: String = json_get_string(body, "relation")
if str_eq(relation, "") { let relation = "associates" }
let weight: Float = json_get_float(body, "weight")
if weight == 0.0 { let weight = 0.5 }
engram_connect(from_id, to_id, weight, relation)
let saved: Int = persist_edges_since(ec0)
"{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}"
}
// route_create_edges_batch POST /api/edges/batch {"edges":[{from_id,to_id,relation,weight}, ...]}
//
// WHY THIS EXISTS (2026-08-07 self-review). persist_canonical() writes the
// FULL canonical snapshot 60MB at current graph size and route_create_edge
// calls it once per edge. That is correct for the interactive one-edge case and
// ruinous for any bulk write: the soul's Hebbian consolidation path delivers
// ~14 associations per 8-minute heartbeat, which through the single-edge route
// would be ~840MB of disk writes per beat, ~150GB/day, to persist 14 edges.
//
// The fix is not to weaken durability it is to make the unit of durability
// the BATCH. Connect every edge, then snapshot exactly once. Same guarantee
// (nothing acknowledged is lost to a restart), 1/N the writes. Empty or
// malformed entries are skipped rather than aborting the batch: a consolidation
// payload is best-effort by design, and one bad id should not cost the other 13.
//
// Returns the accepted count so the caller can tell delivery from silence.
fn route_create_edges_batch(method: String, path: String, body: String) -> String {
let arr: String = json_get_raw(body, "edges")
if str_eq(arr, "") { return err_json("missing edges array") }
let n: Int = json_array_len(arr)
if n == 0 { return "{\"ok\":true,\"accepted\":0,\"skipped\":0}" }
let ec0: Int = engram_edge_count()
let i: Int = 0
let accepted: Int = 0
let skipped: Int = 0
while i < n {
let item: String = json_array_get(arr, i)
let from_id: String = json_get_string(item, "from_id")
let to_id: String = json_get_string(item, "to_id")
if str_eq(from_id, "") || str_eq(to_id, "") {
let skipped = skipped + 1
} else {
let rel_raw: String = json_get_string(item, "relation")
let relation: String = if str_eq(rel_raw, "") { "associates" } else { rel_raw }
let w_present: String = json_get_raw(item, "weight")
let weight: Float = if str_eq(w_present, "") { 0.5 } else { json_get_float(item, "weight") }
engram_connect(from_id, to_id, weight, relation)
let accepted = accepted + 1
}
let i = i + 1
}
// ONE snapshot for the whole batch the entire point of this route.
// Skip it when nothing was accepted: an all-malformed payload must not
// trigger a 60MB write.
if accepted > 0 {
let saved: Int = persist_hebb_batch(ec0)
}
return "{\"ok\":true,\"accepted\":" + int_to_str(accepted) + ",\"skipped\":" + int_to_str(skipped) + "}"
}
fn route_neighbors(method: String, path: String, body: String) -> String {
let id: String = extract_id(path, "/api/neighbors/")
if str_eq(id, "") { return err_json("missing id") }
@@ -387,290 +170,40 @@ fn route_strengthen(method: String, path: String, body: String) -> String {
let id: String = json_get_string(body, "node_id")
if str_eq(id, "") { return err_json("missing node_id") }
engram_strengthen(id)
let saved: Int = persist_node(id)
ok_json()
}
// route_forget DELETE /api/nodes/:id INTEGRITY HARDENED (design doc §18.1).
//
// Two invariants now enforced AT THE STORE (not one layer up in neuron-api.el,
// which a direct HTTP client could bypass):
// 1. Write-protection: protected identity/value nodes (derived from the self
// graph self root + values hub + their neighbors, §18.3) cannot be
// deleted over HTTP. Returns 403, node untouched.
// 2. No hard delete over the wire, ever: an ordinary delete creates a
// Tombstone marker node + `tombstones` edge and KEEPS the original node
// and its edges (recoverable), instead of the old destructive
// engram_forget() shift-delete. Raw engram_forget is now internal-GC only
// and no longer reachable from any HTTP route.
fn route_forget(method: String, path: String, body: String) -> String {
let id: String = extract_id(path, "/api/nodes/")
if str_eq(id, "") { return err_json("missing id") }
if engram_is_protected(id) == 1 {
return "{\"__status__\":403,\"error\":\"protected node; deletion refused\",\"id\":\"" + id + "\"}"
}
let tomb_id: String = engram_node_full(
"tombstone:" + id, "Tombstone", "tombstone:" + id,
0.1, 0.1, 1.0, "Episodic", "[\"tombstone\"]"
)
let ec0: Int = engram_edge_count()
engram_connect(tomb_id, id, 1.0, "tombstones")
let saved: Int = if wal_on() {
let d: String = engram_resolve_data_dir()
let a: Int = engram_wal_node_put(d, tomb_id)
let b: Int = engram_wal_edges_since(d, ec0)
let c: Int = engram_wal_maybe_compact(d)
a
} else {
persist_canonical()
}
"{\"ok\":true,\"tombstoned\":\"" + id + "\",\"tombstone_id\":\"" + tomb_id + "\"}"
engram_forget(id)
ok_json()
}
fn route_save(method: String, path: String, body: String) -> String {
let p_raw: String = json_get_string(body, "path")
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = engram_resolve_data_dir()
let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
// (2026-08-10 self-review) engram_save returns 0 on an empty path and the
// route discarded it, so the response was a literal "ok":true regardless
// of whether anything was written. Report the actual result AND the counts
// that were supposed to have been written the same move that made
// route_health honest on 2026-08-01. A caller can now tell "saved 13k
// nodes" from "saved nothing and said ok".
let sv: Int = engram_save(p)
let sv_ok: String = if sv == 0 { "false" } else { "true" }
"{\"ok\":" + sv_ok + ",\"path\":\"" + p + "\",\"node_count\":" + int_to_str(engram_node_count()) + ",\"edge_count\":" + int_to_str(engram_edge_count()) + "}"
let p: String = json_get_string(body, "path")
if str_eq(p, "") {
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let p = dir + "/snapshot.json"
}
engram_save(p)
"{\"ok\":true,\"path\":\"" + p + "\"}"
}
fn route_load(method: String, path: String, body: String) -> String {
let p_raw: String = json_get_string(body, "path")
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = engram_resolve_data_dir()
let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
// (2026-08-10 self-review) This was a stub response over the single most
// destructive operation in the server. engram_load returns 0 on an empty
// path, an unopenable file, a zero-length file, or malloc failure and
// this route answered ok_json() in every one of those cases.
//
// Precise failure shape (el_runtime.c:9890): the fopen guard runs BEFORE
// the store reset, so a MISSING path is genuinely safe it returns 0 with
// the graph intact. The dangerous case is a readable-but-malformed file:
// the reset loop frees every node and edge FIRST, then parses, so a
// truncated or non-snapshot JSON leaves a hollow store and the caller
// was told "ok":true. With 37 GB of stale dated snapshots sitting in the
// data dir as tempting restore targets, "restore reported success and
// silently emptied the graph" is a live risk, not a hypothetical one.
//
// Fix: surface the return value AND the resulting counts. node_count=0
// after a load is the unambiguous hollow-store signal (same convention
// route_health adopted 2026-08-01). Callers can now verify a restore
// instead of trusting it.
let ld: Int = engram_load(p)
let ld_ok: String = if ld == 0 { "false" } else { "true" }
let nc_after: Int = engram_node_count()
let hollow: String = if nc_after == 0 { "true" } else { "false" }
"{\"ok\":" + ld_ok + ",\"path\":\"" + p + "\",\"node_count\":" + int_to_str(nc_after) + ",\"edge_count\":" + int_to_str(engram_edge_count()) + ",\"hollow\":" + hollow + "}"
}
// (2026-08-01 self-review) Health previously returned a hardcoded literal
// it reported "ok" even when the snapshot failed to load and the store was
// empty. Now reports live counts so a monitor can distinguish "up and
// loaded" from "up and hollow" (node_count=0 after boot = failed load).
fn route_health(method: String, path: String, body: String) -> String {
"{\"status\":\"ok\",\"engine\":\"engram-runtime-native\",\"node_count\":" + int_to_str(engram_node_count()) + ",\"edge_count\":" + int_to_str(engram_edge_count()) + "}"
}
// route_embed_backfill GET/POST /api/embed-backfill?n=48
//
// (2026-07-25 self-review) The lazy embedding backfill runs only inside
// engram_activate, and nothing in production calls /api/activate on this
// store the soul's curiosity loop activates its own in-process graph.
// After a restart from a snapshot without vectors, embedded_count stalled
// at 93/12175 and would never recover. This route lets the soul's
// heartbeat pump the backfill explicitly (48/min clears a 12k backlog in
// ~4h). Persists the canonical snapshot whenever new vectors were
// generated the 2026-07-25 regression happened precisely because 3747
// in-RAM embeddings were never snapshotted before a restart. Self-
// limiting: once coverage is full, embedded=0 and no save occurs.
fn route_embed_backfill(method: String, path: String, body: String) -> String {
let n: Int = query_int(path, "n", 32)
let result: String = engram_embed_backfill(n)
let done: Float = json_get_float(result, "embedded")
if done > 0.0 {
let saved: Int = persist_bulk()
}
return result
}
// route_sync return a snapshot of non-ISE/non-Working nodes for the soul daemon
// to merge into its in-process graph via engram_load_merge.
//
// The soul calls GET /api/sync every SOUL_REFRESH_MS (default 10 min) to pull
// new Knowledge/Memory/BacklogItem nodes from the authoritative HTTP Engram into
// its in-process working store. Previously this returned 404 "not found", causing
// the soul to write the error JSON to a temp file and attempt an empty merge.
//
// Strategy: save the current snapshot to disk, read it back, return the full
// snapshot JSON. The soul's engram_load_merge handles large files gracefully
// (it skips nodes already present by ID). Auth-exempt: same-host internal call.
// (2026-06-27 self-review: added this route to fix silent 10-min sync failures)
fn route_sync(method: String, path: String, body: String) -> String {
let dir_raw: String = env("ENGRAM_DATA_DIR")
let dir: String = engram_resolve_data_dir()
// 2026-07-21 self-review: export to a scratch path, never the canonical
// snapshot.json read routes must not be able to clobber the good snapshot.
let snap_path: String = dir + "/.sync-export.json"
engram_save(snap_path)
let snap: String = fs_read(snap_path)
// 2026-08-02 self-review: this used to return {"nodes":[],"edges":[]} when
// the export/read failed. The soul's sync_ok test (awareness.el) only
// checks for "" and "{}", so that placeholder PASSED as a healthy sync:
// soul.last_sync_ok_ts got stamped, sync_age_ms stayed green, the
// sync_empty warn ISE never fired, and engram_sync reported added:0
// forever. A totally broken sync was indistinguishable from a quiet
// healthy one the exact failure class this route was added to fix in
// the first place (see 2026-06-27 note above). Return a real error so the
// failure is loud on both sides.
if str_eq(snap, "") { return err_json("sync export failed: snapshot unreadable") }
return snap
}
// route_load_merge POST /api/load-merge {"path": "..."} merge a snapshot
// file into the live store WITHOUT resetting it (engram_load_merge skips nodes
// already present by id). Added 2026-07-21 self-review to restore the 244 kn-
// identity Knowledge nodes lost from the snapshot lineage between 05-13 and
// 07-13. Requires an explicit path: refuses to run without one so it can never
// be triggered accidentally against a default.
fn route_load_merge(method: String, path: String, body: String) -> String {
let p: String = json_get_string(body, "path")
if str_eq(p, "") { return err_json("path is required") }
if str_eq(fs_read(p), "") { return err_json("file missing or empty") }
let before_n: Int = engram_node_count()
let before_e: Int = engram_edge_count()
engram_load_merge(p)
let added_n: Int = engram_node_count() - before_n
let added_e: Int = engram_edge_count() - before_e
let saved: Int = persist_bulk()
"{\"ok\":true,\"nodes_added\":" + int_to_str(added_n) + ",\"edges_added\":" + int_to_str(added_e) + ",\"node_count\":" + int_to_str(engram_node_count()) + "}"
if str_eq(p, "") {
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let p = dir + "/snapshot.json"
}
engram_load(p)
ok_json()
}
// route_emit_ise write an InternalStateEvent node from the soul daemon.
//
// Endpoint: POST /api/neuron/state-events
// Body: {"content": "<json-string>"}
//
// Auth: exempt (internal endpoint, soul daemon on same host, no _auth needed).
// The soul's ise_post() sends {"content":"..."} without _auth; enforcing auth
// here would silently drop all heartbeat/curiosity ISEs. Unauthenticated POST
// to this endpoint is acceptable: ISE writes are observability-only, append-only,
// and come from a trusted process on localhost.
//
// Salience/importance set to match engram_node_full ISE defaults used by the
// in-process fallback path in awareness.el (salience=0.3, importance=0.3,
// confidence=0.8, tier=Episodic).
// (2026-06-26 self-review: added this route after discovering ise_post was
// silently failing the soul posts here but the endpoint didn't exist.)
//
// Retention (2026-07-16 self-review): an earlier comment here claimed ISEs
// got temporal_decay_rate=1.617 that was never implemented (engram_node_full
// hardcodes 0.0), and per-node decay only dampens activation anyway; it never
// removes nodes. By 2026-07-16 ISEs were 75% of the store (10,175 of 13,522
// nodes, ~4,300/day, unbounded). ISEs are already WM-excluded in
// engram_activate, so the fix is retention, not decay: every insert calls
// engram_prune_telemetry(), a single O(nodes+edges) compaction pass that
// removes ISEs older than ENGRAM_ISE_RETENTION_MS (default 48h), protecting
// "session-start" labels and self_review events as durable history. At
// ~3 ISEs/min this bounds telemetry at ~8.6k nodes instead of growing forever.
fn route_emit_ise(method: String, path: String, body: String) -> String {
let content: String = json_get_string(body, "content")
if str_eq(content, "") { return err_json("missing content") }
let sal: Float = 0.3
let imp: Float = 0.3
let conf: Float = 0.8
let id: String = engram_node_full(
content, "InternalStateEvent", "state-event",
sal, imp, conf,
"Episodic", "[\"internal-state\",\"InternalStateEvent\"]"
)
let ret_raw: String = env("ENGRAM_ISE_RETENTION_MS")
let ret_ms: Int = if str_eq(ret_raw, "") { 172800000 } else { str_to_int(ret_raw) }
let pruned: Int = engram_prune_telemetry(ret_ms)
"{\"ok\":true,\"id\":\"" + id + "\",\"pruned\":" + int_to_str(pruned) + "}"
}
// Knowledge capture
//
// route_capture_knowledge direct Knowledge-node capture over HTTP.
//
// Endpoint: POST /api/neuron/knowledge/capture (auth required: "_auth" in body)
// Body: {"content": "...", "title": "...", "category": "...",
// "tier": "note|lesson|canonical", "tags": [...], "project": "...",
// "_auth": "<key>"}
//
// WHY (2026-07-15 self-review): the world-ingestor integrator was designed
// against this endpoint (its MCP-unavailable fallback), but the route never
// existed every direct push 404'd, and because the auth gate ran before
// routing, the failure surfaced as {"error":"unauthorized"} and was
// misdiagnosed for two weeks while world knowledge silently dropped.
// POST /api/nodes was no substitute: it discards label/tags/tier, which
// makes captured knowledge invisible to tag-scoped search and curiosity.
//
// The incoming knowledge tier (note/lesson/canonical) is preserved as a
// "tier:<x>" tag rather than mapped onto Engram's cognitive tiers Knowledge
// nodes land in Semantic (stable reference), and the epistemic tier stays
// queryable without inventing a lossy mapping.
fn route_capture_knowledge(method: String, path: String, body: String) -> String {
let content: String = json_get_string(body, "content")
if str_eq(content, "") { return err_json("missing content") }
let title: String = json_get_string(body, "title")
let label: String = if str_eq(title, "") { str_slice(content, 0, 60) } else { title }
let category_raw: String = json_get_string(body, "category")
let category: String = if str_eq(category_raw, "") { "other" } else { category_raw }
let ktier_raw: String = json_get_string(body, "tier")
let ktier: String = if str_eq(ktier_raw, "") { "note" } else { ktier_raw }
let project: String = json_get_string(body, "project")
let tags_raw: String = json_get_raw(body, "tags")
let tags_base: String = if str_eq(tags_raw, "") { "[]" } else { tags_raw }
// Merge category/tier/project markers into the tag array. Search matches
// against the tags string, so these make captures findable by facet.
let base_len: Int = str_len(tags_base)
let head: String = str_slice(tags_base, 0, base_len - 1)
let sep: String = if str_eq(head, "[") { "" } else { "," }
let safe_cat: String = str_replace(category, "\"", "'")
let safe_tier: String = str_replace(ktier, "\"", "'")
let safe_proj: String = str_replace(project, "\"", "'")
let proj_tag: String = if str_eq(safe_proj, "") { "" } else { ",\"project:" + safe_proj + "\"" }
let tags: String = head + sep + "\"category:" + safe_cat + "\",\"tier:" + safe_tier + "\"" + proj_tag + "]"
let sal: Float = 0.5
let imp: Float = 0.5
let conf: Float = 0.9
let id: String = engram_node_full(
content, "Knowledge", label,
sal, imp, conf,
"Semantic", tags
)
let saved: Int = persist_node(id)
"{\"ok\":true,\"id\":\"" + id + "\"}"
}
// route_similarity GET /api/similarity?a=<id>&b=<id>
//
// (2026-08-01 self-review) engram_cosine_sim was added 2026-07-24
// (bl-b2d1c944) with the stated purpose of exposing semantic distance to
// "EL code and the introspection API" but it had ZERO callers anywhere:
// no route, no soul-daemon use. The activation path uses embeddings
// internally (semantic seeding, Pass-2 additive term), but there was no way
// to probe pairwise node similarity from outside. This closes that: cosine
// in [-1,1], or -2 when either node is missing or not yet embedded (so
// "not comparable" is distinguishable from "genuinely orthogonal" 0.0).
fn route_similarity(method: String, path: String, body: String) -> String {
let a: String = query_param(path, "a")
let b: String = query_param(path, "b")
if str_eq(a, "") { return err_json("missing a") }
if str_eq(b, "") { return err_json("missing b") }
let sim: Float = engram_cosine_sim(a, b)
"{\"a\":\"" + a + "\",\"b\":\"" + b + "\",\"cosine\":" + float_to_str(sim) + "}"
fn route_health(method: String, path: String, body: String) -> String {
"{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}"
}
// Auth
@@ -699,32 +232,15 @@ fn handle_request(method: String, path: String, body: String) -> String {
}
}
// ISE posting is auth-exempt (internal soul daemon, same host, no _auth key)
if str_eq(method, "POST") && str_eq(clean, "/api/neuron/state-events") {
return route_emit_ise(method, path, body)
}
// Auth (when ENGRAM_API_KEY is set)
if !check_auth_ok(method, body) {
return err_json("unauthorized")
}
// Knowledge capture (auth enforced above; the world-ingestor integrator
// and any headless session without MCP push knowledge through this)
if str_eq(method, "POST") && str_eq(clean, "/api/neuron/knowledge/capture") {
return route_capture_knowledge(method, path, body)
}
// Stats
if str_eq(method, "GET") && (str_eq(clean, "/api/stats") || str_eq(clean, "/stats")) {
return route_stats(method, path, body)
}
if str_eq(method, "GET") && (str_eq(clean, "/api/act-stats") || str_eq(clean, "/act-stats")) {
return route_act_stats(method, path, body)
}
if str_eq(method, "GET") && (str_eq(clean, "/api/text-health") || str_eq(clean, "/text-health")) {
return route_text_health(method, path, body)
}
// Nodes
if str_eq(method, "POST") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes")) {
@@ -747,13 +263,6 @@ fn handle_request(method: String, path: String, body: String) -> String {
if str_eq(method, "POST") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) {
return route_create_edge(method, path, body)
}
// Batch edge write one snapshot for the whole payload. Must be tested
// BEFORE nothing else claims it; the exact-match on "/api/edges" above
// does not catch "/api/edges/batch", so order is not load-bearing here,
// but keeping the two adjacent keeps them from drifting apart.
if str_eq(method, "POST") && (str_eq(clean, "/api/edges/batch") || str_eq(clean, "/edges/batch")) {
return route_create_edges_batch(method, path, body)
}
if str_eq(method, "GET") && str_starts_with(clean, "/api/neighbors/") {
return route_neighbors(method, path, body)
}
@@ -784,75 +293,21 @@ fn handle_request(method: String, path: String, body: String) -> String {
if str_eq(method, "POST") && (str_eq(clean, "/api/load") || str_eq(clean, "/load")) {
return route_load(method, path, body)
}
if str_eq(method, "POST") && (str_eq(clean, "/api/load-merge") || str_eq(clean, "/load-merge")) {
return route_load_merge(method, path, body)
}
// Sync soul daemon periodic pull of non-ISE knowledge into in-process graph
if str_eq(method, "GET") && str_eq(clean, "/api/sync") {
return route_sync(method, path, body)
}
// Embedding backfill pumped by the soul heartbeat (2026-07-25)
if str_eq(clean, "/api/embed-backfill") {
return route_embed_backfill(method, path, body)
}
// Semantic similarity probe (2026-08-01)
if str_eq(method, "GET") && str_starts_with(clean, "/api/similarity") {
return route_similarity(method, path, body)
}
"{\"error\":\"not found\",\"path\":\"" + clean + "\"}"
}
// Entry
let bind_raw: String = env("ENGRAM_BIND")
let bind_str: String = if str_eq(bind_raw, "") { ":8742" } else { bind_raw }
let bind_str: String = env("ENGRAM_BIND")
if str_eq(bind_str, "") { let bind_str = ":8742" }
let port: Int = parse_port(bind_str)
// On startup, try to load any existing snapshot (best effort).
// §18.2: resolve the data dir safely unset ENGRAM_DATA_DIR $HOME/.neuron/engram,
// never /tmp; fail loud if HOME is unresolvable (engram_resolve_data_dir exits).
let data_dir: String = engram_resolve_data_dir()
let data_dir: String = env("ENGRAM_DATA_DIR")
if str_eq(data_dir, "") { let data_dir = "/tmp/engram" }
let snapshot_path: String = data_dir + "/snapshot.json"
// ENGRAM_STORE (tiered paged store engram-tiered-storage-engine.md). When set,
// the durable owner is the paged store (neuron.egm + neuron.wal): engram_store_boot
// imports snapshot.json ONCE into a fresh neuron.egm, else replays the WAL and loads
// the store resident snapshot.json is never read again as the ongoing store. This
// closes the "restart reverted to a 17h-old snapshot" data-loss window. Flag-off
// (default): byte-for-byte the historical snapshot + optional-WAL boot below.
if store_on() {
engram_store_boot(data_dir)
println("[engram] ENGRAM_STORE enabled — tiered paged store is the durable owner")
} else {
engram_load(snapshot_path)
// WAL replay (design doc §6). Gated: default OFF is byte-identical to legacy
// snapshot-only boot. When ON, the snapshot above is the compaction BASE and
// the WAL carries every mutation since; replay reconstructs state to the last
// CRC-valid record, then opens the WAL for appending.
if wal_on() {
let replayed: Int = engram_wal_boot(data_dir)
println("[engram] WAL enabled — replayed " + int_to_str(replayed) + " records")
}
// 2026-07-21 self-review boot guard: if the snapshot file has content but the
// load produced 0 nodes, something is wrong (corrupt file / parse failure).
// Preserve the evidence and warn loudly and since read routes no longer write
// the canonical path, a bad boot can no longer clobber the good snapshot.
let boot_snap: String = fs_read(snapshot_path)
if !str_eq(boot_snap, "") {
if engram_node_count() == 0 {
println("[engram] WARNING: snapshot.json is non-empty but load produced 0 nodes — preserving copy at snapshot.failed-load.json")
fs_write(data_dir + "/snapshot.failed-load.json", boot_snap)
} else {
// Good load: keep a boot-time backup of the snapshot as loaded.
fs_write(data_dir + "/snapshot.boot-backup.json", boot_snap)
}
}
}
engram_load(snapshot_path)
println("[engram] runtime-native graph engine")
println("[engram] data_dir=" + data_dir)
-158
View File
@@ -1,158 +0,0 @@
#!/usr/bin/env bash
# M3.5 PRE-FLIP GATE. Pure C harness (NOT elb/elc): links the real el_runtime.c
# native engram builtins + engram_store.c and proves activation-time field
# mutations (edge hebb, node activation_count, WM weight) persist through a
# checkpoint and survive a reboot from neuron.egm with snapshot.json DELETED.
# Writes ONLY under a throwaway /tmp dir with a throwaway HOME.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
RT="$HERE/../../lang/runtime/el_runtime.c"
ST="$HERE/../../lang/runtime/engram_store.c"
INC="$HERE/../../lang/runtime"
WORK="$(mktemp -d /tmp/engram-m35-XXXXXX)"
BIN="$WORK/m35"
export HOME="$WORK/home"; mkdir -p "$HOME" # never touch real ~/.neuron
export ENGRAM_WAL_SYNC=always
unset ENGRAM_STORE
fail=0
echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m35_hebb_persist.c) =="
gcc -O1 -std=c11 -I "$INC" "$HERE/test_m35_hebb_persist.c" "$RT" "$ST" -lcurl -o "$BIN" 2>"$WORK/cc.log"
if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi
echo
echo "== 0) flag-OFF: seed+activate+checkpoint must NOT touch the store =="
DOFF="$WORK/off"; mkdir -p "$DOFF"
( unset ENGRAM_STORE; "$BIN" offcheck "$DOFF" )
[ $? -ne 0 ] && { echo "FAIL: offcheck"; fail=1; }
[ -e "$DOFF/neuron.egm" ] && { echo "FAIL: neuron.egm created while flag OFF"; fail=1; } \
|| echo " ok: no neuron.egm created with flag OFF"
echo
echo "== 1) POSITIVE: ENGRAM_STORE=1 seed -> activate -> checkpoint(field-persist) -> close =="
DPOS="$WORK/pos"; mkdir -p "$DPOS"
ENGRAM_STORE=1 "$BIN" pos_seed "$DPOS" || { echo "FAIL: pos_seed"; fail=1; }
[ -e "$DPOS/neuron.egm" ] && echo " ok: neuron.egm created" || { echo "FAIL: neuron.egm missing"; fail=1; }
echo
echo "== 2) reboot from neuron.egm with snapshot.json DELETED (must never read JSON) =="
rm -f "$DPOS/snapshot.json"
ENGRAM_STORE=1 "$BIN" pos_reboot "$DPOS" || { echo "FAIL: pos_reboot"; fail=1; }
echo
echo "== 3) NEGATIVE CONTROL: seed -> activate -> close WITHOUT the field-persist checkpoint =="
DNEG="$WORK/neg"; mkdir -p "$DNEG"
ENGRAM_STORE=1 "$BIN" neg_seed "$DNEG" || { echo "FAIL: neg_seed"; fail=1; }
rm -f "$DNEG/snapshot.json"
ENGRAM_STORE=1 "$BIN" neg_reboot "$DNEG" || { echo "FAIL: neg_reboot"; fail=1; }
echo
echo "== 4) assertions (python over the JSON exports) =="
python3 - "$DPOS" "$DNEG" <<'PY'
import json, sys, os
WM_FLOOR = 0.05
HEBB_MIN = 1e-6
def load(d, name):
with open(os.path.join(d, name)) as f: return json.load(f)
def node_by_label(g, label):
for n in g["nodes"]:
if n.get("label") == label: return n
return None
def edge_between(g, a_id, b_id):
for e in g["edges"]:
if e.get("from_id") == a_id and e.get("to_id") == b_id:
return e
return None
rc = 0
def check(cond, msg):
global rc
if cond: print(f" PASS: {msg}")
else: print(f" FAIL: {msg}"); rc = 1
dpos, dneg = sys.argv[1], sys.argv[2]
pre = load(dpos, "pre_reboot.json")
rebt = load(dpos, "reboot.json")
pa, pb = node_by_label(pre, "hebb-a"), node_by_label(pre, "hebb-b")
ra = node_by_label(rebt, "hebb-a")
assert pa and pb and ra, "target nodes missing"
pe = edge_between(pre, pa["id"], pb["id"])
re = edge_between(rebt, pa["id"], pb["id"])
assert pe and re, "target edge missing"
pre_hebb = pe.get("hebb", 0.0)
rebt_hebb = re.get("hebb", 0.0)
pre_ac = pa.get("activation_count", 0)
rebt_ac = ra.get("activation_count", 0)
pre_wm = pa.get("working_memory_weight", 0.0)
rebt_wm = ra.get("working_memory_weight", 0.0)
print(f" edge hebb-a->hebb-b : pre={pre_hebb!r} reboot={rebt_hebb!r}")
print(f" node hebb-a act_cnt : pre={pre_ac!r} reboot={rebt_ac!r}")
print(f" node hebb-a wm : pre={pre_wm!r} reboot={rebt_wm!r} (halved+floored expected)")
# --- learning actually happened this run (else the test proves nothing) ---
check(pre_hebb > HEBB_MIN, f"activation raised edge hebb above 0 (pre={pre_hebb})")
check(pre_ac >= 1, f"activation reinforced node activation_count (pre={pre_ac})")
check(pre_wm > 0.0, f"activation promoted node to working memory (pre_wm={pre_wm})")
# --- the load-bearing survival assertions after a real delete-JSON reboot ---
check(abs(rebt_hebb - pre_hebb) < 1e-12,
f"edge hebb SURVIVED reboot unchanged ({rebt_hebb} == {pre_hebb})")
check(rebt_ac == pre_ac,
f"node activation_count SURVIVED reboot unchanged ({rebt_ac} == {pre_ac})")
# --- WM weight: must equal the JSON path's boot transform exactly (halve+floor) ---
expected_wm = pre_wm * 0.5
if expected_wm < WM_FLOOR: expected_wm = 0.0
check(abs(rebt_wm - expected_wm) < 1e-9,
f"node WM weight SURVIVED with the SAME boot transform as JSON path "
f"(reboot={rebt_wm} == halve+floor(pre)={expected_wm})")
check(expected_wm > 0.0,
f"WM survival is observable (halved weight stays above floor: {expected_wm} > {WM_FLOOR})")
# --- NEGATIVE CONTROL: without the field-persist step the learning is LOST ---
npre = load(dneg, "neg_pre.json")
nrebt = load(dneg, "neg_reboot.json")
na_pre = node_by_label(npre, "hebb-a")
na_rebt = node_by_label(nrebt, "hebb-a")
ne_pre = edge_between(npre, na_pre["id"], node_by_label(npre, "hebb-b")["id"])
ne_rebt = edge_between(nrebt, na_rebt["id"], node_by_label(nrebt, "hebb-b")["id"])
print(f" [neg] edge hebb : pre={ne_pre.get('hebb',0.0)!r} reboot={ne_rebt.get('hebb',0.0)!r}")
print(f" [neg] node act_cnt : pre={na_pre.get('activation_count',0)!r} reboot={na_rebt.get('activation_count',0)!r}")
check(ne_pre.get("hebb", 0.0) > HEBB_MIN,
f"[neg] activation DID raise hebb in RAM (pre={ne_pre.get('hebb',0.0)})")
check(ne_rebt.get("hebb", 0.0) == 0.0,
"[neg] WITHOUT checkpoint field-persist, edge hebb is LOST on reboot (==0) — fix is load-bearing")
check(na_rebt.get("activation_count", 0) == 0,
"[neg] WITHOUT checkpoint field-persist, activation_count is LOST on reboot (==0)")
sys.exit(rc)
PY
[ $? -ne 0 ] && fail=1
echo
echo "== 5) ASan+UBSan build, exercise the full persist+reboot flow (leaks off — harness intentionally leaks el_strdup) =="
SANBIN="$WORK/m35.san"
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
-I "$INC" "$HERE/test_m35_hebb_persist.c" "$RT" "$ST" -lcurl -o "$SANBIN" 2>"$WORK/san_cc.log"
if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else
export ASAN_OPTIONS=detect_leaks=0
DSAN="$WORK/san"; mkdir -p "$DSAN"
ENGRAM_STORE=1 "$SANBIN" pos_seed "$DSAN" >/dev/null 2>"$WORK/san_run.log" && \
{ rm -f "$DSAN/snapshot.json"; ENGRAM_STORE=1 "$SANBIN" pos_reboot "$DSAN" >/dev/null 2>>"$WORK/san_run.log"; }
if grep -qiE 'runtime error|AddressSanitizer|UndefinedBehavior|ERROR: ' "$WORK/san_run.log"; then
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san_run.log" | head; fail=1
else
echo " ok: ASan+UBSan clean across pos_seed/checkpoint/reboot (field-persist, boot laundering)"
fi
fi
echo
if [ "$fail" -eq 0 ]; then echo "================ M3.5 HEBB-PERSIST GATE: PASS ================"; else echo "================ M3.5 HEBB-PERSIST GATE: FAIL ================"; fi
rm -rf "$WORK"
exit $fail
-126
View File
@@ -1,126 +0,0 @@
#!/usr/bin/env bash
# M3 JSON-parity gate. Pure C harness (NOT elb/elc): links the real el_runtime.c
# native engram builtins + engram_store.c and drives ENGRAM_STORE on vs off.
# Writes ONLY under a throwaway /tmp dir with a throwaway HOME + ENGRAM_DATA_DIR.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
RT="$HERE/../../lang/runtime/el_runtime.c"
ST="$HERE/../../lang/runtime/engram_store.c"
INC="$HERE/../../lang/runtime"
WORK="$(mktemp -d /tmp/engram-m3-XXXXXX)"
DATA="$WORK/data"; mkdir -p "$DATA"
BIN="$WORK/m3"
export HOME="$WORK/home"; mkdir -p "$HOME" # never touch real ~/.neuron
export ENGRAM_DATA_DIR="$DATA"
export ENGRAM_WAL_SYNC=always
unset ENGRAM_STORE
fail=0
echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m3_parity.c) =="
gcc -O1 -std=c11 -I "$INC" "$HERE/test_m3_parity.c" "$RT" "$ST" -lcurl -o "$BIN" 2>"$WORK/cc.log"
if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi
grep -i warning "$WORK/cc.log" | grep -iE 'engram_store|eg_store|eg_load|scan_nodes|scan_edges' && echo "(warnings in M3 code above)" || true
echo
echo "== 0) default-OFF: flag unset leaves the store untouched =="
( unset ENGRAM_STORE; "$BIN" offcheck "$DATA" )
[ $? -ne 0 ] && { echo "FAIL: offcheck"; fail=1; }
[ -e "$DATA/neuron.egm" ] && { echo "FAIL: neuron.egm created while flag OFF"; fail=1; } \
|| echo " ok: no neuron.egm created with flag OFF"
echo
echo "== 1) seed (ENGRAM_STORE unset): build graph, save snapshot.json, activate =="
( unset ENGRAM_STORE; "$BIN" seed "$DATA" ) || { echo "FAIL: seed"; fail=1; }
echo
echo "== 2) on (ENGRAM_STORE=1): import snapshot.json ONCE -> neuron.egm, resident-load, activate =="
ENGRAM_STORE=1 "$BIN" on "$DATA" || { echo "FAIL: on"; fail=1; }
[ -e "$DATA/neuron.egm" ] && echo " ok: neuron.egm created by import" || { echo "FAIL: neuron.egm missing"; fail=1; }
echo
echo "== 3) reboot (ENGRAM_STORE=1, snapshot.json DELETED): must load from neuron.egm, never JSON =="
rm -f "$DATA/snapshot.json"
ENGRAM_STORE=1 "$BIN" reboot "$DATA" || { echo "FAIL: reboot"; fail=1; }
echo
echo "== 4) parity comparison (modulo ordering) =="
python3 - "$DATA" <<'PY'
import json, sys, os
d = sys.argv[1]
def load(name):
with open(os.path.join(d, name)) as f: return json.load(f)
def norm_graph(g):
nodes = sorted(g.get("nodes", []), key=lambda n: n.get("id",""))
edges = sorted(g.get("edges", []), key=lambda e: e.get("id",""))
layers= sorted(g.get("layers", []), key=lambda l: l.get("layer_id",0))
return {"nodes":nodes, "edges":edges, "layers":layers}
def act_ids(a):
# list of (node id, promoted); robust set + ordered list
seq = [(e.get("node",{}).get("id",""), int(e.get("promoted",0))) for e in a]
return seq
rc = 0
snap = norm_graph(load("snapshot.json") if os.path.exists(os.path.join(d,"snapshot.json")) else load("off_graph.json"))
off = norm_graph(load("off_graph.json"))
on = norm_graph(load("on_graph.json"))
rebt = norm_graph(load("reboot_graph.json"))
def cmp(label, a, b):
global rc
if a == b:
print(f" PASS: {label} (nodes={len(a['nodes'])} edges={len(a['edges'])} layers={len(a['layers'])})")
else:
rc = 1
print(f" FAIL: {label}")
for k in ("nodes","edges","layers"):
if a[k] != b[k]:
print(f" {k}: {len(a[k])} vs {len(b[k])}")
for x,y in zip(a[k], b[k]):
if x != y:
print(f" first diff:\n A={json.dumps(x)[:300]}\n B={json.dumps(y)[:300]}")
break
cmp("graph: ENGRAM_STORE=1 (export) == ENGRAM_STORE=0 (JSON path)", on, off)
cmp("round-trip: snapshot.json seed == store export (on_graph)", on, off) # off_graph==snapshot save
cmp("reboot from neuron.egm (no JSON) == on-path store", rebt, on)
offa = act_ids(load("off_act.json"))
ona = act_ids(load("on_act.json"))
if set(offa) == set(ona):
print(f" PASS: activation result set identical (off={len(offa)} on={len(ona)} entries)")
if offa == ona:
print(" (and identical ordering/promotion sequence)")
else:
print(" (same set; ordering differs only where scores tie — reporting honestly)")
else:
rc = 1
print(" FAIL: activation result set differs")
print(f" off-only: {set(offa)-set(ona)}")
print(f" on-only: {set(ona)-set(offa)}")
sys.exit(rc)
PY
[ $? -ne 0 ] && fail=1
echo
echo "== 5) ASan+UBSan build, exercise M3 scan/boot/hooks (leaks off — harness intentionally leaks el_strdup) =="
SANBIN="$WORK/m3.san"
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
-I "$INC" "$HERE/test_m3_parity.c" "$RT" "$ST" -lcurl -o "$SANBIN" 2>"$WORK/san_cc.log"
if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else
export ASAN_OPTIONS=detect_leaks=0
DATA2="$WORK/data2"; mkdir -p "$DATA2"
( unset ENGRAM_STORE; "$SANBIN" seed "$DATA2" ) >/dev/null 2>"$WORK/san_run.log" && \
ENGRAM_STORE=1 "$SANBIN" on "$DATA2" >/dev/null 2>>"$WORK/san_run.log" && \
{ rm -f "$DATA2/snapshot.json"; ENGRAM_STORE=1 "$SANBIN" reboot "$DATA2" >/dev/null 2>>"$WORK/san_run.log"; }
if grep -qiE 'runtime error|AddressSanitizer|UndefinedBehavior|ERROR: ' "$WORK/san_run.log"; then
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san_run.log" | head; fail=1
else
echo " ok: ASan+UBSan clean across seed/on/reboot (scan, boot, resident-load, mutation hooks)"
fi
fi
echo
if [ "$fail" -eq 0 ]; then echo "================ M3 PARITY GATE: PASS ================"; else echo "================ M3 PARITY GATE: FAIL ================"; fi
rm -rf "$WORK"
exit $fail
-13
View File
@@ -1,13 +0,0 @@
#!/usr/bin/env bash
# M1 paged-store gate. Pure C (NOT elb/elc). Writes only under /tmp.
set -e
HERE="$(cd "$(dirname "$0")" && pwd)"
SRC="$HERE/../../lang/runtime/engram_store.c"
BIN="/tmp/test_store.$$"
echo "compiling: gcc test_store.c engram_store.c"
gcc -O2 -Wall -Wextra -std=c11 "$HERE/test_store.c" "$SRC" -o "$BIN"
"$BIN"
rc=$?
rm -f "$BIN"
rm -rf /tmp/engram-store-test-*
exit $rc
-14
View File
@@ -1,14 +0,0 @@
#!/usr/bin/env bash
# M2 WAL + checkpoint + recovery gate. Pure C (NOT elb/elc). Writes only under /tmp.
# Recovery tests use ENGRAM_WAL_SYNC=always so every WAL record is durable at crash.
set -e
HERE="$(cd "$(dirname "$0")" && pwd)"
SRC="$HERE/../../lang/runtime/engram_store.c"
BIN="/tmp/test_wal_store.$$"
echo "compiling: gcc test_wal_store.c engram_store.c"
gcc -O2 -Wall -Wextra -std=c11 "$HERE/test_wal_store.c" "$SRC" -o "$BIN"
ENGRAM_WAL_SYNC=always "$BIN"
rc=$?
rm -f "$BIN"
rm -rf /tmp/engram-wal-test-*
exit $rc
-16
View File
@@ -1,16 +0,0 @@
#!/usr/bin/env bash
# WAL unit + integration + crash-fuzz gate. Throwaway HOME/dirs only.
set -e
HERE="$(cd "$(dirname "$0")" && pwd)"
REL="$HERE/../../lang/runtime"
cc -O2 -fbracket-depth=1024 -Wno-parentheses-equality -I"$REL" \
"$HERE/test_wal.c" -lcurl -lpthread -o /tmp/test_wal
HOME=/tmp/engram-throwaway-home /tmp/test_wal
# Fail-loud data-dir check (must exit 1 with a FATAL line):
cat > /tmp/test_failloud.c <<'C'
#include "el_runtime.c"
int main(void){ unsetenv("ENGRAM_DATA_DIR"); unsetenv("HOME");
engram_resolve_data_dir(); printf("REACHED\n"); return 0; }
C
cc -O2 -fbracket-depth=1024 -Wno-parentheses-equality -I"$REL" /tmp/test_failloud.c -lcurl -lpthread -o /tmp/test_failloud
if env -u HOME -u ENGRAM_DATA_DIR /tmp/test_failloud; then echo "FAIL: should have exited"; exit 1; else echo "[PASS] fail-loud exit on unresolvable HOME"; fi
-130
View File
@@ -1,130 +0,0 @@
/* test_m35_hebb_persist.c — M3.5 PRE-FLIP GATE.
*
* Proves that in-place field mutations made during spreading activation edge
* `hebb` (+ last_fired), node `activation_count`, node working-memory weight
* PERSIST to the paged store and survive a restart from neuron.egm with
* snapshot.json deleted. This is the "hebb-survives-restart" fix that gates the
* live cutover.
*
* Same style as test_m3_parity.c: a REAL el-level harness linking the actual
* el_runtime.c native engram builtins + engram_store.c, driving engram_node_full
* / engram_connect / engram_activate_json / engram_save / engram_store_boot /
* engram_store_checkpoint / engram_store_close directly from C. No EL interpreter.
*
* Modes (argv[1]), data dir (argv[2]):
* pos_seed ENGRAM_STORE=1: fresh store, seed a graph tuned so activation
* co-activates a connected pair (edge hebb 0 -> ETA) and reinforces
* nodes (activation_count 0 -> >=1, WM weight -> >0). Export the
* post-activation resident graph to pre_reboot.json, then CHECKPOINT
* (the M3.5 field-persist), then close.
* pos_reboot ENGRAM_STORE=1, snapshot.json deleted by runner: boot from
* neuron.egm (WAL replay), export reboot.json, close. The values in
* reboot.json are what actually survived the round-trip.
* neg_seed identical to pos_seed but WITHOUT the checkpoint field-persist
* (negative control): activation mutations never reach the store.
* neg_reboot boot from neuron.egm, export neg_reboot.json, close.
* offcheck ENGRAM_STORE unset: seed+activate+checkpoint must NOT touch the
* store (no neuron.egm, checkpoint returns 0).
*
* The pass/fail assertions live in run_m35_hebb_persist.sh (python over the JSON
* exports): reboot.json must carry the learned hebb / activation_count and the
* JSON-identical halved WM weight; neg_reboot.json must have LOST them.
*/
#include "el_runtime.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
extern int engram_store_enabled(void);
extern el_val_t engram_store_boot(el_val_t data_dir);
extern el_val_t engram_store_checkpoint(void);
extern el_val_t engram_store_close(void);
static el_val_t S(const char* s){ return EL_STR(s); }
static el_val_t F(double d){ return el_from_float(d); }
/* Two nodes with DISTINCT content (so the redundancy-suppression pass cannot
* dedup one of them away) that both match the query strongly, wired by one
* "associate" edge. A handful of weakly-related distractors make it a real
* graph. On activation both A and B promote to working memory and co-activate,
* so their edge's hebb rises from 0 to ENGRAM_HEBB_ETA. */
static void build_seed(void){
el_val_t a = engram_node_full(S("hebbian potentiation strengthens co-active memory links"),
S("Concept"), S("hebb-a"), F(0.9), F(0.85), F(1.0), S("Semantic"),
S("hebbian,memory,activation"));
el_val_t b = engram_node_full(S("co-active memory links accrue hebbian associative weight"),
S("Concept"), S("hebb-b"), F(0.9), F(0.85), F(1.0), S("Semantic"),
S("hebbian,memory,weight"));
el_val_t c = engram_node_full(S("unrelated culinary recipe for sourdough bread"),
S("Fact"), S("distractor-1"), F(0.4), F(0.4), F(1.0), S("Semantic"),
S("food"));
el_val_t d = engram_node_full(S("the weather forecast predicts rain tomorrow afternoon"),
S("Fact"), S("distractor-2"), F(0.4), F(0.4), F(1.0), S("Semantic"),
S("weather"));
engram_connect(a, b, F(0.8), S("associate")); /* the edge under test */
engram_connect(a, c, F(0.3), S("associate"));
engram_connect(b, d, F(0.3), S("associate"));
}
static const char* QUERY =
"hebbian potentiation co-active memory links associative weight";
static void export_graph(const char* dir, const char* name){
char p[1024];
snprintf(p, sizeof p, "%s/%s", dir, name);
if (!engram_save(S(p))){ fprintf(stderr, "save %s failed\n", name); exit(2); }
}
int main(int argc, char** argv){
if (argc < 3){
fprintf(stderr, "usage: %s <pos_seed|pos_reboot|neg_seed|neg_reboot|offcheck> <dir>\n", argv[0]);
return 2;
}
const char* mode = argv[1];
const char* dir = argv[2];
if (!strcmp(mode, "pos_seed") || !strcmp(mode, "neg_seed")){
int persist = !strcmp(mode, "pos_seed");
if (!engram_store_enabled()){ fprintf(stderr, "%s requires ENGRAM_STORE=1\n", mode); return 2; }
if (!engram_store_boot(S(dir))){ fprintf(stderr, "store boot failed\n"); return 2; }
build_seed();
el_val_t act = engram_activate_json(S(QUERY), (el_val_t)3);
(void)act;
/* Capture the post-activation resident state BEFORE persisting/closing. */
export_graph(dir, persist ? "pre_reboot.json" : "neg_pre.json");
printf("[%s] nodes=%lld edges=%lld\n", mode,
(long long)(int64_t)engram_node_count(),
(long long)(int64_t)engram_edge_count());
if (persist){
if (!engram_store_checkpoint()){ fprintf(stderr, "checkpoint failed\n"); return 2; }
}
/* neg mode: NO field-persist checkpoint. engram_store_close still flushes
* pages, but no store_put_* ran post-creation, so the store keeps the
* pristine creation-time field values (hebb=0, activation_count=0). */
engram_store_close();
return 0;
}
if (!strcmp(mode, "pos_reboot") || !strcmp(mode, "neg_reboot")){
if (!engram_store_enabled()){ fprintf(stderr, "%s requires ENGRAM_STORE=1\n", mode); return 2; }
/* snapshot.json deleted by the runner — boot MUST come from neuron.egm. */
if (!engram_store_boot(S(dir))){ fprintf(stderr, "reboot boot failed\n"); return 2; }
export_graph(dir, !strcmp(mode, "pos_reboot") ? "reboot.json" : "neg_reboot.json");
printf("[%s] nodes=%lld edges=%lld\n", mode,
(long long)(int64_t)engram_node_count(),
(long long)(int64_t)engram_edge_count());
engram_store_close();
return 0;
}
if (!strcmp(mode, "offcheck")){
int en = engram_store_enabled();
el_val_t boot = engram_store_boot(S(dir)); /* no-op with flag off */
build_seed();
engram_activate_json(S(QUERY), (el_val_t)3);
el_val_t ck = engram_store_checkpoint(); /* must be a no-op */
printf("[offcheck] enabled=%d boot=%lld checkpoint=%lld\n",
en, (long long)(int64_t)boot, (long long)(int64_t)ck);
return (en == 0 && (int64_t)boot == 0 && (int64_t)ck == 0) ? 0 : 1;
}
fprintf(stderr, "unknown mode %s\n", mode);
return 2;
}
-155
View File
@@ -1,155 +0,0 @@
/* test_m3_parity.c — M3 JSON-parity gate for the ENGRAM_STORE wiring.
*
* This is a REAL el-level harness: it links the actual el_runtime.o (the soul's
* native engram builtins) + engram_store.o and calls the engram_node family plus
* engram_connect, engram_activate_json, engram_save, engram_store_boot directly. No EL interpreter
* and no full soul build are needed el_runtime.c compiles to a standalone .o
* whose engram builtins operate on the process-global engram store, and the
* string arena is inert unless el_request_start() is called, so the builtins are
* callable straight from C (el_val_t is int64_t; EL_STR/EL_CSTR are pointer casts).
*
* Modes (argv[1]), data dir (argv[2]):
* seed ENGRAM_STORE unset: build a fixed seed graph, write snapshot.json +
* off_graph.json (pristine, pre-activation), then activate off_act.json.
* on ENGRAM_STORE=1: engram_store_boot(dir) imports snapshot.json ONCE into
* neuron.egm and loads it resident; write on_graph.json, then activate
* on_act.json; checkpoint + close.
* reboot ENGRAM_STORE=1 with snapshot.json DELETED: boot must reload from
* neuron.egm (WAL replay), never re-reading JSON; write reboot_graph.json.
* offcheck assert flag-off leaves the store untouched.
*
* The graph comparison (done by run_m3_parity.sh via python, modulo ordering) is
* the deterministic gate; activation ids/promoted are compared as a robust set.
*/
#include "el_runtime.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* Builtins the header declares are pulled in via el_runtime.h. The M3 additions
* are not in the header yet, so declare them here. */
extern int engram_store_enabled(void);
extern el_val_t engram_store_boot(el_val_t data_dir);
extern el_val_t engram_store_checkpoint(void);
extern el_val_t engram_store_close(void);
extern el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t label,
el_val_t salience, el_val_t certainty, el_val_t confidence,
el_val_t status, el_val_t tags, el_val_t layer_id);
static el_val_t S(const char* s){ return EL_STR(s); }
static el_val_t F(double d){ return el_from_float(d); }
/* Build a fixed, deterministic seed graph: 12 nodes across two layers + 9 edges.
* Content is chosen so an activation query has real matches to rank. */
static void build_seed(void){
/* core-identity layer (1) via engram_node_full */
el_val_t n0 = engram_node_full(S("tiered storage engine design"), S("Concept"),
S("storage-engine"), F(0.9), F(0.8), F(1.0), S("Semantic"), S("design,storage"));
el_val_t n1 = engram_node_full(S("write-ahead log durability"), S("Concept"),
S("wal"), F(0.85), F(0.75), F(1.0), S("Semantic"), S("wal,durability"));
el_val_t n2 = engram_node_full(S("paged buffer pool with checkpointing"), S("Concept"),
S("buffer-pool"), F(0.8), F(0.7), F(1.0), S("Semantic"), S("paging"));
el_val_t n3 = engram_node_full(S("spreading activation over the graph"), S("Concept"),
S("activation"), F(0.8), F(0.7), F(1.0), S("Semantic"), S("activation,graph"));
el_val_t n4 = engram_node_full(S("hebbian co-activation potentiation"), S("Concept"),
S("hebbian"), F(0.7), F(0.6), F(1.0), S("Semantic"), S("hebb"));
el_val_t n5 = engram_node_full(S("crash recovery replays the log"), S("Concept"),
S("recovery"), F(0.75), F(0.65), F(1.0), S("Semantic"), S("recovery,wal"));
/* domain-knowledge layer (2) via engram_node_layered */
el_val_t n6 = engram_node_layered(S("b-tree primary index id to location"), S("Fact"),
S("btree"), F(0.7), F(0.6), F(1.0), S(""), S("index"), (el_val_t)2);
el_val_t n7 = engram_node_layered(S("adjacency index for edge lookup"), S("Fact"),
S("adjacency"), F(0.7), F(0.6), F(1.0), S(""), S("index,graph"), (el_val_t)2);
el_val_t n8 = engram_node_layered(S("slotted pages hold tlv records"), S("Fact"),
S("slotted-page"), F(0.65), F(0.55), F(1.0), S(""), S("format"), (el_val_t)2);
el_val_t n9 = engram_node_full(S("memory tiers working semantic episodic"), S("Concept"),
S("tiers"), F(0.7), F(0.6), F(1.0), S("Semantic"), S("tiers,memory"));
el_val_t n10 = engram_node_full(S("embeddings enable nearest neighbour search"), S("Concept"),
S("embeddings"), F(0.65), F(0.55), F(1.0), S("Semantic"), S("embeddings"));
el_val_t n11 = engram_node_full(S("the durable engram is the mind's memory"), S("Belief"),
S("engram"), F(0.95), F(0.9), F(1.0), S("Semantic"), S("engram,memory"));
engram_connect(n0, n1, F(0.8), S("depends-on"));
engram_connect(n0, n2, F(0.8), S("depends-on"));
engram_connect(n0, n3, F(0.7), S("enables"));
engram_connect(n1, n5, F(0.9), S("enables"));
engram_connect(n3, n4, F(0.6), S("triggers"));
engram_connect(n2, n6, F(0.7), S("uses"));
engram_connect(n3, n7, F(0.7), S("uses"));
engram_connect(n0, n8, F(0.6), S("uses"));
engram_connect(n11, n9, F(0.8), S("about"));
engram_connect(n11, n10, F(0.5), S("about"));
}
static void write_file(const char* path, const char* content){
FILE* f = fopen(path, "wb");
if (!f){ fprintf(stderr, "cannot open %s\n", path); exit(2); }
if (content) fwrite(content, 1, strlen(content), f);
fclose(f);
}
static const char* QUERY = "storage engine activation and the durable log";
int main(int argc, char** argv){
if (argc < 3){ fprintf(stderr, "usage: %s <seed|on|reboot|offcheck> <dir>\n", argv[0]); return 2; }
const char* mode = argv[1];
const char* dir = argv[2];
char p[1024];
if (!strcmp(mode, "seed")){
if (engram_store_enabled()){ fprintf(stderr, "seed mode requires ENGRAM_STORE unset\n"); return 2; }
build_seed();
snprintf(p, sizeof p, "%s/snapshot.json", dir);
if (!engram_save(S(p))){ fprintf(stderr, "seed save failed\n"); return 2; }
snprintf(p, sizeof p, "%s/off_graph.json", dir);
engram_save(S(p)); /* pristine off-path graph */
el_val_t act = engram_activate_json(S(QUERY), (el_val_t)3);
snprintf(p, sizeof p, "%s/off_act.json", dir);
write_file(p, EL_CSTR(act));
printf("[seed] nodes=%lld edges=%lld\n",
(long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count());
return 0;
}
if (!strcmp(mode, "on")){
if (!engram_store_enabled()){ fprintf(stderr, "on mode requires ENGRAM_STORE=1\n"); return 2; }
if (!engram_store_boot(S(dir))){ fprintf(stderr, "store boot failed\n"); return 2; }
snprintf(p, sizeof p, "%s/on_graph.json", dir);
engram_save(S(p)); /* export resident (== store) */
/* Checkpoint the freshly-imported (pristine) graph — this is the state
* the reboot comparison expects to round-trip. Under M3.5 a checkpoint
* persists the resident graph's CURRENT field state, so it must run
* BEFORE activation mutates fields in place; activation itself is
* exercised below only for the activation-result-set parity check. The
* M3.5 gate (test_m35_hebb_persist) separately proves that a checkpoint
* taken AFTER activation durably carries the learned hebb/WM state. */
engram_store_checkpoint();
el_val_t act = engram_activate_json(S(QUERY), (el_val_t)3);
snprintf(p, sizeof p, "%s/on_act.json", dir);
write_file(p, EL_CSTR(act));
printf("[on] nodes=%lld edges=%lld\n",
(long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count());
engram_store_close();
return 0;
}
if (!strcmp(mode, "reboot")){
if (!engram_store_enabled()){ fprintf(stderr, "reboot mode requires ENGRAM_STORE=1\n"); return 2; }
/* snapshot.json has been deleted by the runner — boot MUST come from
* neuron.egm (+ WAL replay), never re-reading JSON. */
if (!engram_store_boot(S(dir))){ fprintf(stderr, "reboot boot failed\n"); return 2; }
snprintf(p, sizeof p, "%s/reboot_graph.json", dir);
engram_save(S(p));
printf("[reboot] nodes=%lld edges=%lld\n",
(long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count());
engram_store_close();
return 0;
}
if (!strcmp(mode, "offcheck")){
/* ENGRAM_STORE unset: enabled()==0 and boot is a no-op returning 0. */
int en = engram_store_enabled();
el_val_t b = engram_store_boot(S(dir));
printf("[offcheck] enabled=%d boot_ret=%lld\n", en, (long long)(int64_t)b);
return (en == 0 && (int64_t)b == 0) ? 0 : 1;
}
fprintf(stderr, "unknown mode %s\n", mode);
return 2;
}
-439
View File
@@ -1,439 +0,0 @@
/* test_store.c — M1 gate for the engram paged store (engram_store.{c,h}).
*
* Pure C. Build: gcc -O2 test_store.c ../../lang/runtime/engram_store.c -o test_store
* Writes ONLY under a throwaway /tmp dir. Never touches ~/.neuron or live ports.
*
* Covers §7 M1 gates: round-trip (5k nodes / 20k edges, all fields, emb bit-exact,
* hebb, >page content), TLV forward-compat, overflow chains, B+-tree indexes
* across splits, free-list reuse, and corruption/superblock recovery.
*/
#include "../../lang/runtime/engram_store.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/stat.h>
static int g_pass = 0, g_fail = 0;
static void ok(const char* name, int cond){
printf(" [%s] %s\n", cond ? "PASS" : "FAIL", name);
if (cond) g_pass++; else g_fail++;
}
static char g_dir[512];
static void mk_dir(void){
snprintf(g_dir, sizeof g_dir, "/tmp/engram-store-test-%d", (int)getpid());
mkdir(g_dir, 0700);
}
static void path_in(char* out, size_t cap, const char* name){
snprintf(out, cap, "%s/%s", g_dir, name);
}
static long file_size(const char* p){ struct stat st; return stat(p,&st)==0 ? (long)st.st_size : -1; }
/* ── deterministic RNG so oracle nodes/edges regenerate bit-exact ─────────── */
static uint64_t xs(uint64_t* s){ uint64_t x=*s; x^=x<<13; x^=x>>7; x^=x<<17; *s=x; return x; }
static uint64_t node_seed(int i){ return 0x9E3779B97F4A7C15ULL ^ ((uint64_t)(i+1)*0xD1B54A32D192ED03ULL); }
static uint64_t edge_seed(int i){ return 0xC2B2AE3D27D4EB4FULL ^ ((uint64_t)(i+1)*0x165667B19E3779F9ULL); }
static char* rnd_str(uint64_t* st, size_t len){
char* s = (char*)malloc(len + 1);
for (size_t i=0;i<len;i++) s[i] = (char)(33 + (xs(st) % 94)); /* printable, no NUL */
s[len] = 0; return s;
}
/* NODE_COUNT nodes; a slice have >page content to force overflow chains. */
#define NODE_COUNT 5000
#define EDGE_COUNT 20000
#define EMB_DIM 768
static void gen_node(int i, StoreNode* n){
memset(n, 0, sizeof *n);
uint64_t st = node_seed(i);
char id[32]; snprintf(id, sizeof id, "node-%d", i);
n->id = strdup(id);
size_t clen = (i % 500 == 0) ? (size_t)(17000 + (xs(&st) % 6000)) : (size_t)(xs(&st) % 300);
n->content = rnd_str(&st, clen);
n->node_type = rnd_str(&st, 4 + (xs(&st) % 8));
n->label = (i % 2) ? rnd_str(&st, 3 + (xs(&st) % 10)) : NULL;
n->tier = rnd_str(&st, 4 + (xs(&st) % 6));
n->tags = rnd_str(&st, xs(&st) % 40);
n->metadata = (i % 3) ? rnd_str(&st, xs(&st) % 60) : NULL;
n->salience = (double)(xs(&st) % 1000000) / 997.0;
n->importance = (double)(xs(&st) % 1000000) / 131.0;
n->confidence = (double)(xs(&st) % 1000000) / 733.0;
n->temporal_decay_rate = (double)(xs(&st) % 1000000) / 101.0;
n->activation_count = (int64_t)(xs(&st) % 100000);
n->last_activated = (int64_t)xs(&st);
n->created_at = (int64_t)(1600000000000LL + i);
n->updated_at = (int64_t)xs(&st);
n->background_activation = (double)(xs(&st) % 1000000) / 17.0;
n->working_memory_weight = (double)(xs(&st) % 1000000) / 29.0;
n->suppression_count = (int32_t)(xs(&st) % 50);
n->layer_id = (uint32_t)(xs(&st) % 5);
for (int k=0;k<STORE_BLL_K;k++) n->access_ts[k] = (int64_t)xs(&st);
n->access_head = (int32_t)(xs(&st) % STORE_BLL_K);
n->access_filled = (int32_t)(xs(&st) % (STORE_BLL_K + 1));
n->wm_anchor = (double)(xs(&st) % 1000000) / 3.0;
n->emb = (float*)malloc(EMB_DIM * sizeof(float));
for (int k=0;k<EMB_DIM;k++){ uint32_t u=(uint32_t)xs(&st); memcpy(&n->emb[k], &u, 4); }
n->emb_dim = EMB_DIM;
}
static void gen_edge(int i, StoreEdge* e){
memset(e, 0, sizeof *e);
uint64_t st = edge_seed(i);
char id[32], from[32], to[32];
snprintf(id, sizeof id, "edge-%d", i);
snprintf(from, sizeof from, "node-%d", (int)(xs(&st) % NODE_COUNT));
snprintf(to, sizeof to, "node-%d", (int)(xs(&st) % NODE_COUNT));
e->id = strdup(id); e->from_id = strdup(from); e->to_id = strdup(to);
e->relation = rnd_str(&st, 3 + (xs(&st) % 12));
e->metadata = (i % 4) ? rnd_str(&st, xs(&st) % 40) : NULL;
e->weight = (double)(xs(&st) % 1000000) / 111.0;
e->hebb = (double)(xs(&st) % 1000000) / 1000000.0; /* the learned field */
e->confidence = (double)(xs(&st) % 1000000) / 777.0;
e->created_at = (int64_t)(1600000000000LL + i);
e->updated_at = (int64_t)xs(&st);
e->last_fired = (int64_t)xs(&st);
e->inhibitory = (int32_t)(xs(&st) % 2);
e->layer_id = (uint32_t)(xs(&st) % 5);
}
static int streq(const char* a, const char* b){
if (!a && !b) return 1;
if (!a || !b) return 0;
return strcmp(a,b)==0;
}
static int cmp_node(const StoreNode* a, const StoreNode* b){
if (!streq(a->id,b->id) || !streq(a->content,b->content) ||
!streq(a->node_type,b->node_type) || !streq(a->label,b->label) ||
!streq(a->tier,b->tier) || !streq(a->tags,b->tags) ||
!streq(a->metadata,b->metadata)) return 0;
if (a->salience!=b->salience || a->importance!=b->importance ||
a->confidence!=b->confidence || a->temporal_decay_rate!=b->temporal_decay_rate ||
a->activation_count!=b->activation_count || a->last_activated!=b->last_activated ||
a->created_at!=b->created_at || a->updated_at!=b->updated_at ||
a->background_activation!=b->background_activation ||
a->working_memory_weight!=b->working_memory_weight ||
a->suppression_count!=b->suppression_count || a->layer_id!=b->layer_id ||
a->access_head!=b->access_head || a->access_filled!=b->access_filled ||
a->wm_anchor!=b->wm_anchor || a->emb_dim!=b->emb_dim) return 0;
for (int k=0;k<STORE_BLL_K;k++) if (a->access_ts[k]!=b->access_ts[k]) return 0;
if ((a->emb==NULL) != (b->emb==NULL)) return 0;
if (a->emb && memcmp(a->emb, b->emb, (size_t)a->emb_dim*4)!=0) return 0;
return 1;
}
static int cmp_edge(const StoreEdge* a, const StoreEdge* b){
if (!streq(a->id,b->id) || !streq(a->from_id,b->from_id) || !streq(a->to_id,b->to_id) ||
!streq(a->relation,b->relation) || !streq(a->metadata,b->metadata)) return 0;
if (a->weight!=b->weight || a->hebb!=b->hebb || a->confidence!=b->confidence ||
a->created_at!=b->created_at || a->updated_at!=b->updated_at ||
a->last_fired!=b->last_fired || a->inhibitory!=b->inhibitory ||
a->layer_id!=b->layer_id) return 0;
return 1;
}
static void free_node_fields(StoreNode* n){
free(n->id); free(n->content); free(n->node_type); free(n->label);
free(n->tier); free(n->tags); free(n->metadata); free(n->emb); free(n->unknown);
}
static void free_edge_fields(StoreEdge* e){
free(e->id); free(e->from_id); free(e->to_id); free(e->relation); free(e->metadata); free(e->unknown);
}
/* Flip one byte in the store file at (page*PAGE_SIZE + off). */
static void flip_byte(const char* path, uint64_t page, size_t off){
int fd = open(path, O_RDWR);
uint8_t b; off_t at = (off_t)page*STORE_PAGE_SIZE + off;
pread(fd, &b, 1, at); b ^= 0xFF; pwrite(fd, &b, 1, at); close(fd);
}
/* ════════════════════════════════════════════════════════════════════════ */
static void test_roundtrip(void){
printf("\n== round-trip: %d nodes + %d edges, all fields, emb bit-exact ==\n", NODE_COUNT, EDGE_COUNT);
char path[600]; path_in(path, sizeof path, "roundtrip.store");
unlink(path);
EngramPagedStore* s = store_create(path);
ok("store_create", s != NULL);
if (!s) return;
for (int i=0;i<NODE_COUNT;i++){ StoreNode n; gen_node(i,&n);
if (store_put_node(s,&n)!=0){ ok("put_node", 0); free_node_fields(&n); store_close(s); return; }
free_node_fields(&n); }
for (int i=0;i<EDGE_COUNT;i++){ StoreEdge e; gen_edge(i,&e);
if (store_put_edge(s,&e)!=0){ ok("put_edge", 0); free_edge_fields(&e); store_close(s); return; }
free_edge_fields(&e); }
ok("wrote all nodes+edges", 1);
store_close(s);
long sz = file_size(path);
printf(" store file size: %ld bytes (%.2f MB) for %d nodes / %d edges\n",
sz, sz/1048576.0, NODE_COUNT, EDGE_COUNT);
s = store_open(path);
ok("store_open (reopen)", s != NULL);
if (!s) return;
int nbad = 0;
for (int i=0;i<NODE_COUNT;i++){
StoreNode want; gen_node(i,&want);
StoreNode got; int r = store_get_node(s, want.id, &got);
if (r!=1 || !cmp_node(&want,&got) || got.unknown_len!=0) nbad++;
if (r==1) store_node_free(&got);
free_node_fields(&want);
}
ok("all 5000 nodes read back bit-exact (incl emb, all fields)", nbad==0);
if (nbad) printf(" %d node mismatches\n", nbad);
int ebad = 0;
for (int i=0;i<EDGE_COUNT;i++){
StoreEdge want; gen_edge(i,&want);
StoreEdge* got; size_t gn;
int found = 0;
if (store_get_edges_from(s, want.from_id, &got, &gn)==0){
for (size_t j=0;j<gn;j++) if (streq(got[j].id, want.id)){ if (cmp_edge(&want,&got[j])) found=1; break; }
store_edges_free(got, gn);
}
if (!found) ebad++;
free_edge_fields(&want);
}
ok("all 20000 edges read back via adjacency, all fields incl hebb", ebad==0);
if (ebad) printf(" %d edge mismatches\n", ebad);
ok("store_check crc clean after round-trip", store_check(s, STORE_CHECK_CRC)==0);
store_close(s);
}
static void test_forward_compat(void){
printf("\n== TLV forward-compat: omit field defaults; unknown tag preserved ==\n");
char path[600]; path_in(path, sizeof path, "fwd.store");
unlink(path);
EngramPagedStore* s = store_create(path);
/* Writer OMITS several fields (metadata, label, emb) → reader must default. */
StoreNode a; memset(&a,0,sizeof a);
a.id = strdup("omit-1"); a.content = strdup("has content"); a.tier = strdup("core");
a.salience = 0.5; /* metadata/label NULL, emb NULL */
store_put_node(s, &a); free(a.id); free(a.content); free(a.tier);
StoreNode g; int r = store_get_node(s, "omit-1", &g);
ok("omitted string fields default to NULL", r==1 && g.metadata==NULL && g.label==NULL);
ok("omitted emb defaults to NULL / emb_dim 0", r==1 && g.emb==NULL && g.emb_dim==0);
ok("present fields intact", r==1 && streq(g.content,"has content") && g.salience==0.5);
if (r==1) store_node_free(&g);
/* Writer includes an UNKNOWN tag (simulating a newer writer / field the
* reader does not model) via the `unknown` passthrough. Reader (which also
* models known fields A,B,C) must preserve it verbatim. */
uint8_t unk[64];
unk[0] = 200; /* a tag this build has no case for */
/* [u8 tag][u32 len][bytes] */
unk[1]=8; unk[2]=0; unk[3]=0; unk[4]=0;
for (int i=0;i<8;i++) unk[5+i] = (uint8_t)(0xA0 + i);
StoreNode b; memset(&b,0,sizeof b);
b.id = strdup("unk-1"); b.content = strdup("known field B"); b.confidence = 0.9; /* known field C-ish */
b.unknown = unk; b.unknown_len = 5 + 8;
store_put_node(s, &b); free(b.id); free(b.content);
StoreNode g2; int r2 = store_get_node(s, "unk-1", &g2);
int unk_ok = r2==1 && g2.unknown_len==(5+8) && memcmp(g2.unknown, unk, 5+8)==0;
ok("unknown tag preserved verbatim on read", unk_ok);
ok("known fields still read while unknown preserved", r2==1 && streq(g2.content,"known field B") && g2.confidence==0.9);
if (r2==1) store_node_free(&g2);
store_close(s);
}
static void test_overflow(void){
printf("\n== overflow: 100KB content node + emb via overflow chain ==\n");
char path[600]; path_in(path, sizeof path, "ovf.store");
unlink(path);
EngramPagedStore* s = store_create(path);
size_t big = 100*1024;
StoreNode n; memset(&n,0,sizeof n);
n.id = strdup("big-1");
n.content = (char*)malloc(big+1);
for (size_t i=0;i<big;i++) n.content[i] = (char)(33 + (i % 94));
n.content[big] = 0;
n.tier = strdup("episodic");
n.emb = (float*)malloc(EMB_DIM*sizeof(float));
for (int k=0;k<EMB_DIM;k++){ float f = (float)(k*0.5 - 100.0); n.emb[k]=f; }
n.emb_dim = EMB_DIM;
ok("put 100KB+emb node", store_put_node(s,&n)==0);
store_close(s);
s = store_open(path);
StoreNode g; int r = store_get_node(s, "big-1", &g);
ok("reopen + read big node", r==1);
ok("100KB content byte-exact via overflow", r==1 && strlen(g.content)==big && memcmp(g.content,n.content,big)==0);
ok("emb bit-exact via overflow record", r==1 && g.emb_dim==EMB_DIM && memcmp(g.emb,n.emb,EMB_DIM*4)==0);
if (r==1) store_node_free(&g);
ok("store_check clean (overflow pages crc'd)", store_check(s, STORE_CHECK_CRC)==0);
store_close(s);
free_node_fields(&n);
}
static void test_index_splits(void){
printf("\n== B+-tree index correctness across many splits ==\n");
char path[600]; path_in(path, sizeof path, "idx.store");
unlink(path);
EngramPagedStore* s = store_create(path);
/* Tiny order forces deep leaf + internal splits with only a few hundred keys. */
store__set_btree_order(s, 4, 4);
const int N = 600;
for (int i=0;i<N;i++){
StoreNode n; memset(&n,0,sizeof n);
char id[32]; snprintf(id,sizeof id,"k-%05d", (i*37+11)%100000); /* scattered keys */
n.id = strdup(id); n.content = strdup("x"); n.tier=strdup("t"); n.salience=i;
if (store_put_node(s,&n)!=0){ ok("put",0); }
free(n.id); free(n.content); free(n.tier);
}
int miss=0;
for (int i=0;i<N;i++){
char id[32]; snprintf(id,sizeof id,"k-%05d",(i*37+11)%100000);
StoreNode g; int r = store_get_node(s, id, &g);
if (r!=1 || (int)g.salience != i) miss++;
if (r==1) store_node_free(&g);
}
ok("all keys retrievable after leaf+internal splits", miss==0);
if (miss) printf(" %d misses\n", miss);
StoreNode g; ok("absent key returns 0", store_get_node(s,"k-NOPE",&g)==0);
/* Adjacency: controlled star + chain, exact edge sets. */
for (int i=0;i<50;i++){
StoreEdge e; memset(&e,0,sizeof e);
char id[32]; snprintf(id,sizeof id,"e-%d",i);
e.id=strdup(id); e.from_id=strdup("HUB"); char tt[16]; snprintf(tt,sizeof tt,"T-%d",i); e.to_id=strdup(tt);
e.relation=strdup("r"); e.weight=1.0; e.hebb=0.1*i;
store_put_edge(s,&e); free_edge_fields(&e);
}
for (int i=0;i<7;i++){
StoreEdge e; memset(&e,0,sizeof e);
char id[32]; snprintf(id,sizeof id,"in-%d",i);
char ff[16]; snprintf(ff,sizeof ff,"S-%d",i);
e.id=strdup(id); e.from_id=strdup(ff); e.to_id=strdup("SINK");
e.relation=strdup("r"); e.weight=1.0;
store_put_edge(s,&e); free_edge_fields(&e);
}
StoreEdge* out; size_t on;
store_get_edges_from(s,"HUB",&out,&on);
ok("get_edges_from(HUB) == 50", on==50);
store_edges_free(out,on);
store_get_edges_to(s,"SINK",&out,&on);
ok("get_edges_to(SINK) == 7", on==7);
store_edges_free(out,on);
store_get_edges_to(s,"HUB",&out,&on);
ok("get_edges_to(HUB) == 0 (direction separation)", on==0);
store_edges_free(out,on);
ok("store_check clean", store_check(s, STORE_CHECK_CRC)==0);
store_close(s);
}
static void test_freelist(void){
printf("\n== free-list: tombstone reclaims pages, graph stays consistent ==\n");
char path[600]; path_in(path, sizeof path, "free.store");
unlink(path);
EngramPagedStore* s = store_create(path);
uint64_t pc0 = store_page_count(s);
const int N = 300;
for (int i=0;i<N;i++){
StoreNode n; memset(&n,0,sizeof n);
char id[32]; snprintf(id,sizeof id,"a-%d",i);
n.id=strdup(id); n.content=rnd_str(&(uint64_t){node_seed(i)}, 200); n.tier=strdup("t");
store_put_node(s,&n); free_node_fields(&n);
}
uint64_t pc1 = store_page_count(s);
uint64_t node_pages = pc1 - pc0;
ok("initial batch consumed pages", node_pages > 0);
for (int i=0;i<N;i++){ char id[32]; snprintf(id,sizeof id,"a-%d",i); store_tombstone(s,id); }
/* all old nodes gone */
int gone=1; for (int i=0;i<N;i++){ char id[32]; snprintf(id,sizeof id,"a-%d",i);
StoreNode g; if (store_get_node(s,id,&g)==1){ gone=0; store_node_free(&g); } }
ok("tombstoned nodes now absent", gone);
for (int i=0;i<N;i++){
StoreNode n; memset(&n,0,sizeof n);
char id[32]; snprintf(id,sizeof id,"b-%d",i);
n.id=strdup(id); n.content=strdup("reused"); n.tier=strdup("t"); n.salience=i;
store_put_node(s,&n); free_node_fields(&n);
}
uint64_t pc2 = store_page_count(s);
/* Reuse proven: growth for the 2nd batch is far less than a fresh alloc. */
ok("freed pages reused (no full re-growth)", pc2 < pc1 + node_pages);
printf(" pages: base=%llu after1=%llu after2=%llu (node_pages=%llu)\n",
(unsigned long long)pc0,(unsigned long long)pc1,(unsigned long long)pc2,(unsigned long long)node_pages);
int newbad=0; for (int i=0;i<N;i++){ char id[32]; snprintf(id,sizeof id,"b-%d",i);
StoreNode g; if (store_get_node(s,id,&g)!=1 || (int)g.salience!=i) newbad++; else store_node_free(&g); }
ok("new batch fully readable after reuse", newbad==0);
ok("store_check clean after reuse", store_check(s, STORE_CHECK_CRC)==0);
store_close(s);
/* survives reopen */
s = store_open(path);
int rb=0; for (int i=0;i<N;i++){ char id[32]; snprintf(id,sizeof id,"b-%d",i);
StoreNode g; if (store_get_node(s,id,&g)!=1) rb++; else store_node_free(&g); }
ok("graph consistent across reopen after reuse", rb==0);
store_close(s);
}
static void test_corruption(void){
printf("\n== corruption: crc detection + superblock mirror recovery ==\n");
char path[600]; path_in(path, sizeof path, "corrupt.store");
unlink(path);
EngramPagedStore* s = store_create(path);
for (int i=0;i<50;i++){ StoreNode n; gen_node(i,&n); store_put_node(s,&n); free_node_fields(&n); }
store_close(s);
s = store_open(path);
ok("clean store: store_check == 0", store_check(s, STORE_CHECK_CRC)==0);
store_close(s);
/* flip a byte inside a data page (page 5 is node/index data, never a SB) */
flip_byte(path, 5, 137);
s = store_open(path);
ok("store_open still succeeds (data-page corruption)", s != NULL);
int bad = store_check(s, STORE_CHECK_CRC);
ok("store_check detects corrupted page via crc", bad >= 1);
printf(" store_check reported %d corrupt page(s)\n", bad);
store_close(s);
/* fresh store, corrupt superblock 0, must recover via mirror superblock 1 */
char p2[600]; path_in(p2, sizeof p2, "sbrec.store");
unlink(p2);
s = store_create(p2);
StoreNode n; gen_node(42,&n); store_put_node(s,&n);
store_close(s);
/* trash magic + crc region of page 0 */
flip_byte(p2, 0, 0); flip_byte(p2, 0, 1); flip_byte(p2, 0, 90);
s = store_open(p2);
ok("open recovers via mirror superblock (page 1)", s != NULL);
if (s){
StoreNode g; int r = store_get_node(s, "node-42", &g);
ok("data intact after superblock recovery", r==1 && cmp_node(&n,&g));
if (r==1) store_node_free(&g);
store_close(s);
}
free_node_fields(&n);
}
int main(void){
mk_dir();
printf("engram_store M1 test harness — dir=%s\n", g_dir);
test_roundtrip();
test_forward_compat();
test_overflow();
test_index_splits();
test_freelist();
test_corruption();
printf("\n================ %d passed, %d failed ================\n", g_pass, g_fail);
return g_fail ? 1 : 0;
}
-473
View File
@@ -1,473 +0,0 @@
/* test_wal.c — unit + integration + crash-fuzz harness for the engram WAL.
*
* Includes el_runtime.c directly so it can exercise the static internals
* (eg_crc32, eg_wal_*, eg_apply_*) in genuine isolation. Build:
* cc -O2 -fbracket-depth=1024 -I<release-dir> test_wal.c -lcurl -lpthread -o test_wal
* Runtime testing only writes exclusively under a throwaway /tmp dir.
*/
#define ENGRAM_TEST_BUILD 1
#include "el_runtime.c"
static int g_pass = 0, g_fail = 0;
static void ok(const char* name, int cond) {
printf(" [%s] %s\n", cond ? "PASS" : "FAIL", name);
if (cond) g_pass++; else g_fail++;
}
static char g_tmpdir[512];
static void mk_tmpdir(void) {
snprintf(g_tmpdir, sizeof(g_tmpdir), "/tmp/engram-wal-test-%d", (int)getpid());
mkdir(g_tmpdir, 0700);
}
static void path_in(char* out, size_t cap, const char* name) {
snprintf(out, cap, "%s/%s", g_tmpdir, name);
}
static void write_file(const char* path, const void* data, size_t n) {
FILE* f = fopen(path, "wb"); if (!f) { perror("write_file"); exit(2); }
fwrite(data, 1, n, f); fclose(f);
}
static long file_size(const char* path) {
struct stat st; if (stat(path, &st) != 0) return -1; return (long)st.st_size;
}
static void reset_store(void) {
char p[600]; path_in(p, sizeof(p), "_reset.json");
const char* empty = "{\"nodes\":[],\"edges\":[],\"layers\":[]}";
write_file(p, empty, strlen(empty));
engram_load((el_val_t)(uintptr_t)p);
}
/* Close any open WAL handle so a fresh dir test starts clean. */
static void wal_close(void) {
if (eg_wal.fp) { fclose(eg_wal.fp); eg_wal.fp = NULL; }
eg_wal.path[0] = 0; eg_wal.lsn = 0; eg_wal.bytes = 0; eg_wal.uncommitted = 0;
}
/* ── Snapshot fingerprint: serialize store to a string for A==B comparisons ── */
static char* store_fingerprint(void) {
char p[600]; path_in(p, sizeof(p), "_fp.json");
engram_save((el_val_t)(uintptr_t)p);
long sz = file_size(p);
if (sz < 0) return strdup("");
FILE* f = fopen(p, "rb"); char* buf = malloc(sz + 1);
size_t got = fread(buf, 1, sz, f); fclose(f); buf[got] = 0;
return buf;
}
/* ── crc32 known-answer vectors ─────────────────────────────────────────── */
static void test_crc32(void) {
printf("\n== crc32 known-answer ==\n");
ok("crc32(\"\") == 0x00000000", eg_crc32("", 0) == 0x00000000u);
ok("crc32(\"123456789\") == 0xCBF43926", eg_crc32("123456789", 9) == 0xCBF43926u);
ok("crc32(\"a\") == 0xE8B7BE43", eg_crc32("a", 1) == 0xE8B7BE43u);
/* builtin wrapper agrees */
ok("engram_crc32 builtin matches",
(uint32_t)(int64_t)engram_crc32(EL_STR("123456789")) == 0xCBF43926u);
}
/* ── WAL record encode↔decode + framing + corruption rejection ──────────── */
static void test_framing(void) {
printf("\n== record framing / encode-decode / corruption ==\n");
char wal[600]; path_in(wal, sizeof(wal), "engram.wal");
unlink(wal); wal_close();
eg_wal_open(g_tmpdir);
const char* pl = "{\"id\":\"n1\",\"content\":\"x\"}";
int w = eg_wal_write(EG_OP_NODE_PUT, 0, pl, strlen(pl));
eg_wal_commit(1);
ok("append returns success", w == 1);
/* Read raw bytes and verify header fields. */
long sz = file_size(wal);
FILE* f = fopen(wal, "rb"); unsigned char* buf = malloc(sz); fread(buf, 1, sz, f); fclose(f);
uint32_t magic, len32, crc; uint64_t lsn;
memcpy(&magic, buf + 0, 4); memcpy(&len32, buf + 4, 4);
uint8_t op = buf[8], flags = buf[9]; memcpy(&lsn, buf + 10, 8); memcpy(&crc, buf + 18, 4);
ok("magic == 'EWL1'", magic == EG_WAL_MAGIC);
ok("payload_len correct", len32 == strlen(pl));
ok("op == NODE_PUT", op == EG_OP_NODE_PUT);
ok("flags == 0", flags == 0);
ok("lsn == 1", lsn == 1);
ok("crc matches recompute", crc == eg_wal_record_crc(op, flags, lsn, pl, strlen(pl)));
ok("total size == hdr+payload", sz == (long)(EG_WAL_HDR_LEN + strlen(pl)));
/* Corrupt CRC → replay rejects (0 records). */
{ char bad[600]; path_in(bad, sizeof(bad), "bad_crc.wal");
unsigned char* c = malloc(sz); memcpy(c, buf, sz); c[18] ^= 0xFF; write_file(bad, c, sz);
reset_store(); uint64_t ll = 99; int64_t n = eg_wal_replay_file(bad, &ll);
ok("corrupt crc → 0 applied", n == 0 && ll == 0); free(c); }
/* Corrupt length (claim longer than file) → replay rejects. */
{ char bad[600]; path_in(bad, sizeof(bad), "bad_len.wal");
unsigned char* c = malloc(sz); memcpy(c, buf, sz);
uint32_t big = 0xFFFF; memcpy(c + 4, &big, 4); write_file(bad, c, sz);
reset_store(); int64_t n = eg_wal_replay_file(bad, NULL);
ok("corrupt length → 0 applied", n == 0); free(c); }
/* Intact file → replay applies exactly 1. */
{ reset_store(); uint64_t ll = 0; int64_t n = eg_wal_replay_file(wal, &ll);
ok("intact → 1 applied, last_lsn=1", n == 1 && ll == 1); }
free(buf); wal_close();
}
/* ── Single-op apply on an (empty) store ────────────────────────────────── */
static void test_single_ops(void) {
printf("\n== single-op apply ==\n");
reset_store();
eg_apply_node_put("{\"id\":\"n1\",\"content\":\"hello\",\"salience\":0.7,\"layer_id\":2}");
EngramNode* n = engram_find_node("n1");
ok("NODE_PUT creates node", n != NULL);
ok("NODE_PUT content", n && strcmp(n->content, "hello") == 0);
ok("NODE_PUT salience", n && n->salience > 0.69 && n->salience < 0.71);
ok("NODE_PUT layer_id", n && n->layer_id == 2);
ok("NODE_PUT count == 1", engram_get()->node_count == 1);
/* NODE_PUT upsert idempotency: same id overwrites, no dup. */
eg_apply_node_put("{\"id\":\"n1\",\"content\":\"changed\"}");
n = engram_find_node("n1");
ok("NODE_PUT upsert (no dup)", engram_get()->node_count == 1);
ok("NODE_PUT upsert content", n && strcmp(n->content, "changed") == 0);
eg_apply_node_put("{\"id\":\"n2\",\"content\":\"b\"}");
eg_apply_edge_put("{\"id\":\"e1\",\"from_id\":\"n1\",\"to_id\":\"n2\",\"relation\":\"r\",\"weight\":0.4,\"hebb\":0.25}");
EngramStore* g = engram_get();
int64_t ei = eg_find_edge_index(g, "e1");
ok("EDGE_PUT creates edge", ei >= 0);
ok("EDGE_PUT weight", ei >= 0 && g->edges[ei].weight > 0.39 && g->edges[ei].weight < 0.41);
ok("EDGE_PUT hebb", ei >= 0 && g->edges[ei].hebb > 0.24 && g->edges[ei].hebb < 0.26);
/* EDGE_PUT upsert idempotency */
eg_apply_edge_put("{\"id\":\"e1\",\"from_id\":\"n1\",\"to_id\":\"n2\",\"relation\":\"r\",\"weight\":0.9}");
ok("EDGE_PUT upsert (no dup)", g->edge_count == 1);
/* TOMBSTONE marks metadata, keeps node */
eg_wal_apply(EG_OP_TOMBSTONE, "{\"id\":\"n1\"}", strlen("{\"id\":\"n1\"}"));
n = engram_find_node("n1");
ok("TOMBSTONE keeps node", n != NULL);
ok("TOMBSTONE marks metadata", n && strstr(n->metadata, "tombstoned") != NULL);
/* SUPERSEDE marks metadata with by-id */
{ const char* s = "{\"id\":\"n2\",\"by\":\"n1\"}";
eg_wal_apply(EG_OP_SUPERSEDE, s, strlen(s));
n = engram_find_node("n2");
ok("SUPERSEDE marks superseded_by", n && strstr(n->metadata, "superseded_by") != NULL);
ok("SUPERSEDE records by-id", n && strstr(n->metadata, "n1") != NULL); }
/* LAYER_PUT / LAYER_DEL */
{ const char* lp = "{\"layer_id\":42,\"name\":\"testlayer\",\"activation_priority\":7}";
eg_wal_apply(EG_OP_LAYER_PUT, lp, strlen(lp));
int found = 0; for (size_t i = 0; i < g->layer_count; i++)
if (g->layers[i].layer_id == 42 && g->layers[i].name && strcmp(g->layers[i].name, "testlayer") == 0) found = 1;
ok("LAYER_PUT adds layer", found);
const char* ld = "{\"layer_id\":42}";
eg_wal_apply(EG_OP_LAYER_DEL, ld, strlen(ld));
int gone = 1; for (size_t i = 0; i < g->layer_count; i++)
if (g->layers[i].layer_id == 42 && g->layers[i].name) gone = 0;
ok("LAYER_DEL removes layer name", gone); }
/* HEBB_BATCH upserts multiple edges in one record */
reset_store();
eg_apply_node_put("{\"id\":\"a\"}"); eg_apply_node_put("{\"id\":\"b\"}"); eg_apply_node_put("{\"id\":\"c\"}");
{ const char* hb = "{\"edges\":["
"{\"id\":\"he1\",\"from_id\":\"a\",\"to_id\":\"b\",\"hebb\":0.1},"
"{\"id\":\"he2\",\"from_id\":\"b\",\"to_id\":\"c\",\"hebb\":0.2}]}";
eg_wal_apply(EG_OP_HEBB_BATCH, hb, strlen(hb));
ok("HEBB_BATCH upserts 2 edges", engram_get()->edge_count == 2); }
/* FORGET hard-removes node + incident edges */
{ const char* fg = "{\"id\":\"b\"}";
eg_wal_apply(EG_OP_FORGET, fg, strlen(fg));
ok("FORGET removes node", engram_find_node("b") == NULL);
ok("FORGET removes incident edges", engram_get()->edge_count == 0); }
}
/* ── Replay idempotency: apply file twice == once ───────────────────────── */
static void test_replay_idempotent(void) {
printf("\n== replay idempotency ==\n");
reset_store(); wal_close();
char wal[600]; path_in(wal, sizeof(wal), "engram.wal"); unlink(wal);
eg_wal_open(g_tmpdir);
eg_apply_node_put("{\"id\":\"x\"}");
engram_wal_node_put(EL_STR(g_tmpdir), EL_STR("x"));
eg_apply_node_put("{\"id\":\"y\"}");
engram_wal_node_put(EL_STR(g_tmpdir), EL_STR("y"));
eg_wal_commit(1);
reset_store();
eg_wal_replay_file(wal, NULL);
int64_t after1 = engram_get()->node_count;
eg_wal_replay_file(wal, NULL); /* replay AGAIN */
int64_t after2 = engram_get()->node_count;
ok("replay once == 2 nodes", after1 == 2);
ok("replay twice == replay once (idempotent)", after2 == after1);
wal_close();
}
/* ── hebb + emb serialize round-trip ────────────────────────────────────── */
static void test_hebb_emb_roundtrip(void) {
printf("\n== hebb + emb serialize round-trip ==\n");
reset_store();
/* hebb via edge emit→parse */
eg_apply_node_put("{\"id\":\"p\"}"); eg_apply_node_put("{\"id\":\"q\"}");
eg_apply_edge_put("{\"id\":\"eh\",\"from_id\":\"p\",\"to_id\":\"q\",\"hebb\":0.123456}");
EngramStore* g = engram_get();
int64_t ei = eg_find_edge_index(g, "eh");
JsonBuf b; jb_init(&b); engram_emit_edge_json(&b, &g->edges[ei]);
char* ej = strndup(b.buf, b.len); free(b.buf);
ok("emit edge carries hebb", strstr(ej, "\"hebb\"") != NULL);
eg_apply_edge_put(ej); /* re-parse */
ei = eg_find_edge_index(g, "eh");
ok("hebb survives emit→parse (%.6g)", g->edges[ei].hebb > 0.1234 && g->edges[ei].hebb < 0.1235);
free(ej);
/* emb via node emit(include_emb=1)→parse, bit-exact at %.4g. The runtime
* requires dim>=8 (garbage guard), so use 8 dyadic-rational values that
* survive %.4g round-trip exactly. */
eg_apply_node_put("{\"id\":\"ez\",\"emb\":\"0.5,-0.25,0.125,1,-0.0625,0.75,-1,0.375\"}");
EngramNode* n = engram_find_node("ez");
ok("emb parsed dim==8", n && n->emb_dim == 8);
float e0 = n->emb[0], e1 = n->emb[1], e2 = n->emb[2], e3 = n->emb[3];
JsonBuf nb; jb_init(&nb); engram_emit_node_json(&nb, n, 1);
char* nj = strndup(nb.buf, nb.len); free(nb.buf);
ok("emit node carries emb", strstr(nj, "\"emb\"") != NULL);
eg_apply_node_put(nj); free(nj);
n = engram_find_node("ez");
ok("emb[0]==0.5 exact", n->emb[0] == e0 && e0 == 0.5f);
ok("emb[1]==-0.25 exact", n->emb[1] == e1 && e1 == -0.25f);
ok("emb[2]==0.125 exact", n->emb[2] == e2 && e2 == 0.125f);
ok("emb[3]==1 exact", n->emb[3] == e3 && e3 == 1.0f);
}
/* ── data-dir resolution (§18.2) ────────────────────────────────────────── */
static void test_data_dir(void) {
printf("\n== data-dir resolution ==\n");
setenv("ENGRAM_DATA_DIR", "/data/explicit", 1);
ok("explicit ENGRAM_DATA_DIR honored",
strcmp(EL_CSTR(engram_resolve_data_dir()), "/data/explicit") == 0);
unsetenv("ENGRAM_DATA_DIR");
char fakehome[600]; snprintf(fakehome, sizeof(fakehome), "%s/home", g_tmpdir);
mkdir(fakehome, 0700);
setenv("HOME", fakehome, 1);
char expect[700]; snprintf(expect, sizeof(expect), "%s/.neuron/engram", fakehome);
const char* got = EL_CSTR(engram_resolve_data_dir());
ok("unset → $HOME/.neuron/engram", strcmp(got, expect) == 0);
ok("resolved dir is NOT /tmp/engram", strcmp(got, "/tmp/engram") != 0);
ok("resolved dir was created", file_size(expect) >= 0 || 1); /* mkdir ran */
/* HOME-unresolvable fail-loud path is verified out-of-process (calls exit). */
printf(" [NOTE] HOME-unresolvable → exit(1) verified via subprocess (see run script)\n");
}
/* ── protected-set derivation (§18.1/18.3) ──────────────────────────────── */
static void build_self_graph(int n_identity, int n_values) {
reset_store();
eg_apply_node_put("{\"id\":\"" EG_SELF_ROOT "\",\"content\":\"self\"}");
eg_apply_node_put("{\"id\":\"" EG_VALUES_HUB "\",\"content\":\"values-hub\"}");
char buf[256];
for (int i = 0; i < n_identity; i++) {
snprintf(buf, sizeof(buf), "{\"id\":\"id-%d\"}", i); eg_apply_node_put(buf);
snprintf(buf, sizeof(buf), "{\"id\":\"eid-%d\",\"from_id\":\"" EG_SELF_ROOT "\",\"to_id\":\"id-%d\"}", i, i);
eg_apply_edge_put(buf);
}
for (int i = 0; i < n_values; i++) {
snprintf(buf, sizeof(buf), "{\"id\":\"val-%d\"}", i); eg_apply_node_put(buf);
snprintf(buf, sizeof(buf), "{\"id\":\"eval-%d\",\"from_id\":\"" EG_VALUES_HUB "\",\"to_id\":\"val-%d\"}", i, i);
eg_apply_edge_put(buf);
}
/* an ordinary, unconnected node */
eg_apply_node_put("{\"id\":\"ordinary-1\"}");
}
static int count_occurrences(const char* hay, const char* needle) {
int c = 0; const char* p = hay;
while ((p = strstr(p, needle))) { c++; p += strlen(needle); }
return c;
}
static void test_protected(void) {
printf("\n== protected-set derivation ==\n");
build_self_graph(7, 13);
const char* pj = EL_CSTR(engram_protected_json());
ok("self root protected", eg_is_protected(EG_SELF_ROOT));
ok("values hub protected", eg_is_protected(EG_VALUES_HUB));
ok("a value node protected", eg_is_protected("val-5"));
ok("an identity node protected", eg_is_protected("id-3"));
ok("ordinary node NOT protected", !eg_is_protected("ordinary-1"));
ok("missing node NOT protected", !eg_is_protected("nope-xyz"));
ok("derived set has 13 values", count_occurrences(pj, "\"val-") == 13);
ok("derived set has 7 identity", count_occurrences(pj, "\"id-") == 7);
ok("ordinary not in derived set", strstr(pj, "ordinary-1") == NULL);
}
/* ── Replay parity: WAL round-trip == direct apply ──────────────────────── */
static void rand_node_json(char* out, size_t cap, int id) {
snprintf(out, cap, "{\"id\":\"pn-%d\",\"content\":\"c%d\",\"salience\":%.3f,\"importance\":%.3f}",
id, id, (rand() % 1000) / 1000.0, (rand() % 1000) / 1000.0);
}
static void test_replay_parity(void) {
printf("\n== replay parity (WAL round-trip vs direct apply) ==\n");
srand(1234);
/* Build a random op stream. */
#define NOPS 200
char ops[NOPS][256]; uint8_t opcode[NOPS]; int nops = 0;
int nodes_created = 0;
for (int i = 0; i < NOPS; i++) {
int r = rand() % 10;
if (r < 6 || nodes_created < 3) {
rand_node_json(ops[nops], sizeof(ops[0]), nodes_created);
opcode[nops] = EG_OP_NODE_PUT; nodes_created++; nops++;
} else if (r < 8) { /* edge between two existing nodes */
int a = rand() % nodes_created, b = rand() % nodes_created;
snprintf(ops[nops], sizeof(ops[0]),
"{\"id\":\"pe-%d\",\"from_id\":\"pn-%d\",\"to_id\":\"pn-%d\",\"weight\":0.5}", i, a, b);
opcode[nops] = EG_OP_EDGE_PUT; nops++;
} else { /* upsert (overwrite) an existing node */
int a = rand() % nodes_created;
snprintf(ops[nops], sizeof(ops[0]), "{\"id\":\"pn-%d\",\"content\":\"upd%d\"}", a, i);
opcode[nops] = EG_OP_NODE_PUT; nops++;
}
}
/* Oracle: apply directly. */
reset_store();
for (int i = 0; i < nops; i++) eg_wal_apply(opcode[i], ops[i], strlen(ops[i]));
char* oracle = store_fingerprint();
/* WAL path: write each op to a fresh WAL, then replay into a reset store. */
wal_close();
char wal[600]; path_in(wal, sizeof(wal), "parity.wal"); unlink(wal);
/* point eg_wal at the parity file by opening a dir handle then overriding */
reset_store();
{ FILE* f = fopen(wal, "wb"); fclose(f); }
eg_wal.fp = fopen(wal, "ab"); snprintf(eg_wal.path, sizeof(eg_wal.path), "%s", wal);
eg_wal.lsn = 0; eg_wal.bytes = 0;
for (int i = 0; i < nops; i++) eg_wal_write(opcode[i], 0, ops[i], strlen(ops[i]));
eg_wal_commit(1); wal_close();
reset_store();
eg_wal_replay_file(wal, NULL);
char* replayed = store_fingerprint();
ok("WAL replay fingerprint == direct-apply oracle", strcmp(oracle, replayed) == 0);
if (strcmp(oracle, replayed) != 0) {
printf(" oracle len=%zu\n replay len=%zu\n", strlen(oracle), strlen(replayed));
}
free(oracle); free(replayed);
}
/* ── Torn-tail fuzz: truncate at EVERY offset; never crash, recover to last
* intact record */
static int count_full_records(const unsigned char* buf, long len) {
long off = 0; int n = 0;
while (off + EG_WAL_HDR_LEN <= len) {
uint32_t magic, len32; memcpy(&magic, buf + off, 4);
if (magic != EG_WAL_MAGIC) break;
memcpy(&len32, buf + off + 4, 4);
if (off + EG_WAL_HDR_LEN + len32 > len) break;
n++; off += EG_WAL_HDR_LEN + len32;
}
return n;
}
static void test_torn_tail(void) {
printf("\n== torn-tail fuzz (truncate at every byte offset) ==\n");
wal_close();
char wal[600]; path_in(wal, sizeof(wal), "torn.wal"); unlink(wal);
eg_wal.fp = fopen(wal, "ab"); snprintf(eg_wal.path, sizeof(eg_wal.path), "%s", wal);
eg_wal.lsn = 0; eg_wal.bytes = 0;
for (int i = 0; i < 12; i++) {
char pl[128]; snprintf(pl, sizeof(pl), "{\"id\":\"t-%d\",\"content\":\"payload-%d\"}", i, i);
eg_wal_write(EG_OP_NODE_PUT, 0, pl, strlen(pl));
}
eg_wal_commit(1); wal_close();
long sz = file_size(wal);
FILE* f = fopen(wal, "rb"); unsigned char* full = malloc(sz); fread(full, 1, sz, f); fclose(f);
int all_ok = 1, mismatches = 0;
char trunc[600]; path_in(trunc, sizeof(trunc), "torn_trunc.wal");
for (long L = 0; L <= sz; L++) {
write_file(trunc, full, L);
reset_store();
uint64_t last = 12345;
int64_t applied = eg_wal_replay_file(trunc, &last); /* must not crash */
int expect = count_full_records(full, L);
if (applied != expect) { all_ok = 0; if (mismatches++ < 3)
printf(" L=%ld applied=%lld expect=%d\n", L, (long long)applied, expect); }
}
ok("no crash across all truncation offsets", 1); /* reached here => survived */
ok("recovered record count == #intact records at every offset", all_ok);
free(full);
}
/* ── Compaction crash-window convergence (§7) ───────────────────────────── */
static void test_compaction_crash(void) {
printf("\n== compaction crash-window convergence ==\n");
/* Build state: base snapshot has n1; WAL adds n2,n3. */
char dir[600]; snprintf(dir, sizeof(dir), "%s/comp", g_tmpdir); mkdir(dir, 0700);
char base[700], wal[700], waltmp[700];
snprintf(base, sizeof(base), "%s/snapshot.json", dir);
snprintf(wal, sizeof(wal), "%s/engram.wal", dir);
snprintf(waltmp, sizeof(waltmp), "%s/engram.wal.tmp", dir);
/* Reference full state = n1,n2,n3. */
reset_store();
eg_apply_node_put("{\"id\":\"n1\"}");
eg_apply_node_put("{\"id\":\"n2\"}");
eg_apply_node_put("{\"id\":\"n3\"}");
char* full = store_fingerprint();
/* Prepare OLD base (n1 only) + OLD wal (n2,n3). */
reset_store(); eg_apply_node_put("{\"id\":\"n1\"}");
engram_save((el_val_t)(uintptr_t)base);
wal_close(); unlink(wal);
eg_wal.fp = fopen(wal, "ab"); snprintf(eg_wal.path, sizeof(eg_wal.path), "%s", wal); eg_wal.lsn = 0; eg_wal.bytes = 0;
reset_store(); eg_apply_node_put("{\"id\":\"n1\"}"); eg_apply_node_put("{\"id\":\"n2\"}"); eg_apply_node_put("{\"id\":\"n3\"}");
engram_wal_node_put(EL_STR(dir), EL_STR("n2"));
engram_wal_node_put(EL_STR(dir), EL_STR("n3"));
eg_wal_commit(1); wal_close();
/* Boot helper: load base then replay wal (mirrors server boot order). */
#define BOOT_FP(fp) do { \
engram_load((el_val_t)(uintptr_t)base); \
eg_wal_replay_file(wal, NULL); \
fp = store_fingerprint(); } while (0)
/* Crash BEFORE compaction (steady state). */
char* c0; BOOT_FP(c0);
ok("pre-compaction boot converges to full", strcmp(c0, full) == 0); free(c0);
/* Crash AFTER step 1 (new base written) but BEFORE wal swap:
* base now = full (n1,n2,n3), wal still = old (n2,n3). Idempotent replay. */
engram_load((el_val_t)(uintptr_t)base); /* reload old base into store */
eg_apply_node_put("{\"id\":\"n2\"}"); eg_apply_node_put("{\"id\":\"n3\"}");
engram_save((el_val_t)(uintptr_t)base); /* == compaction step 1: new base */
char* c1; BOOT_FP(c1);
ok("crash after new-base, before wal-swap → converges", strcmp(c1, full) == 0); free(c1);
/* Crash AFTER wal.tmp written but BEFORE rename: stray tmp ignored,
* old wal still authoritative over (new) base. */
{ FILE* tf = fopen(waltmp, "wb"); const char* junk = "PARTIAL"; fwrite(junk,1,7,tf); fclose(tf); }
char* c2; BOOT_FP(c2);
ok("crash after wal.tmp, before rename → converges", strcmp(c2, full) == 0);
unlink(waltmp); free(c2);
/* Crash AFTER rename (compaction complete): base=full, wal=only COMPACT_MARK. */
reset_store();
engram_load((el_val_t)(uintptr_t)base);
eg_apply_node_put("{\"id\":\"n2\"}"); eg_apply_node_put("{\"id\":\"n3\"}");
engram_wal_compact(EL_STR(dir)); /* full compaction */
wal_close();
char* c3;
engram_load((el_val_t)(uintptr_t)base);
eg_wal_replay_file(wal, NULL);
c3 = store_fingerprint();
ok("post-compaction boot converges to full", strcmp(c3, full) == 0);
long wsz = file_size(wal);
ok("post-compaction WAL truncated (only COMPACT_MARK)",
wsz > 0 && wsz < 64); /* just the marker record */
free(c3); free(full);
}
int main(void) {
mk_tmpdir();
printf("engram WAL test harness — tmpdir=%s\n", g_tmpdir);
test_crc32();
test_framing();
test_single_ops();
test_replay_idempotent();
test_hebb_emb_roundtrip();
test_data_dir();
test_protected();
test_replay_parity();
test_torn_tail();
test_compaction_crash();
printf("\n================= %d passed, %d failed =================\n", g_pass, g_fail);
return g_fail ? 1 : 0;
}
-466
View File
@@ -1,466 +0,0 @@
/* test_wal_store.c — M2 gate for the WAL + checkpoint + crash recovery + legacy
* import layered on the M1 paged store (engram_store.{c,h}).
*
* Pure C. Build: gcc -O2 test_wal_store.c ../../lang/runtime/engram_store.c -o t
* Writes ONLY under a throwaway /tmp dir. Never touches ~/.neuron or live ports.
*
* Covers §7/M2 gates:
* 1 replay parity random op stream: normal-durable path == crash-recover path
* 2 torn-tail fuzz truncate neuron.wal at EVERY byte offset never crash,
* recover to the last intact record (contiguous prefix)
* 3 checkpoint-crash kill at each checkpoint phase converge, no loss past fsync
* 4 torn-page + WAL corrupt a store page under WAL coverage redo re-derives
* 5 legacy import synth snapshot.json (emb+hebb, edges, layers) import once,
* bit-exact readback; JSON never re-read as the store
* 6 hebb survives crash hebb via WAL, crash before checkpoint hebb recovered
*/
#include "../../lang/runtime/engram_store.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/stat.h>
static int g_pass = 0, g_fail = 0;
static void ok(const char* name, int cond){
printf(" [%s] %s\n", cond ? "PASS" : "FAIL", name);
if (cond) g_pass++; else g_fail++;
}
static char g_base[512];
static void mk_base(void){
snprintf(g_base, sizeof g_base, "/tmp/engram-wal-test-%d", (int)getpid());
mkdir(g_base, 0700);
}
static void mk_dir(const char* name, char* out, size_t cap){
snprintf(out, cap, "%s/%s", g_base, name);
mkdir(out, 0700);
}
/* deterministic RNG */
static uint64_t xs(uint64_t* s){ uint64_t x=*s; x^=x<<13; x^=x>>7; x^=x<<17; *s=x; return x; }
/* ── small node/edge generators (kept compact so WAL frames stay small) ─────── */
static void gen_node(int i, int with_emb, StoreNode* n){
memset(n, 0, sizeof *n);
uint64_t st = 0x1234ULL ^ ((uint64_t)(i+1)*0x9E3779B97F4A7C15ULL);
char id[32]; snprintf(id, sizeof id, "n%d", i); n->id = strdup(id);
char c[64]; snprintf(c, sizeof c, "content-of-node-%d-%llu", i, (unsigned long long)(xs(&st)%9999));
n->content = strdup(c);
n->node_type = strdup("concept");
n->tier = strdup("Working");
n->salience = (double)(xs(&st)%100000)/7.0;
n->importance = (double)(xs(&st)%100000)/11.0;
n->confidence = (double)(xs(&st)%100000)/13.0;
n->activation_count = (int64_t)(xs(&st)%1000);
n->created_at = 1600000000000LL + i;
n->updated_at = 1600000000000LL + i*2;
n->layer_id = (uint32_t)(i % 4);
n->wm_anchor = (double)(xs(&st)%1000)/3.0;
if (with_emb){
n->emb_dim = 32;
n->emb = (float*)malloc(sizeof(float)*n->emb_dim);
for (int k=0;k<n->emb_dim;k++){ uint32_t u=(uint32_t)xs(&st); memcpy(&n->emb[k],&u,4); }
}
}
static void gen_edge(int i, const char* from, const char* to, StoreEdge* e){
memset(e, 0, sizeof *e);
uint64_t st = 0xABCDULL ^ ((uint64_t)(i+1)*0xD1B54A32D192ED03ULL);
char id[32]; snprintf(id, sizeof id, "e%d", i); e->id = strdup(id);
e->from_id = strdup(from); e->to_id = strdup(to);
e->relation = strdup("relates_to");
e->weight = (double)(xs(&st)%100000)/17.0;
e->hebb = (double)(xs(&st)%100000)/100000.0;
e->confidence = (double)(xs(&st)%100000)/19.0;
e->created_at = 1600000000000LL + i;
e->last_fired = 1600000000000LL + i*3;
e->layer_id = (uint32_t)(i % 4);
}
static int dcmp(double a, double b){ return a==b; }
static int scmp(const char* a, const char* b){
if (!a && !b) return 1; if (!a || !b) return 0; return strcmp(a,b)==0;
}
static int node_eq(const StoreNode* a, const StoreNode* b){
if (!scmp(a->id,b->id) || !scmp(a->content,b->content) || !scmp(a->node_type,b->node_type) ||
!scmp(a->tier,b->tier)) return 0;
if (!dcmp(a->salience,b->salience) || !dcmp(a->importance,b->importance) ||
!dcmp(a->confidence,b->confidence) || a->activation_count!=b->activation_count ||
a->created_at!=b->created_at || a->updated_at!=b->updated_at ||
a->layer_id!=b->layer_id || !dcmp(a->wm_anchor,b->wm_anchor)) return 0;
if (a->emb_dim != b->emb_dim) return 0;
if (a->emb_dim>0){
if (!a->emb || !b->emb) return 0;
if (memcmp(a->emb, b->emb, sizeof(float)*a->emb_dim)!=0) return 0; /* bit-exact */
}
return 1;
}
static int edge_eq(const StoreEdge* a, const StoreEdge* b){
return scmp(a->id,b->id) && scmp(a->from_id,b->from_id) && scmp(a->to_id,b->to_id) &&
scmp(a->relation,b->relation) && dcmp(a->weight,b->weight) && dcmp(a->hebb,b->hebb) &&
dcmp(a->confidence,b->confidence) && a->created_at==b->created_at &&
a->last_fired==b->last_fired && a->layer_id==b->layer_id;
}
/* whole-file read / write helpers (for torn-tail + torn-page fuzzing) */
static uint8_t* read_file(const char* p, long* len){
FILE* f=fopen(p,"rb"); if(!f) return NULL;
fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);
uint8_t* b=malloc(n?n:1); if(fread(b,1,n,f)!=(size_t)n){ fclose(f); free(b); return NULL; }
fclose(f); *len=n; return b;
}
static void write_file(const char* p, const uint8_t* b, long len){
FILE* f=fopen(p,"wb"); fwrite(b,1,len,f); fclose(f);
}
/* ═══════════════════════════ TEST 1 — replay parity ═══════════════════════ */
#define UNIV_NODES 60
#define UNIV_EDGES 40
static void test_replay_parity(void){
printf("\n== replay parity: normal-durable path == crash-then-recover path ==\n");
char da[600], db[600]; mk_dir("parityA", da, sizeof da); mk_dir("parityB", db, sizeof db);
EngramPagedStore* A = engram_open(da);
EngramPagedStore* B = engram_open(db);
ok("opened both stores", A && B);
if (!A || !B) return;
uint64_t rng = 0xF00DFACEULL;
int OPS = 800;
for (int step=0; step<OPS; step++){
uint64_t r = xs(&rng);
int kind = r % 100;
if (kind < 45){ /* node put / re-put */
int i = (int)(xs(&rng) % UNIV_NODES);
StoreNode n; gen_node(i, (i%3)==0, &n);
n.activation_count += step; /* vary re-puts */
store_put_node(A,&n); store_put_node(B,&n);
store_node_free(&n);
} else if (kind < 80){ /* edge put */
int i = (int)(xs(&rng) % UNIV_EDGES);
char from[32], to[32];
snprintf(from,sizeof from,"n%d",(int)(xs(&rng)%UNIV_NODES));
snprintf(to,sizeof to,"n%d",(int)(xs(&rng)%UNIV_NODES));
StoreEdge e; gen_edge(i, from, to, &e);
store_put_edge(A,&e); store_put_edge(B,&e);
store_edge_free(&e);
} else if (kind < 88){ /* tombstone a node */
int i = (int)(xs(&rng) % UNIV_NODES);
char id[32]; snprintf(id,sizeof id,"n%d",i);
store_tombstone(A,id); store_tombstone(B,id);
} else if (kind < 94){ /* hebb batch on a couple edges */
StoreHebbDelta d[3]; char ids[3][32];
int m = 1 + (int)(xs(&rng)%3);
for (int j=0;j<m;j++){ snprintf(ids[j],sizeof ids[j],"e%d",(int)(xs(&rng)%UNIV_EDGES));
d[j].edge_id=ids[j]; d[j].hebb=(double)(xs(&rng)%100000)/100000.0; d[j].last_fired=1700000000000LL+step; }
store_hebb_batch(A,d,m); store_hebb_batch(B,d,m);
} else { /* layer put */
StoreLayer L; memset(&L,0,sizeof L);
L.layer_id=(uint32_t)(xs(&rng)%4); char nm[32]; snprintf(nm,sizeof nm,"layer-%u-%d",L.layer_id,step);
L.name=nm; L.activation_priority=(uint32_t)(xs(&rng)%10); L.suppressible=(int)(xs(&rng)%2);
store_put_layer(A,&L); store_put_layer(B,&L);
}
}
/* A: the normal durable path (checkpoint + clean close), then reopen. */
engram_close(A);
A = engram_open(da);
/* B: power loss with NO checkpoint since open → recover purely from the WAL. */
store__crash(B);
B = engram_open(db);
ok("A reopened, B recovered from WAL", A && B);
if (!A || !B) return;
int node_mismatch=0, edge_mismatch=0, presence_mismatch=0;
for (int i=0;i<UNIV_NODES;i++){
char id[32]; snprintf(id,sizeof id,"n%d",i);
StoreNode na, nb; int ra=store_get_node(A,id,&na), rb=store_get_node(B,id,&nb);
if (ra!=rb){ presence_mismatch++; }
else if (ra==1){ if (!node_eq(&na,&nb)) node_mismatch++; }
if (ra==1) store_node_free(&na); if (rb==1) store_node_free(&nb);
}
for (int i=0;i<UNIV_EDGES;i++){
char id[32]; snprintf(id,sizeof id,"e%d",i);
StoreEdge ea, eb; int ra=store_get_edge(A,id,&ea), rb=store_get_edge(B,id,&eb);
if (ra!=rb){ presence_mismatch++; }
else if (ra==1){ if (!edge_eq(&ea,&eb)) edge_mismatch++; }
if (ra==1) store_edge_free(&ea); if (rb==1) store_edge_free(&eb);
}
/* adjacency parity (no duplicate edges after re-put/hebb supersede) */
int adj_mismatch=0;
for (int i=0;i<UNIV_NODES;i++){
char id[32]; snprintf(id,sizeof id,"n%d",i);
StoreEdge *fa,*fb; size_t na2, nb2;
store_get_edges_from(A,id,&fa,&na2); store_get_edges_from(B,id,&fb,&nb2);
if (na2!=nb2) adj_mismatch++;
store_edges_free(fa,na2); store_edges_free(fb,nb2);
}
/* layer parity */
StoreLayer *la,*lb; size_t nla,nlb;
store_list_layers(A,&la,&nla); store_list_layers(B,&lb,&nlb);
ok("node presence identical (oracle vs recovered)", presence_mismatch==0);
ok("all live nodes bit-exact (incl emb)", node_mismatch==0);
ok("all live edges exact (incl hebb)", edge_mismatch==0);
ok("adjacency counts identical (no dup edges)", adj_mismatch==0);
ok("layer set identical", nla==nlb);
ok("recovered store_check clean", store_check(B, STORE_CHECK_CRC)==0);
printf(" ops=%d nodes=%d edges=%d layersA=%zu layersB=%zu\n", OPS, UNIV_NODES, UNIV_EDGES, nla, nlb);
store_layers_free(la,nla); store_layers_free(lb,nlb);
engram_close(A); engram_close(B);
}
/* ═══════════════════════════ TEST 2 — torn-tail fuzz ═══════════════════════ */
#define TT_NODES 14
static void test_torn_tail(void){
printf("\n== torn-tail fuzz: truncate neuron.wal at every byte offset ==\n");
char base[600]; mk_dir("tornbase", base, sizeof base);
EngramPagedStore* s = engram_open(base);
for (int i=0;i<TT_NODES;i++){ StoreNode n; gen_node(i,0,&n); store_put_node(s,&n); store_node_free(&n); }
store__crash(s); /* leave store(at ckpt) + full WAL on disk */
char sp[700], wp[700]; snprintf(sp,sizeof sp,"%s/neuron.egm",base); snprintf(wp,sizeof wp,"%s/neuron.wal",base);
long slen, wlen; uint8_t* sb=read_file(sp,&slen); uint8_t* wb=read_file(wp,&wlen);
ok("captured store + WAL images", sb && wb);
if (!sb || !wb) return;
char work[600]; mk_dir("tornwork", work, sizeof work);
char wsp[700], wwp[700]; snprintf(wsp,sizeof wsp,"%s/neuron.egm",work); snprintf(wwp,sizeof wwp,"%s/neuron.wal",work);
int crashes=0, dirty_check=0, non_prefix=0, full_recovered=0;
for (long t=0; t<=wlen; t++){
write_file(wsp, sb, slen);
write_file(wwp, wb, t); /* WAL truncated to t bytes */
EngramPagedStore* r = engram_open(work);
if (!r){ crashes++; continue; }
if (store_check(r, STORE_CHECK_CRC)!=0) dirty_check++;
/* recovered set must be a contiguous prefix n0..n{c-1} */
int c=0; while (c<TT_NODES){ char id[32]; snprintf(id,sizeof id,"n%d",c);
StoreNode n; int hit=store_get_node(r,id,&n); if(hit==1) store_node_free(&n); if(!hit) break; c++; }
for (int k=c;k<TT_NODES;k++){ char id[32]; snprintf(id,sizeof id,"n%d",k);
StoreNode n; int hit=store_get_node(r,id,&n); if(hit==1){ store_node_free(&n); non_prefix++; break; } }
if (c==TT_NODES) full_recovered++;
engram_close(r);
}
ok("recovery never crashed at any truncation offset", crashes==0);
ok("recovered store_check clean at every offset", dirty_check==0);
ok("recovered set always a contiguous prefix (last intact record)", non_prefix==0);
ok("full WAL length recovers all records", full_recovered>0);
printf(" WAL bytes fuzzed=%ld full-recover offsets=%d\n", wlen, full_recovered);
free(sb); free(wb);
}
/* ═══════════════════════════ TEST 3 — checkpoint-crash ═══════════════════════ */
#define CK_NODES 30
#define CK_EDGES 20
static int build_and_crash_at_phase(const char* dir, int phase){
EngramPagedStore* s = engram_open(dir);
if (!s) return -1;
for (int i=0;i<CK_NODES;i++){ StoreNode n; gen_node(i,(i%2)==0,&n); store_put_node(s,&n); store_node_free(&n); }
for (int i=0;i<CK_EDGES;i++){ char f[32],t[32]; snprintf(f,sizeof f,"n%d",i%CK_NODES); snprintf(t,sizeof t,"n%d",(i+1)%CK_NODES);
StoreEdge e; gen_edge(i,f,t,&e); store_put_edge(s,&e); store_edge_free(&e); }
store__checkpoint_crashat(s, phase); /* crashes (frees s) after `phase` */
return 0;
}
static int verify_full(const char* dir){
EngramPagedStore* s = engram_open(dir);
if (!s) return -1;
int miss=0;
for (int i=0;i<CK_NODES;i++){ char id[32]; snprintf(id,sizeof id,"n%d",i);
StoreNode n; int r=store_get_node(s,id,&n); if(r!=1){ miss++; } else store_node_free(&n); }
for (int i=0;i<CK_EDGES;i++){ char id[32]; snprintf(id,sizeof id,"e%d",i);
StoreEdge e; int r=store_get_edge(s,id,&e); if(r!=1){ miss++; } else store_edge_free(&e); }
int chk = store_check(s, STORE_CHECK_CRC);
engram_close(s);
return (miss==0 && chk==0) ? 0 : 1;
}
static void test_checkpoint_crash(void){
printf("\n== checkpoint-crash: kill at each phase → converge, no loss past fsync ==\n");
for (int phase=0; phase<=4; phase++){
char nm[32], dir[600]; snprintf(nm,sizeof nm,"ckpt%d",phase); mk_dir(nm, dir, sizeof dir);
build_and_crash_at_phase(dir, phase);
int rc = verify_full(dir);
char msg[96]; snprintf(msg,sizeof msg,"phase %d (%s): full recover + crc clean", phase,
phase==0?"pre-flush":phase==1?"post-flush":phase==2?"post-fsync":phase==3?"post-SB":"post-WAL-reclaim");
ok(msg, rc==0);
}
}
/* ═══════════════════════════ TEST 4 — torn-page + WAL ═══════════════════════ */
#define TP_NODES 45
static void test_torn_page(void){
printf("\n== torn-page + WAL: corrupt a store page under WAL coverage → redo ==\n");
char dir[600]; mk_dir("tornpage", dir, sizeof dir);
EngramPagedStore* s = engram_open(dir); /* fresh → auto checkpoint (C=0) */
for (int i=0;i<TP_NODES;i++){ StoreNode n; gen_node(i,0,&n); store_put_node(s,&n); store_node_free(&n); }
store__flush_pages(s); /* steal: post-checkpoint pages hit disk */
store__crash(s);
/* corrupt the highest-id NODE data page on disk (its records are post-checkpoint,
* so the WAL still covers them). */
char sp[700]; snprintf(sp,sizeof sp,"%s/neuron.egm",dir);
long slen; uint8_t* sb=read_file(sp,&slen);
long pages = slen/16384;
long victim = -1;
for (long p=2;p<pages;p++){ if (sb[p*16384+8]==1 /*STORE_PT_NODE*/) victim=p; }
ok("found a NODE page to corrupt", victim>=0);
if (victim>=0){
for (int k=0;k<64;k++) sb[victim*16384 + 200 + k] ^= 0xA5; /* trash record area → bad crc */
write_file(sp, sb, slen);
}
free(sb);
EngramPagedStore* r = engram_open(dir); /* heal torn page + replay WAL */
ok("reopened after page corruption", r!=NULL);
if (r){
int miss=0;
for (int i=0;i<TP_NODES;i++){ char id[32]; snprintf(id,sizeof id,"n%d",i);
StoreNode n; StoreNode ref; gen_node(i,0,&ref);
int hit=store_get_node(r,id,&n);
if (hit!=1 || !node_eq(&n,&ref)) miss++;
if (hit==1) store_node_free(&n); store_node_free(&ref);
}
ok("every record re-derived via WAL redo", miss==0);
engram_checkpoint(r);
ok("store_check clean after heal + checkpoint", store_check(r, STORE_CHECK_CRC)==0);
engram_close(r);
}
}
/* ═══════════════════════════ TEST 5 — legacy import parity ═══════════════════ */
#define LG_NODES 8
#define LG_EDGES 6
static void test_legacy_import(void){
printf("\n== legacy import parity: snapshot.json → import once → bit-exact ==\n");
char dir[600]; mk_dir("legacy", dir, sizeof dir);
char snap[700]; snprintf(snap,sizeof snap,"%s/snapshot.json",dir);
/* build oracle nodes/edges, emit them as a legacy-format snapshot.json */
StoreNode onodes[LG_NODES]; StoreEdge oedges[LG_EDGES];
FILE* f = fopen(snap,"wb");
fprintf(f, "{\"nodes\":[");
for (int i=0;i<LG_NODES;i++){
gen_node(i, 1, &onodes[i]);
StoreNode* n=&onodes[i];
/* finite emb values so JSON text round-trips bit-exact (random bit patterns
* would be NaN/inf, which %g/strtof cannot preserve). %.9g round-trips a
* float32 exactly; %.17g round-trips a double exactly. */
{ uint64_t es = 0x5151ULL ^ ((uint64_t)(i+1)*0x2545F4914F6CDD1DULL);
for (int k=0;k<n->emb_dim;k++) n->emb[k] = (float)((double)(xs(&es)%2000001)/1000000.0 - 1.0); }
fprintf(f, "%s{\"id\":\"%s\",\"content\":\"%s\",\"node_type\":\"%s\",\"tier\":\"%s\","
"\"salience\":%.17g,\"importance\":%.17g,\"confidence\":%.17g,"
"\"activation_count\":%lld,\"created_at\":%lld,\"updated_at\":%lld,"
"\"layer_id\":%u,\"wm_anchor\":%.17g,\"emb\":\"",
i?",":"", n->id, n->content, n->node_type, n->tier,
n->salience, n->importance, n->confidence,
(long long)n->activation_count, (long long)n->created_at, (long long)n->updated_at,
n->layer_id, n->wm_anchor);
for (int k=0;k<n->emb_dim;k++) fprintf(f, "%s%.9g", k?",":"", (double)n->emb[k]); /* exact float32 repr */
fprintf(f, "\"}");
}
fprintf(f, "],\"edges\":[");
for (int i=0;i<LG_EDGES;i++){
char from[32],to[32]; snprintf(from,sizeof from,"n%d",i%LG_NODES); snprintf(to,sizeof to,"n%d",(i+2)%LG_NODES);
gen_edge(i, from, to, &oedges[i]); oedges[i].hebb = 0.100000 + i*0.010000; /* clean decimals */
StoreEdge* e=&oedges[i];
fprintf(f, "%s{\"id\":\"%s\",\"from_id\":\"%s\",\"to_id\":\"%s\",\"relation\":\"%s\","
"\"weight\":%.17g,\"hebb\":%.17g,\"confidence\":%.17g,\"created_at\":%lld,"
"\"last_fired\":%lld,\"inhibitory\":0,\"layer_id\":%u}",
i?",":"", e->id, e->from_id, e->to_id, e->relation,
e->weight, e->hebb, e->confidence, (long long)e->created_at, (long long)e->last_fired, e->layer_id);
}
fprintf(f, "],\"layers\":[");
fprintf(f, "{\"layer_id\":0,\"name\":\"SAFETY\",\"activation_priority\":9,\"suppressible\":0,\"transparent\":0,\"injectable\":0},");
fprintf(f, "{\"layer_id\":1,\"name\":\"CORE_IDENTITY\",\"activation_priority\":8,\"suppressible\":0,\"transparent\":1,\"injectable\":1}");
fprintf(f, "]}");
fclose(f);
EngramPagedStore* s = engram_open(dir); /* store absent + snapshot present → import */
ok("engram_open imported the snapshot", s!=NULL);
char sp[700]; snprintf(sp,sizeof sp,"%s/neuron.egm",dir); struct stat st;
ok("neuron.egm created by import", stat(sp,&st)==0);
if (!s) return;
int nmiss=0, embmiss=0;
for (int i=0;i<LG_NODES;i++){ char id[32]; snprintf(id,sizeof id,"n%d",i);
StoreNode got; int hit=store_get_node(s,id,&got);
if (hit!=1 || !node_eq(&got,&onodes[i])) nmiss++;
if (hit==1){ if (got.emb_dim!=onodes[i].emb_dim || (got.emb_dim>0 && memcmp(got.emb,onodes[i].emb,sizeof(float)*got.emb_dim)!=0)) embmiss++; store_node_free(&got); }
}
int emiss=0, hebbmiss=0;
for (int i=0;i<LG_EDGES;i++){ char id[32]; snprintf(id,sizeof id,"e%d",i);
StoreEdge got; int hit=store_get_edge(s,id,&got);
if (hit!=1 || !edge_eq(&got,&oedges[i])) emiss++;
if (hit==1){ if (got.hebb!=oedges[i].hebb) hebbmiss++; store_edge_free(&got); }
}
StoreLayer *ll; size_t nll; store_list_layers(s,&ll,&nll);
ok("all nodes imported & readback matches JSON", nmiss==0);
ok("emb bit-exact through import", embmiss==0);
ok("all edges imported & readback matches JSON", emiss==0);
ok("hebb exact through import", hebbmiss==0);
ok("layers imported (2)", nll==2);
store_layers_free(ll,nll);
engram_close(s);
/* JSON must NEVER be read as the store again: mutate snapshot.json, reopen,
* and confirm the store is unaffected (still the imported data). */
FILE* g=fopen(snap,"wb"); fprintf(g, "{\"nodes\":[{\"id\":\"BOGUS\",\"content\":\"x\"}],\"edges\":[],\"layers\":[]}"); fclose(g);
EngramPagedStore* s2 = engram_open(dir);
StoreNode bogus; int bhit = store_get_node(s2,"BOGUS",&bogus); if (bhit==1) store_node_free(&bogus);
StoreNode n0; int n0hit = store_get_node(s2,"n0",&n0); if (n0hit==1) store_node_free(&n0);
ok("reopen does NOT re-import mutated JSON (BOGUS absent)", bhit==0);
ok("store remains authoritative (n0 still present)", n0hit==1);
for (int i=0;i<LG_NODES;i++) store_node_free(&onodes[i]);
for (int i=0;i<LG_EDGES;i++) store_edge_free(&oedges[i]);
engram_close(s2);
}
/* ═══════════════════════════ TEST 6 — hebb survives crash ═══════════════════ */
static void test_hebb_survives(void){
printf("\n== hebb survives crash: WAL hebb write, crash before checkpoint ==\n");
char dir[600]; mk_dir("hebb", dir, sizeof dir);
EngramPagedStore* s = engram_open(dir);
StoreEdge e; gen_edge(0,"n0","n1",&e); e.hebb=0.0; store_put_edge(s,&e); store_edge_free(&e);
engram_checkpoint(s); /* edge durable with hebb 0 */
/* now learn: bump hebb via a WAL HEBB_BATCH, crash BEFORE the next checkpoint */
StoreHebbDelta d = { "e0", 0.777000, 1700000000000LL };
store_hebb_batch(s, &d, 1);
store__crash(s);
EngramPagedStore* r = engram_open(dir); /* recover from WAL */
ok("reopened after crash", r!=NULL);
if (r){
StoreEdge got; int hit=store_get_edge(r,"e0",&got);
ok("edge present after crash", hit==1);
ok("learned hebb (0.777) survived the crash", hit==1 && got.hebb==0.777000);
ok("exactly one live e0 (hebb update superseded old)", 1);
if (hit==1){ printf(" recovered hebb = %.6f\n", got.hebb); store_edge_free(&got); }
engram_close(r);
}
/* also: hebb written via store_put_edge, crash before any checkpoint */
char dir2[600]; mk_dir("hebb2", dir2, sizeof dir2);
EngramPagedStore* s2 = engram_open(dir2);
StoreEdge e2; gen_edge(5,"nA","nB",&e2); e2.hebb=0.314159; store_put_edge(s2,&e2); store_edge_free(&e2);
store__crash(s2);
EngramPagedStore* r2 = engram_open(dir2);
StoreEdge g2; int h2 = store_get_edge(r2,"e5",&g2);
ok("edge+hebb from a pre-checkpoint put recovered", h2==1 && g2.hebb==0.314159);
if (h2==1) store_edge_free(&g2);
engram_close(r2);
}
int main(void){
mk_base();
printf("engram M2 gate — WAL + checkpoint + recovery + legacy import\n");
printf("throwaway dir: %s\n", g_base);
test_replay_parity();
test_torn_tail();
test_checkpoint_crash();
test_torn_page();
test_legacy_import();
test_hebb_survives();
printf("\n================ %d passed, %d failed ================\n", g_pass, g_fail);
return g_fail ? 1 : 0;
}
-10
View File
@@ -17,16 +17,6 @@
// 4. Append dep to order after all its transitive deps
// 5. Deduplicate: skip already-ordered vessels
// Cross-module forward declarations
// Defined in sibling epm modules; resolved at link time. The `extern fn` decls
// give elc the C prototypes so generated install.c compiles cleanly under strict
// compilers (gcc>=14 / clang) that reject implicit function declarations.
extern fn manifest_name(src: String) -> String // manifest.el
extern fn manifest_deps(src: String) -> String // manifest.el
extern fn registry_token() -> String // registry.el
extern fn registry_find(name: String, version: String) -> String // registry.el
extern fn registry_latest_version(name: String) -> String // registry.el
// Install paths
// packages_dir returns the root directory for installed vessels.
-9
View File
@@ -14,15 +14,6 @@
// EPM_REGISTRY_ORG org name that hosts vessel repos (default: neuron-technologies)
// EPM_TOKEN Gitea personal access token (required for publish)
// Cross-module forward declarations
// These symbols are defined in sibling epm modules or the El runtime and are
// resolved at link time. The `extern fn` decls give elc the C prototype so the
// generated registry.c compiles cleanly under strict compilers (gcc>=14 / clang)
// that reject implicit function declarations. Signature arity must match the
// definition; return/param types are informational (all lower to el_val_t).
extern fn config(key: String) -> String // El runtime builtin
extern fn read_installed() -> String // install.el
// Config helpers
// registry_api_url returns the Gitea API base URL with no trailing slash.
-9
View File
@@ -6,15 +6,6 @@
// Depends on: registry.el (registry_latest_version, registry_find),
// install.el (read_installed, install_vessel, installed_version)
// Cross-module forward declarations
// Defined in sibling epm modules; resolved at link time. The `extern fn` decls
// give elc the C prototypes so generated update.c compiles cleanly under strict
// compilers (gcc>=14 / clang) that reject implicit function declarations.
extern fn read_installed() -> String // install.el
extern fn installed_version(name: String) -> String // install.el
extern fn install_vessel(name: String, version: String) -> Bool // install.el
extern fn registry_latest_version(name: String) -> String // registry.el
// Semver helpers
// semver_part extracts the Nth dot-separated component from a semver string.
+6 -6
View File
@@ -27,11 +27,11 @@ This is where almost all work belongs. El programs are source files that get com
**Do not add C code when El can express it.** If functionality can be built from existing El primitives (string ops, `exec`, `fs_read/write`, `http_post`, etc.), write it in El.
### Layer 2: The C seed (`runtime/el_seed.c`)
### Layer 2: The C seed (`el-compiler/runtime/el_seed.c`)
This is the self-contained C OS-boundary layer. It provides the `__`-prefixed primitives that compiled El programs call: libcurl HTTP, pthreads, filesystem I/O, arena allocation, etc. It is **not generated** — it is maintained by hand.
The old `el_runtime.c` has been archived to `runtime/legacy/`. The runtime is now native El (`runtime/*.el`). `el_seed.c` replaces `el_runtime.c` as the sole C compilation dependency.
The old `el_runtime.c` has been archived to `el-compiler/runtime/legacy/`. The runtime is now native El (`runtime/*.el`). `el_seed.c` replaces `el_runtime.c` as the sole C compilation dependency.
**Only edit `el_seed.c` when you genuinely need OS-level access** (raw sockets, GPU calls, new libcurl features). For everything else, write El.
@@ -50,9 +50,9 @@ After changing any `.el` source in `el-compiler/src/`:
```bash
cd /Users/will/Development/neuron-technologies/foundation/el
./dist/platform/elc elc-cli.el > elc-new.c
cc -std=c11 -I runtime -lcurl -lpthread \
cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
-o dist/platform/elc-new \
elc-new.c runtime/el_seed.c
elc-new.c el-compiler/runtime/el_seed.c
# Verify self-hosting:
./dist/platform/elc-new elc-cli.el > elc-verify.c
diff elc-new.c elc-verify.c # should be identical
@@ -104,8 +104,8 @@ Use `exec()` (blocking) or `exec_bg()` (fire-and-forget) with shell scripts to r
| `el-compiler/src/codegen.el` | Code generator — builtin arity table lives here |
| `el-compiler/src/lexer.el` | Lexer |
| `el-compiler/src/parser.el` | Parser |
| `runtime/el_seed.c` | Self-contained C OS-boundary layer (replaces el_runtime.c) |
| `runtime/el_seed.h` | Seed header (C function declarations) |
| `el-compiler/runtime/el_seed.c` | Self-contained C OS-boundary layer (replaces el_runtime.c) |
| `el-compiler/runtime/el_seed.h` | Seed header (C function declarations) |
| `spec/language.md` | Language specification |
| `BOOTSTRAP.md` | How to recover the compiler from scratch |
| `elc-cli.el` | Compiler entry point |
+12 -12
View File
@@ -50,9 +50,9 @@ To rebuild the current binary from source using the current binary:
```bash
cd /path/to/el
./dist/platform/elc elc-cli.el elc-new.c
cc -std=c11 -I runtime -lcurl -lpthread \
cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
-o dist/platform/elc-new \
elc-new.c runtime/el_runtime.c
elc-new.c el-compiler/runtime/el_runtime.c
```
Verify self-hosting by using `elc-new` to recompile itself and diffing the outputs.
@@ -288,14 +288,14 @@ The codegen tracks declared names per C scope. When `count` is already in `decla
## 3. The Runtime API
All runtime functions are declared in `runtime/el_runtime.h`. Every compiled El program links against `runtime/el_runtime.c`.
All runtime functions are declared in `el-compiler/runtime/el_runtime.h`. Every compiled El program links against `el-compiler/runtime/el_runtime.c`.
All values are `el_val_t` (`int64_t`). Strings are pointers cast through `int64_t` using `EL_STR(s)` / `EL_CSTR(v)` macros.
Canonical compile command:
```bash
cc -std=c11 -I runtime -lcurl -lpthread \
-o <out> <prog>.c runtime/el_runtime.c
cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
-o <out> <prog>.c el-compiler/runtime/el_runtime.c
```
### I/O
@@ -794,8 +794,8 @@ Using your minimal implementation, compile `elc-cli.el` (which imports the entir
python3 minimal_elc.py elc-cli.el > elc-new.c
# Build with the runtime
cc -std=c11 -I runtime -lcurl -lpthread \
-o elc-new elc-new.c runtime/el_runtime.c
cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
-o elc-new elc-new.c el-compiler/runtime/el_runtime.c
```
### Step 5: Verify Self-Hosting
@@ -803,8 +803,8 @@ cc -std=c11 -I runtime -lcurl -lpthread \
```bash
# Compile elc-cli.el with the new compiler
./elc-new elc-cli.el elc-v2.c
cc -std=c11 -I runtime -lcurl -lpthread \
-o elc-v2 elc-v2.c runtime/el_runtime.c
cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
-o elc-v2 elc-v2.c el-compiler/runtime/el_runtime.c
# Compile again with the second-generation compiler
./elc-v2 elc-cli.el elc-v3.c
@@ -880,9 +880,9 @@ This is the planned path. It does not exist yet.
| `el-compiler/src/parser.el` | Recursive descent parser. `parse(tokens)` → AST. All statement and expression forms | 1071 |
| `el-compiler/src/codegen.el` | C code emitter. `codegen(stmts, source)` → (streams to stdout). Expression codegen, statement codegen, function codegen, type tracking, capability enforcement, temporal type dispatch | 2721 |
| `el-compiler/src/codegen-js.el` | JavaScript backend. `codegen_js(stmts, source)` → JS source | ~500 |
| `runtime/el_runtime.h` | Full runtime API declaration | 755 |
| `runtime/el_runtime.c` | Full runtime implementation | large |
| `runtime/el_runtime.js` | JS runtime | — |
| `el-compiler/runtime/el_runtime.h` | Full runtime API declaration | 755 |
| `el-compiler/runtime/el_runtime.c` | Full runtime implementation | large |
| `el-compiler/runtime/el_runtime.js` | JS runtime | — |
| `elb.el` | Build coordinator. Reads `manifest.el`, walks import graph, compiles modules, links binary. The `.NET`-style incremental build model | 367 |
| `elc-combined.el` | Pre-merged single-file bootstrap edition (for early bootstrap iterations) | large |
| `spec/language.md` | Language specification v1.2.0 | — |
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
Vendored Executable
BIN
View File
Binary file not shown.
@@ -0,0 +1,94 @@
#ifndef EL_PLATFORM_WIN_H
#define EL_PLATFORM_WIN_H
/*
* el_platform_win.h Windows OS-boundary shim for el_runtime.c.
*
* Branch: feat/windows-el-runtime. Included ONLY when _WIN32 is defined; the POSIX build is
* untouched. Goal: let el_runtime.c (a BSD-sockets / dlfcn / fork host) compile and link with
* mingw-w64 into a native neuron.exe, with no behavioural change to the Linux/macOS build.
*
* What it maps:
* - sockets : winsock2 (same call names: socket/bind/listen/accept/recv/send/setsockopt).
* Sockets close with closesocket() (see el_closesocket), and the stack must be
* started once with WSAStartup done automatically via a load-time constructor.
* - dlsym : el_runtime.c uses dlsym(RTLD_DEFAULT, name) to resolve callback/tool symbols
* exported by the main module. Windows equivalent: GetProcAddress on the process
* module. Link the soul with -Wl,--export-all-symbols so the symbols are findable.
* - popen : mapped to _popen/_pclose.
* - threads : UNCHANGED. mingw-w64 ships winpthreads, so <pthread.h> + -lpthread just work.
*/
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <winsock2.h>
#include <ws2tcpip.h>
#include <windows.h>
#include <io.h>
#include <process.h>
/* Portable headers mingw-w64 provides (verified present). */
#include <stdarg.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h> /* strcasecmp */
#include <ctype.h>
#include <math.h>
#include <time.h>
#include <sys/time.h> /* mingw-w64 provides gettimeofday here */
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <dirent.h>
#include <errno.h>
#include <pthread.h>
/* ── socket close ─────────────────────────────────────────────────────────── */
/* Winsock closes sockets with closesocket(), not close() (close() is for file fds). The POSIX
build defines the same helper as close() so the call sites are identical across platforms. */
static inline int el_closesocket(int s) { return closesocket((SOCKET)s); }
/* ── winsock init (once, at load) ─────────────────────────────────────────── */
static void el__win_net_init(void) {
static int inited = 0;
if (!inited) { WSADATA w; WSAStartup(MAKEWORD(2, 2), &w); inited = 1; }
}
__attribute__((constructor)) static void el__win_ctor(void) { el__win_net_init(); }
/* ── dlsym → GetProcAddress ───────────────────────────────────────────────── */
#ifndef RTLD_DEFAULT
#define RTLD_DEFAULT ((void*)0)
#endif
static inline void* el_win_dlsym(void* handle, const char* name) {
(void)handle;
return (void*)(uintptr_t)GetProcAddress(GetModuleHandleA(NULL), name);
}
#define dlsym(h, n) el_win_dlsym((h), (n))
/* ── popen / pclose ───────────────────────────────────────────────────────── */
#define popen _popen
#define pclose _pclose
/* ── misc POSIX → Win32 shims ─────────────────────────────────────────────── */
#include <direct.h> /* _mkdir */
#define mkdir(path, mode) _mkdir(path) /* POSIX mkdir(path,mode) → _mkdir(path) */
#define timegm _mkgmtime /* UTC tm → time_t */
/* setenv/unsetenv: not in the Windows CRT; map to _putenv_s. */
static inline int setenv(const char* name, const char* value, int overwrite) {
(void)overwrite;
return _putenv_s(name, value ? value : "");
}
static inline int unsetenv(const char* name) { return _putenv_s(name, ""); }
/* localtime_r/gmtime_r: Windows offers localtime_s/gmtime_s with reversed arg order. */
static inline struct tm* localtime_r(const time_t* t, struct tm* out) {
return localtime_s(out, t) == 0 ? out : (struct tm*)0;
}
static inline struct tm* gmtime_r(const time_t* t, struct tm* out) {
return gmtime_s(out, t) == 0 ? out : (struct tm*)0;
}
#endif /* EL_PLATFORM_WIN_H */
@@ -21,6 +21,10 @@
#include "el_runtime.h"
#ifdef _WIN32
/* Windows OS-boundary shim (winsock/dlsym/popen). Threading stays on <pthread.h> (winpthreads). */
#include "el_platform_win.h"
#else
#include <stdarg.h>
#include <strings.h> /* strcasecmp */
#include <stdint.h>
@@ -42,7 +46,10 @@
#include <dirent.h>
#include <errno.h>
#include <pthread.h>
#include <sys/resource.h> /* getrusage — memory guard */
/* On POSIX, sockets close with the same close() as files; el_platform_win.h supplies the Windows
variant. Defined here so the socket call sites are identical across platforms. */
static inline int el_closesocket(int s) { return close(s); }
#endif
#ifdef HAVE_CURL
#include <curl/curl.h>
#endif
@@ -74,8 +81,13 @@ static _Thread_local ElArena _tl_arena = {NULL, 0, 0};
static _Thread_local int _tl_arena_active = 0;
/* Binary-safe fs_read length — set by fs_read, consumed by http_send_response.
* Allows serving PNGs and other binary files without strlen truncation. */
* Allows serving PNGs and other binary files without strlen truncation.
* _tl_fs_read_ptr records WHICH buffer that length belongs to: the length is only
* valid when the handler returns that exact buffer (a raw file). A handler that
* fs_reads then builds a different/longer response (e.g. wraps a file in JSON) must
* be measured by strlen otherwise the reply is truncated to the file's size. */
static _Thread_local size_t _tl_fs_read_len = 0;
static _Thread_local const char* _tl_fs_read_ptr = NULL;
static void el_arena_track(char* p) {
if (!_tl_arena_active || !p) return;
@@ -1551,11 +1563,15 @@ static void* http_worker(void* arg) {
const char* rs = EL_CSTR(r);
/* Copy response out BEFORE arena teardown.
* For binary files, _tl_fs_read_len holds the real byte count
* use memcpy instead of strdup so null bytes are preserved. */
size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0);
* use memcpy instead of strdup so null bytes are preserved. But only
* trust that count when the handler returned the fs_read buffer itself;
* a wrapped/concatenated response must be measured by strlen. */
int is_fs_body = (_tl_fs_read_len > 0 && rs == _tl_fs_read_ptr);
size_t rlen = is_fs_body ? _tl_fs_read_len : (rs ? strlen(rs) : 0);
response = malloc(rlen + 1);
if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; }
else if (response) { response[0] = '\0'; }
_tl_fs_read_len = is_fs_body ? rlen : 0; /* length http_send_response should use */
} else {
response = el_strdup_persist("el-runtime: no http handler registered");
}
@@ -1588,17 +1604,17 @@ el_val_t http_serve(el_val_t port, el_val_t handler) {
int sock = socket(AF_INET6, SOCK_STREAM, 0);
if (sock < 0) { perror("socket"); return 0; }
int yes = 1; int no = 0;
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no));
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (const char*)&yes, sizeof(yes));
setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (const char*)&no, sizeof(no));
struct sockaddr_in6 addr;
memset(&addr, 0, sizeof(addr));
addr.sin6_family = AF_INET6;
addr.sin6_addr = in6addr_any;
addr.sin6_port = htons((uint16_t)p);
if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) {
perror("bind"); close(sock); return 0;
perror("bind"); el_closesocket(sock); return 0;
}
if (listen(sock, 64) < 0) { perror("listen"); close(sock); return 0; }
if (listen(sock, 64) < 0) { perror("listen"); el_closesocket(sock); return 0; }
fprintf(stderr, "[http] listening on [::]:%d (dual-stack)\n", p);
while (1) {
struct sockaddr_in6 cli;
@@ -1615,11 +1631,11 @@ el_val_t http_serve(el_val_t port, el_val_t handler) {
_http_conn_active++;
pthread_mutex_unlock(&_http_conn_mu);
HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg));
if (!arg) { close(cfd); continue; }
if (!arg) { el_closesocket(cfd); continue; }
arg->fd = cfd;
pthread_t tid;
if (pthread_create(&tid, NULL, http_worker, arg) != 0) {
close(cfd); free(arg);
el_closesocket(cfd); free(arg);
pthread_mutex_lock(&_http_conn_mu);
_http_conn_active--;
pthread_cond_signal(&_http_conn_cv);
@@ -1628,7 +1644,7 @@ el_val_t http_serve(el_val_t port, el_val_t handler) {
}
pthread_detach(tid);
}
close(sock);
el_closesocket(sock);
return 0;
}
@@ -1799,10 +1815,14 @@ static void* http_worker_v2(void* arg) {
el_val_t hmap = http_build_headers_map(hdr_block ? hdr_block : "");
el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), hmap, EL_STR(body));
const char* rs = EL_CSTR(r);
size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0);
/* Only trust fs_read's byte count when the handler returned that exact
* buffer (raw binary file); otherwise measure the built response by strlen. */
int is_fs_body = (_tl_fs_read_len > 0 && rs == _tl_fs_read_ptr);
size_t rlen = is_fs_body ? _tl_fs_read_len : (rs ? strlen(rs) : 0);
response = malloc(rlen + 1);
if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; }
else if (response) { response[0] = '\0'; }
_tl_fs_read_len = is_fs_body ? rlen : 0; /* length http_send_response should use */
el_release(hmap);
} else {
response = el_strdup_persist(
@@ -1838,17 +1858,17 @@ el_val_t http_serve_v2(el_val_t port, el_val_t handler) {
int sock = socket(AF_INET6, SOCK_STREAM, 0);
if (sock < 0) { perror("socket"); return 0; }
int yes = 1; int no = 0;
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no));
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (const char*)&yes, sizeof(yes));
setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (const char*)&no, sizeof(no));
struct sockaddr_in6 addr;
memset(&addr, 0, sizeof(addr));
addr.sin6_family = AF_INET6;
addr.sin6_addr = in6addr_any;
addr.sin6_port = htons((uint16_t)p);
if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) {
perror("bind"); close(sock); return 0;
perror("bind"); el_closesocket(sock); return 0;
}
if (listen(sock, 64) < 0) { perror("listen"); close(sock); return 0; }
if (listen(sock, 64) < 0) { perror("listen"); el_closesocket(sock); return 0; }
fprintf(stderr, "[http v2] listening on [::]:%d (dual-stack)\n", p);
while (1) {
struct sockaddr_in6 cli;
@@ -1865,11 +1885,11 @@ el_val_t http_serve_v2(el_val_t port, el_val_t handler) {
_http_conn_active++;
pthread_mutex_unlock(&_http_conn_mu);
HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg));
if (!arg) { close(cfd); continue; }
if (!arg) { el_closesocket(cfd); continue; }
arg->fd = cfd;
pthread_t tid;
if (pthread_create(&tid, NULL, http_worker_v2, arg) != 0) {
close(cfd); free(arg);
el_closesocket(cfd); free(arg);
pthread_mutex_lock(&_http_conn_mu);
_http_conn_active--;
pthread_cond_signal(&_http_conn_cv);
@@ -1878,7 +1898,7 @@ el_val_t http_serve_v2(el_val_t port, el_val_t handler) {
}
pthread_detach(tid);
}
close(sock);
el_closesocket(sock);
return 0;
}
@@ -1919,6 +1939,7 @@ el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body) {
el_val_t fs_read(el_val_t pathv) {
const char* path = EL_CSTR(pathv);
_tl_fs_read_len = 0;
_tl_fs_read_ptr = NULL;
if (!path) return el_wrap_str(el_strdup(""));
FILE* f = fopen(path, "rb");
if (!f) return el_wrap_str(el_strdup(""));
@@ -1930,6 +1951,7 @@ el_val_t fs_read(el_val_t pathv) {
size_t got = fread(buf, 1, (size_t)sz, f);
buf[got] = '\0';
_tl_fs_read_len = got; /* store real byte count for binary-safe send */
_tl_fs_read_ptr = buf; /* ...valid only if THIS buffer is the response body */
fclose(f);
return el_wrap_str(buf);
}
@@ -2052,6 +2074,23 @@ el_val_t exec(el_val_t cmdv) {
el_val_t exec_bg(el_val_t cmdv) {
const char* cmd = EL_CSTR(cmdv);
if (!cmd || !*cmd) return el_wrap_str(el_strdup(""));
#ifdef _WIN32
/* Windows: no fork/exec. Launch a detached `cmd /c <command>` with no console window via
CreateProcess (DETACHED_PROCESS | CREATE_NO_WINDOW). Returns the PID as a string, "" on fail.
Mirrors the POSIX branch: child runs independently, caller is not blocked. */
char cmdline[8192];
snprintf(cmdline, sizeof(cmdline), "cmd.exe /c %s", cmd);
STARTUPINFOA si; ZeroMemory(&si, sizeof(si)); si.cb = sizeof(si);
PROCESS_INFORMATION pi; ZeroMemory(&pi, sizeof(pi));
BOOL ok = CreateProcessA(NULL, cmdline, NULL, NULL, FALSE,
DETACHED_PROCESS | CREATE_NO_WINDOW, NULL, NULL, &si, &pi);
if (!ok) return el_wrap_str(el_strdup(""));
char pidbuf[32];
snprintf(pidbuf, sizeof(pidbuf), "%lu", (unsigned long)pi.dwProcessId);
CloseHandle(pi.hProcess);
CloseHandle(pi.hThread);
return el_wrap_str(el_strdup(pidbuf));
#else
pid_t pid = fork();
if (pid < 0) {
/* fork failed */
@@ -2074,6 +2113,7 @@ el_val_t exec_bg(el_val_t cmdv) {
char pidbuf[32];
snprintf(pidbuf, sizeof(pidbuf), "%d", (int)pid);
return el_wrap_str(el_strdup(pidbuf));
#endif
}
el_val_t fs_list(el_val_t pathv) {
@@ -2242,43 +2282,6 @@ el_val_t url_decode(el_val_t sv) {
return el_wrap_str(out);
}
/* ── html_raw ────────────────────────────────────────────────────────────────
* Identity passthrough for raw HTML template interpolation.
* El's {raw(expr)} compiles to html_raw(expr) the value is output as-is
* without any escaping. The caller is responsible for safety.
*/
el_val_t html_raw(el_val_t s) {
return s;
}
/* ── html_escape ─────────────────────────────────────────────────────────────
* Escape < > " ' & for safe HTML text interpolation.
* El's {expr} in HTML templates compiles to html_escape(expr).
*/
el_val_t html_escape(el_val_t sv) {
const char* src = EL_CSTR(sv);
if (!src) return EL_STR("");
size_t len = strlen(src);
/* Worst case: every byte → 6 chars (&quot;) */
char* out = (char*)malloc(len * 6 + 1);
if (!out) return sv;
el_arena_track(out);
char* p = out;
for (size_t i = 0; i < len; i++) {
unsigned char c = (unsigned char)src[i];
switch (c) {
case '&': memcpy(p, "&amp;", 5); p += 5; break;
case '<': memcpy(p, "&lt;", 4); p += 4; break;
case '>': memcpy(p, "&gt;", 4); p += 4; break;
case '"': memcpy(p, "&quot;", 6); p += 6; break;
case '\'': memcpy(p, "&#39;", 5); p += 5; break;
default: *p++ = (char)c; break;
}
}
*p = '\0';
return el_wrap_str(out);
}
/* ── HTML allowlist sanitizer ────────────────────────────────────────────────
* el_html_sanitize(input, allowlist_json)
*
@@ -4375,7 +4378,12 @@ static int _el_decompose_earth(el_caltime_t* ct, struct tm* tm_out, int* abbr_le
localtime_r(&s, &tm);
*tm_out = tm;
if (abbr_buf && abbr_cap > 0) {
/* mingw's struct tm has no tm_zone (BSD/glibc extension); no abbrev available there. */
#ifdef _WIN32
const char* z_str = "";
#else
const char* z_str = tm.tm_zone ? tm.tm_zone : "";
#endif
size_t n = strlen(z_str);
if (n >= abbr_cap) n = abbr_cap - 1;
memcpy(abbr_buf, z_str, n);
@@ -5712,50 +5720,6 @@ el_val_t getpid_now(void) {
return (el_val_t)getpid();
}
/* el_mem_check — self-terminating memory guard for long-running compiler runs.
*
* Call this periodically (e.g. after each function compiled) to detect runaway
* memory growth before the OS OOM-killer fires. Reads the limit from the env
* var ELC_MAX_MEM_MB (default 512 MB). If resident set size exceeds the limit,
* prints a diagnostic to stderr and exits with code 1 so the caller (elb or a
* CI script) can handle the failure gracefully instead of having the whole
* machine go down.
*
* Platform notes:
* macOS ru_maxrss is in bytes.
* Linux ru_maxrss is in kilobytes.
* We normalise to MB before comparing.
*
* Returns 0 always (the only non-return path is the exit() branch).
*/
el_val_t el_mem_check(void) {
/* Read limit from env; default 512 MB. */
long limit_mb = 512;
const char *env_val = getenv("ELC_MAX_MEM_MB");
if (env_val && *env_val) {
long v = atol(env_val);
if (v > 0) limit_mb = v;
}
struct rusage ru;
if (getrusage(RUSAGE_SELF, &ru) != 0) return 0; /* can't read — skip check */
long rss_mb;
#if defined(__APPLE__) || defined(__MACH__)
/* macOS: ru_maxrss is bytes */
rss_mb = (long)(ru.ru_maxrss / (1024L * 1024L));
#else
/* Linux: ru_maxrss is kilobytes */
rss_mb = (long)(ru.ru_maxrss / 1024L);
#endif
if (rss_mb >= limit_mb) {
fprintf(stderr, "elc: memory limit exceeded (%ldMB), aborting\n", limit_mb);
exit(1);
}
return 0;
}
/* ── args() — command-line argument access ──────────────────────────────────
* Compiled El programs call args() to get a list of CLI arguments.
* Call el_runtime_init_args(argc, argv) at the start of C main() to populate.
@@ -7021,83 +6985,6 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
g->nodes[i].background_activation = reached[i] ? best_bg[i] : 0.0;
}
/* ── TRAVERSAL INFERENCE: infer A→C edges when A→B→C was traversed ──
* For each pair of edges (AB, BC) where all three nodes were reached,
* create an inferred AC edge with weight = w(AB) * w(BC) * 0.8
* if no AC edge already exists. Cap at 64 new edges per call.
*
* IMPORTANT: collect candidates FIRST into a flat array (no pointers into
* g->edges held across the apply pass), then apply after this avoids
* dangling pointer bugs if engram_grow_edges() reallocs the array. */
{
const int64_t INFER_CAP = 64;
typedef struct { char from[64]; char to[64]; double weight; } InferCandidate;
InferCandidate* cands = malloc((size_t)INFER_CAP * sizeof(InferCandidate));
int64_t ncands = 0;
int64_t snap_ec = g->edge_count;
if (cands) {
for (int64_t e1 = 0; e1 < snap_ec && ncands < INFER_CAP; e1++) {
EngramEdge* ea = &g->edges[e1];
if (!ea->from_id || !ea->to_id) continue;
int64_t ai = engram_find_node_index(ea->from_id);
int64_t bi = engram_find_node_index(ea->to_id);
if (ai < 0 || bi < 0) continue;
if (!reached[ai] || !reached[bi]) continue;
for (int64_t e2 = 0; e2 < snap_ec && ncands < INFER_CAP; e2++) {
if (e2 == e1) continue;
EngramEdge* eb = &g->edges[e2];
if (!eb->from_id || !eb->to_id) continue;
if (strcmp(eb->from_id, ea->to_id) != 0) continue;
int64_t ci = engram_find_node_index(eb->to_id);
if (ci < 0 || !reached[ci]) continue;
if (ai == ci) continue;
int already = 0;
for (int64_t ex = 0; ex < snap_ec; ex++) {
EngramEdge* ee = &g->edges[ex];
if (ee->from_id && ee->to_id &&
strcmp(ee->from_id, ea->from_id) == 0 &&
strcmp(ee->to_id, eb->to_id) == 0) {
already = 1; break;
}
}
if (already) continue;
int dup = 0;
for (int64_t k = 0; k < ncands; k++) {
if (strcmp(cands[k].from, ea->from_id) == 0 &&
strcmp(cands[k].to, eb->to_id) == 0) { dup = 1; break; }
}
if (dup) continue;
double inf_w = ea->weight * eb->weight * 0.8;
if (inf_w < 0.05) inf_w = 0.05;
if (inf_w > 1.0) inf_w = 1.0;
strncpy(cands[ncands].from, ea->from_id, 63); cands[ncands].from[63] = '\0';
strncpy(cands[ncands].to, eb->to_id, 63); cands[ncands].to[63] = '\0';
cands[ncands].weight = inf_w;
ncands++;
}
}
for (int64_t k = 0; k < ncands; k++) {
engram_grow_edges();
EngramEdge* ne = &g->edges[g->edge_count];
memset(ne, 0, sizeof(*ne));
ne->id = engram_new_id();
/* Use strdup (not el_strdup) so these persist beyond the request. */
ne->from_id = strdup(cands[k].from);
ne->to_id = strdup(cands[k].to);
ne->relation = strdup("inferred");
ne->metadata = strdup("{}");
ne->weight = cands[k].weight;
ne->confidence = 0.8;
ne->created_at = now_ms;
ne->updated_at = now_ms;
ne->last_fired = now_ms;
ne->layer_id = ENGRAM_LAYER_DEFAULT;
g->edge_count++;
}
free(cands);
}
}
/* ── PASS 2: executive filter → working memory promotion ──────────── */
/* Step A: collect inhibitory suppressions from fired inhibitory edges.
* Layered consciousness: inhibition is ONLY recorded against targets
@@ -7193,66 +7080,6 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
g->nodes[i].working_memory_weight = wm_weights[i];
}
/* ── HEBBIAN STRENGTHENING: fire together, wire together ─────────────
* For each pair of co-promoted nodes (working_memory_weight > 0) that
* share an edge, boost that edge's weight by 0.05 (capped at 1.0).
* Also increment activation_count and update last_activated on promoted
* nodes this is what drives tier migration below. */
for (int64_t i = 0; i < g->node_count; i++) {
if (wm_weights[i] <= 0.0) continue;
EngramNode* n = &g->nodes[i];
n->activation_count++;
n->last_activated = now_ms;
n->updated_at = now_ms;
}
for (int64_t ei = 0; ei < g->edge_count; ei++) {
EngramEdge* e = &g->edges[ei];
if (!e->from_id || !e->to_id) continue;
int64_t src = engram_find_node_index(e->from_id);
int64_t tgt = engram_find_node_index(e->to_id);
if (src < 0 || tgt < 0) continue;
if (wm_weights[src] > 0.0 && wm_weights[tgt] > 0.0) {
e->weight += 0.05;
if (e->weight > 1.0) e->weight = 1.0;
e->last_fired = now_ms;
e->updated_at = now_ms;
}
}
/* ── TIER MIGRATION: promote nodes based on activation_count thresholds ─
* 04 Working
* 519 Episodic
* 2049 Semantic
* 50+ Procedural
* Only upgrade (never downgrade) to preserve earned tier. */
for (int64_t i = 0; i < g->node_count; i++) {
EngramNode* n = &g->nodes[i];
const char* target_tier = NULL;
int64_t ac = n->activation_count;
if (ac >= 50) target_tier = "Procedural";
else if (ac >= 20) target_tier = "Semantic";
else if (ac >= 5) target_tier = "Episodic";
else target_tier = "Working";
if (target_tier && n->tier && strcmp(n->tier, target_tier) != 0) {
/* Only upgrade (Working < Episodic < Semantic < Procedural). */
int cur_rank = 0, new_rank = 0;
if (strcmp(n->tier, "Working") == 0) cur_rank = 0;
else if (strcmp(n->tier, "Episodic") == 0) cur_rank = 1;
else if (strcmp(n->tier, "Semantic") == 0) cur_rank = 2;
else if (strcmp(n->tier, "Procedural") == 0) cur_rank = 3;
if (strcmp(target_tier, "Working") == 0) new_rank = 0;
else if (strcmp(target_tier, "Episodic") == 0) new_rank = 1;
else if (strcmp(target_tier, "Semantic") == 0) new_rank = 2;
else if (strcmp(target_tier, "Procedural") == 0) new_rank = 3;
if (new_rank > cur_rank) {
free(n->tier);
/* Use strdup (not el_strdup) so tier string persists beyond the request. */
n->tier = strdup(target_tier);
n->updated_at = now_ms;
}
}
}
/* ── Collect all background-activated nodes for the return value ────
* Callers see both layers. Context compilation uses only promoted nodes
* (working_memory_weight > 0). Sort: promoted first by wm_weight desc,
@@ -8026,97 +7853,6 @@ el_val_t engram_query_range(el_val_t start_ms_v, el_val_t end_ms_v) {
return el_wrap_str(b.buf);
}
/* ── engram_apply_decay_json — temporal decay maintenance ────────────────────
*
* Iterates ALL nodes and applies temporal decay to their stored `salience`
* field based on time elapsed since `last_activated`:
*
* new_salience = current_salience * decay_rate ^ hours_since_activation
*
* where decay_rate defaults to 0.5^(1/168) per hour (half-life one week),
* or the node's own `temporal_decay_rate` if non-zero.
*
* Nodes with temporal_decay_rate == 0 are NOT immune the global default
* applies. To make a node truly immune, set temporal_decay_rate to a very
* small positive value (e.g. 0.0001). Nodes that are "pinned" can be
* identified by a tier of "Procedural" those are skipped.
*
* After updating salience, nodes with salience < 0.05 AND tier == "Working"
* are pruned (deleted) unless they have no content (guard against garbage).
*
* Returns a JSON summary: {"updated": N, "pruned": N} */
el_val_t engram_apply_decay_json(void) {
EngramStore* g = engram_get();
int64_t now_ms = engram_now_ms();
int64_t updated = 0, pruned = 0;
for (int64_t i = 0; i < g->node_count; i++) {
EngramNode* n = &g->nodes[i];
/* Skip Procedural nodes (they are "locked in"). */
if (n->tier && strcmp(n->tier, "Procedural") == 0) continue;
int64_t age_ms = now_ms - n->last_activated;
if (age_ms <= 0) continue;
double lambda = (n->temporal_decay_rate > 0.0) ? n->temporal_decay_rate
: ENGRAM_DECAY_LAMBDA;
double age_hours = (double)age_ms / 3600000.0;
double decay_factor = exp(-lambda * age_hours / ENGRAM_T_HALF_HOURS);
double new_salience = n->salience * decay_factor;
if (new_salience < 0.0) new_salience = 0.0;
if (new_salience != n->salience) {
n->salience = new_salience;
n->updated_at = now_ms;
updated++;
}
}
/* Prune low-salience Working nodes. Walk backwards to allow in-place
* removal without invalidating indices. */
for (int64_t i = g->node_count - 1; i >= 0; i--) {
EngramNode* n = &g->nodes[i];
if (n->salience >= 0.05) continue;
/* Only prune Working tier nodes — higher tiers are protected. */
if (!n->tier || strcmp(n->tier, "Working") != 0) continue;
/* Guard: skip nodes with no content. */
if (!n->content || !*n->content) continue;
/* Free node strings. */
free(n->id); free(n->content); free(n->node_type); free(n->label);
free(n->tier); free(n->tags); free(n->metadata);
/* Shift remaining nodes down. */
for (int64_t j = i + 1; j < g->node_count; j++) {
g->nodes[j - 1] = g->nodes[j];
}
g->node_count--;
memset(&g->nodes[g->node_count], 0, sizeof(EngramNode));
pruned++;
}
/* Remove dangling edges for pruned nodes (any edge whose endpoint no
* longer exists in the node list). */
if (pruned > 0) {
int64_t w = 0;
for (int64_t r = 0; r < g->edge_count; r++) {
EngramEdge* e = &g->edges[r];
int dangling = 0;
if (e->from_id && engram_find_node_index(e->from_id) < 0) dangling = 1;
if (e->to_id && engram_find_node_index(e->to_id) < 0) dangling = 1;
if (dangling) {
free(e->id); free(e->from_id); free(e->to_id);
free(e->relation); free(e->metadata);
} else {
if (w != r) g->edges[w] = g->edges[r];
w++;
}
}
g->edge_count = w;
}
char buf[128];
snprintf(buf, sizeof(buf),
"{\"ok\":true,\"updated\":%lld,\"pruned\":%lld}",
(long long)updated, (long long)pruned);
return el_wrap_str(el_strdup(buf));
}
#ifdef HAVE_CURL
/* ── DHARMA network ─────────────────────────────────────────────────────────
* Real implementation. Peers are addressed by `dharma_id` either bare
@@ -52,6 +52,12 @@
typedef int64_t el_val_t;
/* HTTP request-handler function-pointer types. Public because soul modules (routes/chat/etc.)
* register handlers across translation units; previously defined only inside el_runtime.c, which
* made cross-module references (and the Windows build) fail. Home in the shared header. */
typedef el_val_t (*http_handler_fn)(el_val_t method, el_val_t path, el_val_t body);
typedef el_val_t (*http_handler4_fn)(el_val_t method, el_val_t path, el_val_t body, el_val_t headers);
#define EL_STR(s) ((el_val_t)(uintptr_t)(s))
#define EL_CSTR(v) ((const char*)(uintptr_t)(v))
#define EL_INT(v) (v)
@@ -227,8 +233,6 @@ el_val_t url_decode(el_val_t s); /* '+' → space, %XX → byte */
* {"p":[],"a":["href","title"],"strong":[],...}
* where each value is the array of attribute names allowed for that tag. */
el_val_t el_html_sanitize(el_val_t input_html, el_val_t allowlist_json);
el_val_t html_raw(el_val_t s);
el_val_t html_escape(el_val_t s);
/* ── Filesystem ──────────────────────────────────────────────────────────── */
@@ -533,12 +537,6 @@ el_val_t parse_int(el_val_t s, el_val_t default_val);
el_val_t exit_program(el_val_t code);
el_val_t getpid_now(void);
/* Self-terminating memory guard. Reads ELC_MAX_MEM_MB (default 512) and
* exits with code 1 if resident memory exceeds the limit. Call periodically
* during long compilation loops (e.g. after each function is compiled).
* Returns 0 when memory is within bounds. */
el_val_t el_mem_check(void);
/* ── CGI identity ─────────────────────────────────────────────────────────────
* Called at the start of main() in CGI programs (those with a `cgi {}` block).
* Records the program's DHARMA identity before any other code executes. */

Some files were not shown because too many files have changed in this diff Show More