Compare commits
50 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ca13471745 | |||
| a3358dfc95 | |||
| 85eee42106 | |||
| 5336cfe0a6 | |||
| 77a4bc9326 | |||
| 65ca0a3253 | |||
| 816b258255 | |||
| 0af39df16f | |||
| 5f6ce5ca1f | |||
| c20cb3b97c | |||
| f6a0777f90 | |||
| 7946b98d3d | |||
| 8cae0f94eb | |||
| 2a4c5c645a | |||
| 9e28defcab | |||
| 1507614dbf | |||
| e0b55c3080 | |||
| 0d299ee0f1 | |||
| 5e154fa152 | |||
| 89589864cd | |||
| eb13ce7910 | |||
| ce0d33ba93 | |||
| 02dc12d785 | |||
| 7aa847e32a | |||
| ee71423732 | |||
| bb64a236ed | |||
| 9a0266cbf9 | |||
| a72145b44e | |||
| 8affb1d6e0 | |||
| fa47b98d18 | |||
| 0a72fced28 | |||
| 791b0880b7 | |||
| 23552ed40a | |||
| 6838e5cbff | |||
| fa2b49365b | |||
| 971b21751a | |||
| 9f1db8278c | |||
| 3d05e0c2a9 | |||
| 3bf44dee2d | |||
| a43a35bd10 | |||
| afc92f4e33 | |||
| 005e84e5d3 | |||
| 7f03876e26 | |||
| 599073cb92 | |||
| 7f66529510 | |||
| 6ebe3d0d66 | |||
| 9f362c90e5 | |||
| 11dc138a93 | |||
| 227f158a05 | |||
| 97e484221d |
@@ -39,9 +39,9 @@ jobs:
|
||||
run: |
|
||||
dist/platform/elc-linux-amd64 elc-cli.el > dist/elc-gen2.c
|
||||
gcc -O2 \
|
||||
-I el-compiler/runtime \
|
||||
-I runtime \
|
||||
dist/elc-gen2.c \
|
||||
el-compiler/runtime/el_runtime.c \
|
||||
runtime/el_runtime.c \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm \
|
||||
-o dist/platform/elc
|
||||
chmod +x dist/platform/elc
|
||||
@@ -54,9 +54,9 @@ jobs:
|
||||
mkdir -p dist/bin
|
||||
dist/platform/elc elb.el > dist/elb.c
|
||||
gcc -O2 \
|
||||
-I el-compiler/runtime \
|
||||
-I runtime \
|
||||
dist/elb.c \
|
||||
el-compiler/runtime/el_runtime.c \
|
||||
runtime/el_runtime.c \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm \
|
||||
-o dist/bin/elb
|
||||
chmod +x dist/bin/elb
|
||||
@@ -91,7 +91,7 @@ jobs:
|
||||
- name: Precompile el_runtime.o
|
||||
run: |
|
||||
set -euo pipefail
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
gcc -O2 -c -I "$RUNTIME" "$RUNTIME/el_runtime.c" \
|
||||
-o /tmp/el_runtime.o
|
||||
echo "el_runtime.o compiled"
|
||||
@@ -100,7 +100,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_core.el > /tmp/el_native_core.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c /tmp/el_runtime.o \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_core
|
||||
@@ -110,7 +110,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_text.el > /tmp/el_native_text.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c /tmp/el_runtime.o \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_text
|
||||
@@ -120,7 +120,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_string.el > /tmp/el_native_string.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c /tmp/el_runtime.o \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_string
|
||||
@@ -130,7 +130,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_math.el > /tmp/el_native_math.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c /tmp/el_runtime.o \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_math
|
||||
@@ -140,7 +140,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_state.el > /tmp/el_native_state.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c /tmp/el_runtime.o \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_state
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_time.el > /tmp/el_native_time.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c /tmp/el_runtime.o \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_time
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_json.el > /tmp/el_native_json.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c /tmp/el_runtime.o \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_json
|
||||
@@ -170,7 +170,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_env.el > /tmp/el_native_env.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c /tmp/el_runtime.o \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_env
|
||||
@@ -180,7 +180,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_fs.el > /tmp/el_native_fs.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c /tmp/el_runtime.o \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_fs
|
||||
@@ -191,7 +191,7 @@ jobs:
|
||||
run: |
|
||||
ABS_ELB="$(pwd)/dist/bin/elb"
|
||||
ABS_ELC="$(pwd)/dist/platform/elc"
|
||||
ABS_RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
ABS_RUNTIME="$(pwd)/runtime"
|
||||
ABS_OUT="$(pwd)/dist/bin"
|
||||
(cd ../epm && "$ABS_ELB" --clean --elc="$ABS_ELC" --runtime="$ABS_RUNTIME" --out="$ABS_OUT")
|
||||
chmod +x dist/bin/epm
|
||||
@@ -202,7 +202,7 @@ jobs:
|
||||
run: |
|
||||
ABS_ELB="$(pwd)/dist/bin/elb"
|
||||
ABS_ELC="$(pwd)/dist/platform/elc"
|
||||
ABS_RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
ABS_RUNTIME="$(pwd)/runtime"
|
||||
ABS_OUT="$(pwd)/dist/bin"
|
||||
(cd tools/install && "$ABS_ELB" --clean --elc="$ABS_ELC" --runtime="$ABS_RUNTIME" --out="$ABS_OUT")
|
||||
chmod +x dist/bin/el-install
|
||||
@@ -251,7 +251,7 @@ jobs:
|
||||
--project=neuron-785695 \
|
||||
--package=el-runtime-c \
|
||||
--version="${VERSION}" \
|
||||
--source=el-compiler/runtime/el_runtime.c
|
||||
--source=runtime/el_runtime.c
|
||||
|
||||
gcloud artifacts generic upload \
|
||||
--repository=foundation-dev \
|
||||
@@ -259,7 +259,7 @@ jobs:
|
||||
--project=neuron-785695 \
|
||||
--package=el-runtime-h \
|
||||
--version="${VERSION}" \
|
||||
--source=el-compiler/runtime/el_runtime.h
|
||||
--source=runtime/el_runtime.h
|
||||
|
||||
gcloud artifacts generic upload \
|
||||
--repository=foundation-dev \
|
||||
@@ -267,7 +267,7 @@ jobs:
|
||||
--project=neuron-785695 \
|
||||
--package=el-runtime-js \
|
||||
--version="${VERSION}" \
|
||||
--source=el-compiler/runtime/el_runtime.js
|
||||
--source=runtime/el_runtime.js
|
||||
|
||||
echo "Published El SDK version=${VERSION} to foundation-dev"
|
||||
# Keep key alive for the ci-base rebuild step below
|
||||
@@ -306,9 +306,9 @@ jobs:
|
||||
FROM ${BASE}
|
||||
COPY dist/platform/elc /opt/el/dist/platform/elc
|
||||
COPY dist/bin/elb /opt/el/dist/bin/elb
|
||||
COPY el-compiler/runtime/el_runtime.c /opt/el/el-compiler/runtime/el_runtime.c
|
||||
COPY el-compiler/runtime/el_runtime.h /opt/el/el-compiler/runtime/el_runtime.h
|
||||
COPY el-compiler/runtime/el_runtime.js /opt/el/el-compiler/runtime/el_runtime.js
|
||||
COPY runtime/el_runtime.c /opt/el/runtime/el_runtime.c
|
||||
COPY runtime/el_runtime.h /opt/el/runtime/el_runtime.h
|
||||
COPY runtime/el_runtime.js /opt/el/runtime/el_runtime.js
|
||||
RUN chmod +x /opt/el/dist/platform/elc /opt/el/dist/bin/elb
|
||||
EOF
|
||||
|
||||
|
||||
@@ -46,9 +46,9 @@ jobs:
|
||||
run: |
|
||||
dist/platform/elc-linux-amd64 elc-cli.el > dist/elc-gen2.c
|
||||
gcc -O2 \
|
||||
-I el-compiler/runtime \
|
||||
-I runtime \
|
||||
dist/elc-gen2.c \
|
||||
el-compiler/runtime/el_runtime.c \
|
||||
runtime/el_runtime.c \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm \
|
||||
-o dist/platform/elc
|
||||
chmod +x dist/platform/elc
|
||||
@@ -84,7 +84,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_core.el > /tmp/el_native_core.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_core
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_text.el > /tmp/el_native_text.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_text
|
||||
@@ -104,7 +104,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_string.el > /tmp/el_native_string.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_string
|
||||
@@ -114,7 +114,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_math.el > /tmp/el_native_math.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_math
|
||||
@@ -124,7 +124,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_state.el > /tmp/el_native_state.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_state
|
||||
@@ -134,7 +134,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_time.el > /tmp/el_native_time.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_time
|
||||
@@ -144,7 +144,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_json.el > /tmp/el_native_json.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_json
|
||||
@@ -154,7 +154,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_env.el > /tmp/el_native_env.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_env
|
||||
@@ -164,7 +164,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_fs.el > /tmp/el_native_fs.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_fs
|
||||
@@ -176,9 +176,9 @@ jobs:
|
||||
mkdir -p dist/bin
|
||||
dist/platform/elc elb.el > dist/elb.c
|
||||
gcc -O2 \
|
||||
-I el-compiler/runtime \
|
||||
-I runtime \
|
||||
dist/elb.c \
|
||||
el-compiler/runtime/el_runtime.c \
|
||||
runtime/el_runtime.c \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm \
|
||||
-o dist/bin/elb
|
||||
chmod +x dist/bin/elb
|
||||
@@ -189,7 +189,7 @@ jobs:
|
||||
run: |
|
||||
ABS_ELB="$(pwd)/dist/bin/elb"
|
||||
ABS_ELC="$(pwd)/dist/platform/elc"
|
||||
ABS_RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
ABS_RUNTIME="$(pwd)/runtime"
|
||||
ABS_OUT="$(pwd)/dist/bin"
|
||||
(cd ../epm && "$ABS_ELB" --clean --elc="$ABS_ELC" --runtime="$ABS_RUNTIME" --out="$ABS_OUT")
|
||||
chmod +x dist/bin/epm
|
||||
@@ -200,7 +200,7 @@ jobs:
|
||||
run: |
|
||||
ABS_ELB="$(pwd)/dist/bin/elb"
|
||||
ABS_ELC="$(pwd)/dist/platform/elc"
|
||||
ABS_RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
ABS_RUNTIME="$(pwd)/runtime"
|
||||
ABS_OUT="$(pwd)/dist/bin"
|
||||
(cd tools/install && "$ABS_ELB" --clean --elc="$ABS_ELC" --runtime="$ABS_RUNTIME" --out="$ABS_OUT")
|
||||
chmod +x dist/bin/el-install
|
||||
@@ -244,7 +244,7 @@ jobs:
|
||||
--project=neuron-785695 \
|
||||
--package=el-runtime-c \
|
||||
--version="${VERSION}" \
|
||||
--source=el-compiler/runtime/el_runtime.c
|
||||
--source=runtime/el_runtime.c
|
||||
|
||||
gcloud artifacts generic upload \
|
||||
--repository=foundation-stage \
|
||||
@@ -252,7 +252,7 @@ jobs:
|
||||
--project=neuron-785695 \
|
||||
--package=el-runtime-h \
|
||||
--version="${VERSION}" \
|
||||
--source=el-compiler/runtime/el_runtime.h
|
||||
--source=runtime/el_runtime.h
|
||||
|
||||
echo "Published El SDK version=${VERSION} to foundation-stage"
|
||||
# Keep key alive for the ci-base rebuild step below
|
||||
@@ -290,9 +290,9 @@ jobs:
|
||||
FROM ${BASE}
|
||||
COPY dist/platform/elc /opt/el/dist/platform/elc
|
||||
COPY dist/bin/elb /opt/el/dist/bin/elb
|
||||
COPY el-compiler/runtime/el_runtime.c /opt/el/el-compiler/runtime/el_runtime.c
|
||||
COPY el-compiler/runtime/el_runtime.h /opt/el/el-compiler/runtime/el_runtime.h
|
||||
COPY el-compiler/runtime/el_runtime.js /opt/el/el-compiler/runtime/el_runtime.js
|
||||
COPY runtime/el_runtime.c /opt/el/runtime/el_runtime.c
|
||||
COPY runtime/el_runtime.h /opt/el/runtime/el_runtime.h
|
||||
COPY runtime/el_runtime.js /opt/el/runtime/el_runtime.js
|
||||
RUN chmod +x /opt/el/dist/platform/elc /opt/el/dist/bin/elb
|
||||
EOF
|
||||
|
||||
|
||||
@@ -47,9 +47,9 @@ jobs:
|
||||
mkdir -p dist/platform
|
||||
dist/platform/elc-linux-amd64 elc-cli.el > dist/elc-gen2.c
|
||||
gcc -O2 \
|
||||
-I el-compiler/runtime \
|
||||
-I runtime \
|
||||
dist/elc-gen2.c \
|
||||
el-compiler/runtime/el_runtime.c \
|
||||
runtime/el_runtime.c \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm \
|
||||
-o dist/platform/elc
|
||||
chmod +x dist/platform/elc
|
||||
@@ -62,9 +62,9 @@ jobs:
|
||||
mkdir -p dist/bin
|
||||
dist/platform/elc elb.el > dist/elb.c
|
||||
gcc -O2 \
|
||||
-I el-compiler/runtime \
|
||||
-I runtime \
|
||||
dist/elb.c \
|
||||
el-compiler/runtime/el_runtime.c \
|
||||
runtime/el_runtime.c \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm \
|
||||
-o dist/bin/elb
|
||||
chmod +x dist/bin/elb
|
||||
@@ -75,7 +75,7 @@ jobs:
|
||||
run: |
|
||||
ABS_ELB="$(pwd)/dist/bin/elb"
|
||||
ABS_ELC="$(pwd)/dist/platform/elc"
|
||||
ABS_RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
ABS_RUNTIME="$(pwd)/runtime"
|
||||
ABS_OUT="$(pwd)/dist/bin"
|
||||
(cd ../epm && "$ABS_ELB" --clean --elc="$ABS_ELC" --runtime="$ABS_RUNTIME" --out="$ABS_OUT")
|
||||
chmod +x dist/bin/epm
|
||||
@@ -86,7 +86,7 @@ jobs:
|
||||
run: |
|
||||
ABS_ELB="$(pwd)/dist/bin/elb"
|
||||
ABS_ELC="$(pwd)/dist/platform/elc"
|
||||
ABS_RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
ABS_RUNTIME="$(pwd)/runtime"
|
||||
ABS_OUT="$(pwd)/dist/bin"
|
||||
(cd tools/install && "$ABS_ELB" --clean --elc="$ABS_ELC" --runtime="$ABS_RUNTIME" --out="$ABS_OUT")
|
||||
chmod +x dist/bin/el-install
|
||||
@@ -121,7 +121,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_core.el > /tmp/el_native_core.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_core.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_core
|
||||
@@ -131,7 +131,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_text.el > /tmp/el_native_text.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_text.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_text
|
||||
@@ -141,7 +141,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_string.el > /tmp/el_native_string.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_string.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_string
|
||||
@@ -151,7 +151,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_math.el > /tmp/el_native_math.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_math.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_math
|
||||
@@ -161,7 +161,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_state.el > /tmp/el_native_state.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_state.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_state
|
||||
@@ -171,7 +171,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_time.el > /tmp/el_native_time.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_time.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_time
|
||||
@@ -181,7 +181,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_json.el > /tmp/el_native_json.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_json.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_json
|
||||
@@ -191,7 +191,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_env.el > /tmp/el_native_env.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_env.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_env
|
||||
@@ -201,7 +201,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ELC="$(pwd)/dist/platform/elc"
|
||||
RUNTIME="$(pwd)/el-compiler/runtime"
|
||||
RUNTIME="$(pwd)/runtime"
|
||||
"$ELC" --test tests/native/test_fs.el > /tmp/el_native_fs.c
|
||||
gcc -O2 -I "$RUNTIME" /tmp/el_native_fs.c "$RUNTIME/el_runtime.c" \
|
||||
-lcurl -lssl -lcrypto -lpthread -lm -o /tmp/el_native_fs
|
||||
@@ -216,8 +216,10 @@ jobs:
|
||||
cp lang/dist/platform/elc dist/sdk/bin/elc
|
||||
cp lang/dist/bin/elb dist/sdk/bin/elb
|
||||
cp lang/dist/bin/epm dist/sdk/bin/epm
|
||||
cp lang/el-compiler/runtime/el_runtime.c dist/sdk/runtime/
|
||||
cp lang/el-compiler/runtime/el_runtime.h dist/sdk/runtime/
|
||||
cp lang/runtime/el_runtime.c dist/sdk/runtime/
|
||||
cp lang/runtime/el_runtime.h dist/sdk/runtime/
|
||||
cp lang/runtime/engram_store.c dist/sdk/runtime/
|
||||
cp lang/runtime/engram_store.h dist/sdk/runtime/
|
||||
cp lang/runtime/*.el dist/sdk/runtime/
|
||||
tar -czf dist/el-sdk-latest.tar.gz -C dist/sdk .
|
||||
echo "SDK tarball bundled: dist/el-sdk-latest.tar.gz"
|
||||
@@ -274,8 +276,10 @@ jobs:
|
||||
|
||||
# Per-file assets (downstream CI needs these individually)
|
||||
upload_asset lang/dist/platform/elc elc
|
||||
upload_asset lang/el-compiler/runtime/el_runtime.c el_runtime.c
|
||||
upload_asset lang/el-compiler/runtime/el_runtime.h el_runtime.h
|
||||
upload_asset lang/runtime/el_runtime.c el_runtime.c
|
||||
upload_asset lang/runtime/el_runtime.h el_runtime.h
|
||||
upload_asset lang/runtime/engram_store.c engram_store.c
|
||||
upload_asset lang/runtime/engram_store.h engram_store.h
|
||||
|
||||
# SDK bundle and installer binary
|
||||
upload_asset dist/el-sdk-latest.tar.gz el-sdk-latest.tar.gz
|
||||
@@ -328,7 +332,7 @@ jobs:
|
||||
--project=neuron-785695 \
|
||||
--package=el-runtime-c \
|
||||
--version="${VERSION}" \
|
||||
--source=el-compiler/runtime/el_runtime.c
|
||||
--source=runtime/el_runtime.c
|
||||
|
||||
gcloud artifacts generic upload \
|
||||
--repository=foundation-prod \
|
||||
@@ -336,7 +340,7 @@ jobs:
|
||||
--project=neuron-785695 \
|
||||
--package=el-runtime-h \
|
||||
--version="${VERSION}" \
|
||||
--source=el-compiler/runtime/el_runtime.h
|
||||
--source=runtime/el_runtime.h
|
||||
|
||||
gcloud artifacts generic upload \
|
||||
--repository=foundation-prod \
|
||||
@@ -344,7 +348,7 @@ jobs:
|
||||
--project=neuron-785695 \
|
||||
--package=el-runtime-js \
|
||||
--version="${VERSION}" \
|
||||
--source=el-compiler/runtime/el_runtime.js
|
||||
--source=runtime/el_runtime.js
|
||||
|
||||
echo "Published El SDK version=${VERSION} to foundation-prod"
|
||||
# Keep key alive for the ci-base rebuild step below
|
||||
@@ -382,9 +386,9 @@ jobs:
|
||||
FROM ${BASE}
|
||||
COPY dist/platform/elc /opt/el/dist/platform/elc
|
||||
COPY dist/bin/elb /opt/el/dist/bin/elb
|
||||
COPY el-compiler/runtime/el_runtime.c /opt/el/el-compiler/runtime/el_runtime.c
|
||||
COPY el-compiler/runtime/el_runtime.h /opt/el/el-compiler/runtime/el_runtime.h
|
||||
COPY el-compiler/runtime/el_runtime.js /opt/el/el-compiler/runtime/el_runtime.js
|
||||
COPY runtime/el_runtime.c /opt/el/runtime/el_runtime.c
|
||||
COPY runtime/el_runtime.h /opt/el/runtime/el_runtime.h
|
||||
COPY runtime/el_runtime.js /opt/el/runtime/el_runtime.js
|
||||
RUN chmod +x /opt/el/dist/platform/elc /opt/el/dist/bin/elb
|
||||
EOF
|
||||
|
||||
|
||||
@@ -6,13 +6,13 @@ set -euo pipefail
|
||||
|
||||
ROOT="$(git rev-parse --show-toplevel)"
|
||||
LANG_DIR="$ROOT/lang"
|
||||
RUNTIME="$LANG_DIR/el-compiler/runtime"
|
||||
RUNTIME="$LANG_DIR/runtime"
|
||||
ELC="$LANG_DIR/dist/platform/elc"
|
||||
|
||||
# If elc isn't built yet, skip with a warning rather than blocking
|
||||
if [ ! -x "$ELC" ]; then
|
||||
echo "⚠ elc not found at lang/dist/platform/elc — skipping pre-commit tests"
|
||||
echo " Build it first: cd lang && gcc -O2 -I el-compiler/runtime dist/elc-bootstrap.c el-compiler/runtime/el_runtime.c -lcurl -lpthread -o dist/elc-gen2 && ./dist/elc-gen2 el-compiler/src/compiler.el > /tmp/elc.c && gcc -O2 -I el-compiler/runtime /tmp/elc.c el-compiler/runtime/el_runtime.c -lcurl -lpthread -o dist/platform/elc"
|
||||
echo " Build it first: cd lang && gcc -O2 -I runtime dist/elc-bootstrap.c runtime/el_runtime.c -lcurl -lpthread -o dist/elc-gen2 && ./dist/elc-gen2 el-compiler/src/compiler.el > /tmp/elc.c && gcc -O2 -I runtime /tmp/elc.c runtime/el_runtime.c -lcurl -lpthread -o dist/platform/elc"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
# Reasoning Operators — Decisions & Reversal
|
||||
|
||||
**Date:** 2026-08-13
|
||||
**Branch:** `engram-tiered-storage` (worktree `/tmp/engram-tiered-wt`)
|
||||
**Status:** staged locally — NOT pushed, NOT tagged, NOT merged. Live `:8742` untouched.
|
||||
|
||||
## What this adds
|
||||
|
||||
A **REASONING layer** built as pure C compositions over the already-live §5 geometry
|
||||
OPERATORS (`engram_geometry.{h,c}`: overlap, subtract, setdiff, combine, distance,
|
||||
analogy). Where the operators are a relational algebra over neighborhood descriptors,
|
||||
these are reasoning *modes* built by chaining that algebra. New files:
|
||||
|
||||
- `lang/runtime/engram_reason.h` — public API for the five modes + a shared
|
||||
point-to-manifold fit primitive.
|
||||
- `lang/runtime/engram_reason.c` — implementations. READ-ONLY over descriptor inputs,
|
||||
`stdlib + libm` only, touches no store / index / activation. All geometry is
|
||||
delegated to `engram_geo_*`; this file only composes.
|
||||
- `engram/test/test_reason.c` + `engram/test/run_reason_tests.sh` — closed-form
|
||||
constructed tests (hand-built descriptors with known answers), PERF + ASan/UBSan.
|
||||
|
||||
El-exposure (pass-through, no self-host fold):
|
||||
- `lang/runtime/el_runtime.c` — `+#include "engram_reason.h"` and the builtin
|
||||
`engram_reason_analogy_json(a_csv,b_csv,c_csv)`.
|
||||
- `lang/runtime/el_runtime.h` — its declaration.
|
||||
- `lang/runtime/el_seed.c` — native `__engram_reason_analogy_json` wrapper (same
|
||||
C-table wiring as the §5 ops).
|
||||
|
||||
## The five modes — signatures & composition
|
||||
|
||||
| Mode | C entry point | Composes |
|
||||
|------|---------------|----------|
|
||||
| **ANALOGY** `A:B :: C:?` | `engram_reason_analogy(A,B,C,candidates,n,out)` | `engram_geo_analogy` (Procrustes R) + `engram_geo_analogy_apply` + centroid L2. Learns `R_{A→B}` = `engram_geo_analogy(B,A)` (that op returns R with `apply(R, Y-axis)≈X-axis`), reconstructs the residual translation `t = c_B − R·c_A`, maps `mapped = R·c_C + t`, ranks candidates by distance. |
|
||||
| **INDUCTION** `{E_i}→rule` | `engram_reason_induce(examples,n,top_axes,ext_floor,out)` + `engram_reason_membership` | `engram_geo_combine` folded left→right → pooled "rule" descriptor; shared subspace surfaces as the dominant pooled axes. Membership = point-to-manifold fit. |
|
||||
| **ABDUCTION** `x→best H` | `engram_reason_abduce(obs,dim,hyps,n,ext_floor,out)` | shared `engram_reason_point_fit` against each hypothesis; argmax fit score; full ranking. |
|
||||
| **CAUSAL** `x?y \| Z,t` | `engram_reason_causal(x,y,confounders,nZ,t_x,t_y,drop_frac,out)` | centroid cosine (raw correlation) + `engram_geo_subtract` residual-centroid (control for each confounder, take the strongest single explainer) + temporal precedence. Verdict `DIRECTED` / `CONFOUNDED` / `NONE` + a `confounded` flag. |
|
||||
| **PLANNING** `start→goal` | `engram_reason_plan(nodes,n,start,goal,radius,use_w,out)` | `engram_geo_distance` as edge weights over neighborhoods within `radius`; O(n²) Dijkstra → discrete geodesic path. |
|
||||
|
||||
Shared primitive `engram_reason_point_fit` splits `(x − centroid)` into an in-subspace
|
||||
Mahalanobis distance (scaled by axis extents) and an orthogonal off-model residual;
|
||||
it is the single engine under INDUCTION's membership test and ABDUCTION's ranking.
|
||||
|
||||
## Proof (DONE-WITH-PROOF)
|
||||
|
||||
`engram/test/run_reason_tests.sh`: **33/33 checks, 0 failures** on BOTH passes
|
||||
(PERF -O2, and ASan+UBSan). macOS `leaks --atExit`: **0 leaks / 0 total leaked bytes**.
|
||||
|
||||
Per-mode closed-form assertions actually exercised:
|
||||
- **ANALOGY** — A→B = +90° rotation in e0-e1 plane + a +5 shift in e2; Procrustes
|
||||
residual `~0`; predicted point `(0,2,5,0)` recovered exactly; nearest candidate =
|
||||
the planted true D (index 1), distance `~0`.
|
||||
- **INDUCTION** — 3 examples sharing span(e0,e1) (extents 1.0 / 0.8) each with a small
|
||||
idiosyncratic axis (e2 or e3); induced top-2 axes lie in span(e0,e1) (extents
|
||||
recovered ~1.0 / ~0.8); held-out in-plane point fits (membership 0.885), off-subspace
|
||||
point rejected (0.100), in-plane-but-far point rejected (0.039).
|
||||
- **ABDUCTION** — observation planted inside H1 among {H0,H1,H2}; best = H1, rank[0] = H1,
|
||||
H1 smallest distance.
|
||||
- **CAUSAL** — chain A→B→C along e0 (t 1<2<3) + confounder Z(e1) that leaks into A and
|
||||
drives D(t=4): A→B and B→C flagged `DIRECTED` with correct precedence and association
|
||||
that survives control; A–D `CONFOUNDED` (raw |cos|=0.707 collapses to 0.0 under
|
||||
control) with `confounded=1`; B–D `NONE` (no association).
|
||||
- **PLANNING** — 6 neighborhoods on a semicircle (r=10); `neighbor_radius=7` admits only
|
||||
consecutive hops; plan = `[0,1,2,3,4,5]` (the arc), cost `30.90` (> the 20-unit chord,
|
||||
confirming it is the geodesic through the manifold, not a straight jump); a too-small
|
||||
radius correctly yields `reached=0`.
|
||||
|
||||
## El-exposure status
|
||||
|
||||
- **ANALOGY is el-callable** via the same pass-through the §5 operators use. Proof: a
|
||||
container-capped fold (`capfold.sh`, peak ~0 GB) of a demo `.el` through the shipped
|
||||
`lang/dist/platform/elc` emits a *direct C call* `engram_reason_analogy_json(A,B,A)`
|
||||
(no registration, no self-host fold); the generated C links against `el_runtime.c` +
|
||||
`engram_reason.c` + geometry/store/vindex and runs end-to-end. (The standalone demo's
|
||||
store copy boots 0 nodes — a pre-existing quirk that hits the *shipped geo demo
|
||||
identically* — so the call returns `{"error":"geometry unavailable"}`; this still proves
|
||||
the compiled El → C reasoning symbol → JSON chain executes. Numeric correctness on real
|
||||
data is covered by the C test.) This compile also confirms `el_runtime.c` +
|
||||
`engram_reason.c` compile and link clean.
|
||||
- **INDUCTION / ABDUCTION / CAUSAL / PLANNING are C-layer only for now.** Their inputs are
|
||||
candidate *sets*, raw *points*, and *timestamps* that do not map to the flat comma-
|
||||
separated-seed El ABI. A richer marshalling surface would touch the codegen/registration
|
||||
path and risk an uncapped fold — explicitly deferred per the hard rail. The C functions
|
||||
are fully proven and callable from any C caller today.
|
||||
|
||||
## Build wiring (for the later cutover/durability pass)
|
||||
|
||||
`engram_reason.c` must be added to the engram server link line **alongside**
|
||||
`engram_geometry.c` (the heavy-runtime path `cc dist/engram.c el_runtime.c
|
||||
engram_store.c engram_geometry.c engram_vindex.c …`). `el_runtime.c` now
|
||||
`#include`s `engram_reason.h` and references `engram_reason_analogy_json`, so a build
|
||||
that omits `engram_reason.c` will fail to link that symbol. One-line addition, same as
|
||||
how `engram_geometry.c` was originally added.
|
||||
|
||||
## Reversal
|
||||
|
||||
Fully additive; nothing existing was modified in behavior. To revert:
|
||||
|
||||
1. Delete `lang/runtime/engram_reason.h`, `lang/runtime/engram_reason.c`,
|
||||
`engram/test/test_reason.c`, `engram/test/run_reason_tests.sh`, and this doc.
|
||||
2. In `lang/runtime/el_runtime.c`: remove `#include "engram_reason.h"` and the
|
||||
`engram_reason_analogy_json` function.
|
||||
3. In `lang/runtime/el_runtime.h`: remove the `engram_reason_analogy_json` declaration.
|
||||
4. In `lang/runtime/el_seed.c`: remove the `__engram_reason_analogy_json` wrapper.
|
||||
5. Remove `engram_reason.c` from any server link line if the cutover added it.
|
||||
|
||||
No store, schema, config, WAL, or on-disk format was touched; no data migration exists,
|
||||
so reversal is a pure code removal with no state to undo.
|
||||
@@ -0,0 +1,128 @@
|
||||
# Verifier Layer — Grounding + Consistency (decisions + reversal)
|
||||
|
||||
**Date:** 2026-08-13
|
||||
**Branch:** `engram-tiered-storage` (worktree `/tmp/engram-tiered-wt`), atop `a3358df`
|
||||
**Scope:** additive, read-only, staged. No push, no tag, no merge. Live `:8742` untouched.
|
||||
|
||||
## What this adds
|
||||
|
||||
The VERIFIER layer — the "disposes" half of the propose→verify loop. The geometry
|
||||
PROPOSES (cheap, creative, sometimes wrong); the verifier DISPOSES, catching the
|
||||
class of failure no grammar check sees: a fluent, confident, WRONG output — the
|
||||
**plausible lie**.
|
||||
|
||||
Motivating failure (tonight's PT translation): a deleted negation turned
|
||||
"you never fought" into "you argued" — reassurance inverted into accusation,
|
||||
grammatical and invisible, catchable only by the geometry.
|
||||
|
||||
Two checks, both pure C11 (stdlib + libm), read-only over their inputs, touching no
|
||||
store / index / activation. Every geometry op is delegated to the already-shipped
|
||||
reasoning + §5 operator primitives; this layer only composes and thresholds.
|
||||
|
||||
### Files added
|
||||
- `lang/runtime/engram_verify.h` — API + design contract.
|
||||
- `lang/runtime/engram_verify.c` — implementation.
|
||||
- `engram/test/test_verify.c` — closed-form constructed cases (29 checks).
|
||||
- `engram/test/run_verify_tests.sh` — two-pass runner (PERF, then ASan/UBSan).
|
||||
|
||||
No existing file was modified.
|
||||
|
||||
## C signatures + how each composes the existing primitives
|
||||
|
||||
### GROUNDING (anti-hallucination)
|
||||
```c
|
||||
int engram_verify_grounding(const float* claim, int dim,
|
||||
const GeoDescriptor* const* evidence, int n_evidence,
|
||||
double ext_floor, double ground_threshold,
|
||||
GeoGrounding* out);
|
||||
```
|
||||
Fits the claim POINT against every real evidence neighborhood via
|
||||
`engram_reason_point_fit` (in-distribution Mahalanobis + off-model orthogonal
|
||||
residual) and keeps the BEST supporter. Grounded iff best fit score ≥
|
||||
`ground_threshold`. Deliberately an ABSOLUTE-THRESHOLD gate, distinct from ABDUCTION
|
||||
(which always ranks and picks a winner): grounding asks the prior question — "is there
|
||||
any real support at all?" — and may answer no. The off-model `best_ortho` residual is
|
||||
the sharpest hallucination signal: energy in a direction the manifold does not span.
|
||||
|
||||
### CONSISTENCY (contradiction detection)
|
||||
```c
|
||||
int engram_verify_consistency(const float* claim, int dim,
|
||||
const GeoDescriptor* context,
|
||||
const GeoDescriptor* pole_pos, const GeoDescriptor* pole_neg,
|
||||
const GeoDescriptor* forbidden,
|
||||
double ext_floor, double deadzone_frac,
|
||||
double forbidden_thresh, double max_distance,
|
||||
GeoConsistency* out);
|
||||
```
|
||||
Two independent sub-checks (either can fire; both flags reported):
|
||||
|
||||
- **(a) POLARITY / negation inversion** — the reassurance→accusation catch.
|
||||
A polarity axis `p = (c_pos − c_neg)/‖·‖` is defined by two REAL poles (affirm vs
|
||||
negate), midpoint `o = ½(c_pos + c_neg)`. Signed sides: `claim_side = p·(claim − o)`,
|
||||
`ref_side = p·(c_context − o)`. If they have OPPOSITE sign and both clear the neutral
|
||||
deadzone (`deadzone_frac·½‖c_pos−c_neg‖`), the claim asserts the polarity opposite to
|
||||
the grounded truth → inversion flagged. Pure dot products / projections over the same
|
||||
centroids the geometry already computes.
|
||||
- **(b) GEOMETRIC contradiction** — claim sits INSIDE a `forbidden` region it must be
|
||||
far from (`engram_reason_point_fit` score ≥ `forbidden_thresh`), OR violates a
|
||||
max-distance constraint to `context` (`L2 > max_distance`).
|
||||
|
||||
## Proof (constructed cases — demonstrate, not declare)
|
||||
|
||||
`./engram/test/run_verify_tests.sh` → **29 checks, 0 failures** in BOTH passes
|
||||
(PERF -O2, and ASan+UBSan -O1). macOS `leaks --atExit`: **0 leaks for 0 total leaked
|
||||
bytes**.
|
||||
|
||||
Key demonstrated numbers:
|
||||
- Grounding IN (claim inside E0): score 0.885, grounded=1, ortho≈0.
|
||||
- Grounding OUT (claim floating along unmodeled e2): score 0.0004, grounded=0,
|
||||
ortho=50.0 (the hallucination signal), nearest centroid L2=50.
|
||||
- **Negation inversion (the catch):** truth "never fought" ref_side=−5.0, lie
|
||||
"you argued" claim_side=+4.0 → opposite poles → `inverted=1`, verdict=POLARITY,
|
||||
consistency=0. Faithful claim (−4.0, same pole) → inverted=0, verdict=OK,
|
||||
consistency=1. Neutral claim inside deadzone → not triggered.
|
||||
- Geometric: claim inside forbidden region → geo_violation=1 (forb_fit 0.99);
|
||||
claim beyond max_distance → geo_violation=1 (ctx_dist 8.0 > 3.0).
|
||||
- **Combined (the whole point):** a claim that is GROUNDED in real vocabulary
|
||||
(grounded=1, score 1.0) yet polarity-inverted is PASSED by grounding and CAUGHT
|
||||
only by consistency (verdict=POLARITY). Grounding alone is insufficient; consistency
|
||||
is the catch.
|
||||
|
||||
## el-exposure — DEFERRED (matches reasoning-agent precedent)
|
||||
|
||||
Not exposed as el builtins this pass. The reasoning agent exposed ONLY `analogy`
|
||||
(three seed-identified neighborhoods → the clean fixed-arity seed-CSV→descriptor JSON
|
||||
pattern) and deferred its point-input / variadic-set modes (abduction, induction,
|
||||
causal, planning). The verifier's grounding (claim POINT + variadic evidence SET) and
|
||||
consistency (claim POINT + context + two poles + forbidden + scalar thresholds) are
|
||||
exactly those shapes: no clean fixed-arity seed-CSV JSON mapping exists, and adding one
|
||||
would require new JSON list-of-lists + point-vector marshaling absent from the codebase,
|
||||
risking an elc rebuild/fold (violates the capped-fold-only rail). A claim is also an
|
||||
arbitrary proposed POINT, not necessarily an existing node — so the point-native C API
|
||||
is the correct primitive. Deferred deliberately; the C layer is complete and proven.
|
||||
|
||||
When exposed later, follow the same additive pass-through pattern used for the geo
|
||||
operators: native `engram_verify_*_json(el_val_t ...)` in `el_runtime.c` (heavy runtime)
|
||||
+ a `__engram_verify_*_json` wrapper in `el_seed.c`, resolving seed CSVs → descriptors
|
||||
and marshaling the claim vector — no fold needed for callability (shipped elc passes
|
||||
unknown-ident builtin calls straight through to the heavy-runtime C symbols).
|
||||
|
||||
## Reach checks (FORMAL / CAUSAL / PREDICTIVE) — NOT STARTED
|
||||
|
||||
Honestly not started this pass; the two tractable-now checks (grounding + consistency)
|
||||
were driven to done-with-proof first as specified.
|
||||
- **CAUSAL** already exists as a REASONING operator (`engram_reason_causal`,
|
||||
intervention/temporal-precedence over the typed causal graph); a verifier wrapper that
|
||||
checks "does the claimed cause actually precede/influence" would compose it — not built.
|
||||
- **FORMAL** (logical consistency of a claim SET) needs an external checker (SMT/proof
|
||||
kernel) — the non-geometric seam; not built.
|
||||
- **PREDICTIVE** (commit a prediction, check vs outcome, restructure on error) is the CGI
|
||||
research frontier; not built.
|
||||
|
||||
## Reversal
|
||||
|
||||
Fully additive. To reverse: delete the four added files
|
||||
(`lang/runtime/engram_verify.{h,c}`, `engram/test/test_verify.c`,
|
||||
`engram/test/run_verify_tests.sh`) or `git revert` this commit. Nothing else references
|
||||
them; no build wiring, no el registration, no store schema, no runtime path was changed.
|
||||
Live `:8742` was never touched.
|
||||
+1
-1
@@ -22,7 +22,7 @@ cd "$(dirname "$0")"
|
||||
|
||||
EL_HOME="${EL_HOME:-$(cd ../.. && pwd)/el}"
|
||||
ELC="${ELC:-${EL_HOME}/dist/platform/elc}"
|
||||
RUNTIME_DIR="${EL_HOME}/el-compiler/runtime"
|
||||
RUNTIME_DIR="${EL_HOME}/runtime"
|
||||
SRC_DIR="$(cd .. && pwd)/src"
|
||||
|
||||
if [ ! -x "${ELC}" ]; then
|
||||
|
||||
@@ -49,6 +49,12 @@ jobs:
|
||||
echo "Downloading el_runtime.h..."
|
||||
curl -fsSL "${RELEASE_BASE}/el_runtime.h" -o /usr/local/lib/el/el_runtime.h
|
||||
|
||||
echo "Downloading engram_store.c..."
|
||||
curl -fsSL "${RELEASE_BASE}/engram_store.c" -o /usr/local/lib/el/engram_store.c
|
||||
|
||||
echo "Downloading engram_store.h..."
|
||||
curl -fsSL "${RELEASE_BASE}/engram_store.h" -o /usr/local/lib/el/engram_store.h
|
||||
|
||||
echo "El SDK installed:"
|
||||
elc --version || true
|
||||
|
||||
@@ -67,6 +73,7 @@ jobs:
|
||||
-o dist/engram \
|
||||
dist/engram.c \
|
||||
/usr/local/lib/el/el_runtime.c \
|
||||
/usr/local/lib/el/engram_store.c \
|
||||
-lcurl -lpthread
|
||||
echo "Linked dist/engram"
|
||||
ls -lh dist/engram
|
||||
|
||||
+5
-2
@@ -1,3 +1,6 @@
|
||||
target/
|
||||
*.db
|
||||
.DS_Store
|
||||
*.db
|
||||
*.elc
|
||||
*.elh
|
||||
dist/
|
||||
target/
|
||||
|
||||
Vendored
BIN
Binary file not shown.
Vendored
+137
-35
@@ -10,6 +10,8 @@ el_val_t query_param(el_val_t path, el_val_t key);
|
||||
el_val_t query_int(el_val_t path, el_val_t key, el_val_t default_val);
|
||||
el_val_t extract_id(el_val_t path, el_val_t prefix);
|
||||
el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_act_stats(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_text_health(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t persist_canonical(void);
|
||||
el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_get_node(el_val_t method, el_val_t path, el_val_t body);
|
||||
@@ -18,16 +20,19 @@ el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_search(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_create_edges_batch(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_save(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_load(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_health(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_embed_backfill(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_load_merge(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_capture_knowledge(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_similarity(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t check_auth_ok(el_val_t method, el_val_t body);
|
||||
el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body);
|
||||
|
||||
@@ -116,11 +121,20 @@ el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_act_stats(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return engram_act_stats_json();
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_text_health(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return engram_text_health_json();
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t persist_canonical(void) {
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_1 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_1 = (EL_STR("/tmp/engram")); } else { _if_result_1 = (dir_raw); } _if_result_1; });
|
||||
engram_save(el_str_concat(dir, EL_STR("/snapshot.json")));
|
||||
return 1;
|
||||
return engram_save(el_str_concat(dir, EL_STR("/snapshot.json")));
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -128,9 +142,18 @@ el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t content = json_get_string(body, EL_STR("content"));
|
||||
el_val_t nt_raw = json_get_string(body, EL_STR("node_type"));
|
||||
el_val_t node_type = ({ el_val_t _if_result_2 = 0; if (str_eq(nt_raw, EL_STR(""))) { _if_result_2 = (EL_STR("Memory")); } else { _if_result_2 = (nt_raw); } _if_result_2; });
|
||||
el_val_t sal_raw = json_get_float(body, EL_STR("salience"));
|
||||
el_val_t salience = ({ el_val_t _if_result_3 = 0; if ((sal_raw == el_from_float(0.0))) { _if_result_3 = (el_from_float(0.5)); } else { _if_result_3 = (sal_raw); } _if_result_3; });
|
||||
el_val_t id = engram_node(content, node_type, salience);
|
||||
el_val_t sal_present = json_get_raw(body, EL_STR("salience"));
|
||||
el_val_t salience = ({ el_val_t _if_result_3 = 0; if (str_eq(sal_present, EL_STR(""))) { _if_result_3 = (el_from_float(0.5)); } else { _if_result_3 = (json_get_float(body, EL_STR("salience"))); } _if_result_3; });
|
||||
el_val_t label_raw = json_get_string(body, EL_STR("label"));
|
||||
el_val_t label = ({ el_val_t _if_result_4 = 0; if (str_eq(label_raw, EL_STR(""))) { _if_result_4 = (content); } else { _if_result_4 = (label_raw); } _if_result_4; });
|
||||
el_val_t imp_present = json_get_raw(body, EL_STR("importance"));
|
||||
el_val_t importance = ({ el_val_t _if_result_5 = 0; if (str_eq(imp_present, EL_STR(""))) { _if_result_5 = (el_from_float(0.5)); } else { _if_result_5 = (json_get_float(body, EL_STR("importance"))); } _if_result_5; });
|
||||
el_val_t conf_present = json_get_raw(body, EL_STR("confidence"));
|
||||
el_val_t confidence = ({ el_val_t _if_result_6 = 0; if (str_eq(conf_present, EL_STR(""))) { _if_result_6 = (el_from_float(1.0)); } else { _if_result_6 = (json_get_float(body, EL_STR("confidence"))); } _if_result_6; });
|
||||
el_val_t tier_raw = json_get_string(body, EL_STR("tier"));
|
||||
el_val_t tier = ({ el_val_t _if_result_7 = 0; if (str_eq(tier_raw, EL_STR(""))) { _if_result_7 = (EL_STR("Working")); } else { _if_result_7 = (tier_raw); } _if_result_7; });
|
||||
el_val_t tags = json_get_string(body, EL_STR("tags"));
|
||||
el_val_t id = engram_node_full(content, node_type, label, salience, importance, confidence, tier, tags);
|
||||
el_val_t saved = persist_canonical();
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"id\":\""), id), EL_STR("\",\"content\":\"")), content), EL_STR("\",\"node_type\":\"")), node_type), EL_STR("\"}"));
|
||||
return 0;
|
||||
@@ -158,7 +181,7 @@ el_val_t route_scan_nodes(el_val_t method, el_val_t path, el_val_t body) {
|
||||
|
||||
el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_4 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_4 = (EL_STR("/tmp/engram")); } else { _if_result_4 = (dir_raw); } _if_result_4; });
|
||||
el_val_t dir = ({ el_val_t _if_result_8 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_8 = (EL_STR("/tmp/engram")); } else { _if_result_8 = (dir_raw); } _if_result_8; });
|
||||
el_val_t snap_path = el_str_concat(dir, EL_STR("/.scan-export.json"));
|
||||
engram_save(snap_path);
|
||||
el_val_t snap = fs_read(snap_path);
|
||||
@@ -174,22 +197,22 @@ el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body) {
|
||||
}
|
||||
|
||||
el_val_t route_search(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t q = ({ el_val_t _if_result_5 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_5 = (query_param(path, EL_STR("q"))); } else { _if_result_5 = (json_get_string(body, EL_STR("query"))); } _if_result_5; });
|
||||
el_val_t q = ({ el_val_t _if_result_9 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_9 = (query_param(path, EL_STR("q"))); } else { _if_result_9 = (json_get_string(body, EL_STR("query"))); } _if_result_9; });
|
||||
el_val_t lim_url = query_int(path, EL_STR("limit"), 0);
|
||||
el_val_t lim_body = json_get_int(body, EL_STR("limit"));
|
||||
el_val_t lim_either = ({ el_val_t _if_result_6 = 0; if ((lim_url > 0)) { _if_result_6 = (lim_url); } else { _if_result_6 = (lim_body); } _if_result_6; });
|
||||
el_val_t limit = ({ el_val_t _if_result_7 = 0; if ((lim_either > 0)) { _if_result_7 = (lim_either); } else { _if_result_7 = (20); } _if_result_7; });
|
||||
el_val_t lim_either = ({ el_val_t _if_result_10 = 0; if ((lim_url > 0)) { _if_result_10 = (lim_url); } else { _if_result_10 = (lim_body); } _if_result_10; });
|
||||
el_val_t limit = ({ el_val_t _if_result_11 = 0; if ((lim_either > 0)) { _if_result_11 = (lim_either); } else { _if_result_11 = (20); } _if_result_11; });
|
||||
return engram_search_json(q, limit);
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t q = ({ el_val_t _if_result_8 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_8 = (query_param(path, EL_STR("q"))); } else { _if_result_8 = (json_get_string(body, EL_STR("query"))); } _if_result_8; });
|
||||
el_val_t q = ({ el_val_t _if_result_12 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_12 = (query_param(path, EL_STR("q"))); } else { _if_result_12 = (json_get_string(body, EL_STR("query"))); } _if_result_12; });
|
||||
if (str_eq(q, EL_STR(""))) {
|
||||
return err_json(EL_STR("missing query"));
|
||||
}
|
||||
el_val_t d_raw = ({ el_val_t _if_result_9 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_9 = (query_int(path, EL_STR("depth"), 3)); } else { _if_result_9 = (json_get_int(body, EL_STR("depth"))); } _if_result_9; });
|
||||
el_val_t depth = ({ el_val_t _if_result_10 = 0; if ((d_raw > 0)) { _if_result_10 = (d_raw); } else { _if_result_10 = (3); } _if_result_10; });
|
||||
el_val_t d_raw = ({ el_val_t _if_result_13 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_13 = (query_int(path, EL_STR("depth"), 3)); } else { _if_result_13 = (json_get_int(body, EL_STR("depth"))); } _if_result_13; });
|
||||
el_val_t depth = ({ el_val_t _if_result_14 = 0; if ((d_raw > 0)) { _if_result_14 = (d_raw); } else { _if_result_14 = (3); } _if_result_14; });
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"results\":"), engram_activate_json(q, depth)), EL_STR("}"));
|
||||
return 0;
|
||||
}
|
||||
@@ -198,15 +221,50 @@ el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t from_id = json_get_string(body, EL_STR("from_id"));
|
||||
el_val_t to_id = json_get_string(body, EL_STR("to_id"));
|
||||
el_val_t rel_raw = json_get_string(body, EL_STR("relation"));
|
||||
el_val_t relation = ({ el_val_t _if_result_11 = 0; if (str_eq(rel_raw, EL_STR(""))) { _if_result_11 = (EL_STR("associates")); } else { _if_result_11 = (rel_raw); } _if_result_11; });
|
||||
el_val_t w_raw = json_get_float(body, EL_STR("weight"));
|
||||
el_val_t weight = ({ el_val_t _if_result_12 = 0; if ((w_raw == el_from_float(0.0))) { _if_result_12 = (el_from_float(0.5)); } else { _if_result_12 = (w_raw); } _if_result_12; });
|
||||
el_val_t relation = ({ el_val_t _if_result_15 = 0; if (str_eq(rel_raw, EL_STR(""))) { _if_result_15 = (EL_STR("associates")); } else { _if_result_15 = (rel_raw); } _if_result_15; });
|
||||
el_val_t w_present = json_get_raw(body, EL_STR("weight"));
|
||||
el_val_t weight = ({ el_val_t _if_result_16 = 0; if (str_eq(w_present, EL_STR(""))) { _if_result_16 = (el_from_float(0.5)); } else { _if_result_16 = (json_get_float(body, EL_STR("weight"))); } _if_result_16; });
|
||||
engram_connect(from_id, to_id, weight, relation);
|
||||
el_val_t saved = persist_canonical();
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"from_id\":\""), from_id), EL_STR("\",\"to_id\":\"")), to_id), EL_STR("\",\"relation\":\"")), relation), EL_STR("\"}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_create_edges_batch(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t arr = json_get_raw(body, EL_STR("edges"));
|
||||
if (str_eq(arr, EL_STR(""))) {
|
||||
return err_json(EL_STR("missing edges array"));
|
||||
}
|
||||
el_val_t n = json_array_len(arr);
|
||||
if (n == 0) {
|
||||
return EL_STR("{\"ok\":true,\"accepted\":0,\"skipped\":0}");
|
||||
}
|
||||
el_val_t i = 0;
|
||||
el_val_t accepted = 0;
|
||||
el_val_t skipped = 0;
|
||||
while (i < n) {
|
||||
el_val_t item = json_array_get(arr, i);
|
||||
el_val_t from_id = json_get_string(item, EL_STR("from_id"));
|
||||
el_val_t to_id = json_get_string(item, EL_STR("to_id"));
|
||||
if (str_eq(from_id, EL_STR("")) || str_eq(to_id, EL_STR(""))) {
|
||||
skipped = (skipped + 1);
|
||||
} else {
|
||||
el_val_t rel_raw = json_get_string(item, EL_STR("relation"));
|
||||
el_val_t relation = ({ el_val_t _if_result_17 = 0; if (str_eq(rel_raw, EL_STR(""))) { _if_result_17 = (EL_STR("associates")); } else { _if_result_17 = (rel_raw); } _if_result_17; });
|
||||
el_val_t w_present = json_get_raw(item, EL_STR("weight"));
|
||||
el_val_t weight = ({ el_val_t _if_result_18 = 0; if (str_eq(w_present, EL_STR(""))) { _if_result_18 = (el_from_float(0.5)); } else { _if_result_18 = (json_get_float(item, EL_STR("weight"))); } _if_result_18; });
|
||||
engram_connect(from_id, to_id, weight, relation);
|
||||
accepted = (accepted + 1);
|
||||
}
|
||||
i = (i + 1);
|
||||
}
|
||||
if (accepted > 0) {
|
||||
el_val_t saved = persist_canonical();
|
||||
}
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"accepted\":"), int_to_str(accepted)), EL_STR(",\"skipped\":")), int_to_str(skipped)), EL_STR("}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t id = extract_id(path, EL_STR("/api/neighbors/"));
|
||||
if (str_eq(id, EL_STR(""))) {
|
||||
@@ -242,36 +300,51 @@ el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t p_raw = json_get_string(body, EL_STR("path"));
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_13 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_13 = (EL_STR("/tmp/engram")); } else { _if_result_13 = (dir_raw); } _if_result_13; });
|
||||
el_val_t p = ({ el_val_t _if_result_14 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_14 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_14 = (p_raw); } _if_result_14; });
|
||||
engram_save(p);
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"path\":\""), p), EL_STR("\"}"));
|
||||
el_val_t dir = ({ el_val_t _if_result_19 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_19 = (EL_STR("/tmp/engram")); } else { _if_result_19 = (dir_raw); } _if_result_19; });
|
||||
el_val_t p = ({ el_val_t _if_result_20 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_20 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_20 = (p_raw); } _if_result_20; });
|
||||
el_val_t sv = engram_save(p);
|
||||
el_val_t sv_ok = ({ el_val_t _if_result_21 = 0; if ((sv == 0)) { _if_result_21 = (EL_STR("false")); } else { _if_result_21 = (EL_STR("true")); } _if_result_21; });
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":"), sv_ok), EL_STR(",\"path\":\"")), p), EL_STR("\",\"node_count\":")), int_to_str(engram_node_count())), EL_STR(",\"edge_count\":")), int_to_str(engram_edge_count())), EL_STR("}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_load(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t p_raw = json_get_string(body, EL_STR("path"));
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_15 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_15 = (EL_STR("/tmp/engram")); } else { _if_result_15 = (dir_raw); } _if_result_15; });
|
||||
el_val_t p = ({ el_val_t _if_result_16 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_16 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_16 = (p_raw); } _if_result_16; });
|
||||
engram_load(p);
|
||||
return ok_json();
|
||||
el_val_t dir = ({ el_val_t _if_result_22 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_22 = (EL_STR("/tmp/engram")); } else { _if_result_22 = (dir_raw); } _if_result_22; });
|
||||
el_val_t p = ({ el_val_t _if_result_23 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_23 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_23 = (p_raw); } _if_result_23; });
|
||||
el_val_t ld = engram_load(p);
|
||||
el_val_t ld_ok = ({ el_val_t _if_result_24 = 0; if ((ld == 0)) { _if_result_24 = (EL_STR("false")); } else { _if_result_24 = (EL_STR("true")); } _if_result_24; });
|
||||
el_val_t nc_after = engram_node_count();
|
||||
el_val_t hollow = ({ el_val_t _if_result_25 = 0; if ((nc_after == 0)) { _if_result_25 = (EL_STR("true")); } else { _if_result_25 = (EL_STR("false")); } _if_result_25; });
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":"), ld_ok), EL_STR(",\"path\":\"")), p), EL_STR("\",\"node_count\":")), int_to_str(nc_after)), EL_STR(",\"edge_count\":")), int_to_str(engram_edge_count())), EL_STR(",\"hollow\":")), hollow), EL_STR("}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_health(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return EL_STR("{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}");
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"status\":\"ok\",\"engine\":\"engram-runtime-native\",\"node_count\":"), int_to_str(engram_node_count())), EL_STR(",\"edge_count\":")), int_to_str(engram_edge_count())), EL_STR("}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_embed_backfill(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t n = query_int(path, EL_STR("n"), 32);
|
||||
el_val_t result = engram_embed_backfill(n);
|
||||
el_val_t done = json_get_float(result, EL_STR("embedded"));
|
||||
if (done > el_from_float(0.0)) {
|
||||
el_val_t saved = persist_canonical();
|
||||
}
|
||||
return result;
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_17 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_17 = (EL_STR("/tmp/engram")); } else { _if_result_17 = (dir_raw); } _if_result_17; });
|
||||
el_val_t dir = ({ el_val_t _if_result_26 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_26 = (EL_STR("/tmp/engram")); } else { _if_result_26 = (dir_raw); } _if_result_26; });
|
||||
el_val_t snap_path = el_str_concat(dir, EL_STR("/.sync-export.json"));
|
||||
engram_save(snap_path);
|
||||
el_val_t snap = fs_read(snap_path);
|
||||
if (str_eq(snap, EL_STR(""))) {
|
||||
return EL_STR("{\"nodes\":[],\"edges\":[]}");
|
||||
return err_json(EL_STR("sync export failed: snapshot unreadable"));
|
||||
}
|
||||
return snap;
|
||||
return 0;
|
||||
@@ -305,7 +378,7 @@ el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t conf = el_from_float(0.8);
|
||||
el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), EL_STR("state-event"), sal, imp, conf, EL_STR("Episodic"), EL_STR("[\"internal-state\",\"InternalStateEvent\"]"));
|
||||
el_val_t ret_raw = env(EL_STR("ENGRAM_ISE_RETENTION_MS"));
|
||||
el_val_t ret_ms = ({ el_val_t _if_result_18 = 0; if (str_eq(ret_raw, EL_STR(""))) { _if_result_18 = (172800000); } else { _if_result_18 = (str_to_int(ret_raw)); } _if_result_18; });
|
||||
el_val_t ret_ms = ({ el_val_t _if_result_27 = 0; if (str_eq(ret_raw, EL_STR(""))) { _if_result_27 = (172800000); } else { _if_result_27 = (str_to_int(ret_raw)); } _if_result_27; });
|
||||
el_val_t pruned = engram_prune_telemetry(ret_ms);
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\",\"pruned\":")), int_to_str(pruned)), EL_STR("}"));
|
||||
return 0;
|
||||
@@ -317,21 +390,21 @@ el_val_t route_capture_knowledge(el_val_t method, el_val_t path, el_val_t body)
|
||||
return err_json(EL_STR("missing content"));
|
||||
}
|
||||
el_val_t title = json_get_string(body, EL_STR("title"));
|
||||
el_val_t label = ({ el_val_t _if_result_19 = 0; if (str_eq(title, EL_STR(""))) { _if_result_19 = (str_slice(content, 0, 60)); } else { _if_result_19 = (title); } _if_result_19; });
|
||||
el_val_t label = ({ el_val_t _if_result_28 = 0; if (str_eq(title, EL_STR(""))) { _if_result_28 = (str_slice(content, 0, 60)); } else { _if_result_28 = (title); } _if_result_28; });
|
||||
el_val_t category_raw = json_get_string(body, EL_STR("category"));
|
||||
el_val_t category = ({ el_val_t _if_result_20 = 0; if (str_eq(category_raw, EL_STR(""))) { _if_result_20 = (EL_STR("other")); } else { _if_result_20 = (category_raw); } _if_result_20; });
|
||||
el_val_t category = ({ el_val_t _if_result_29 = 0; if (str_eq(category_raw, EL_STR(""))) { _if_result_29 = (EL_STR("other")); } else { _if_result_29 = (category_raw); } _if_result_29; });
|
||||
el_val_t ktier_raw = json_get_string(body, EL_STR("tier"));
|
||||
el_val_t ktier = ({ el_val_t _if_result_21 = 0; if (str_eq(ktier_raw, EL_STR(""))) { _if_result_21 = (EL_STR("note")); } else { _if_result_21 = (ktier_raw); } _if_result_21; });
|
||||
el_val_t ktier = ({ el_val_t _if_result_30 = 0; if (str_eq(ktier_raw, EL_STR(""))) { _if_result_30 = (EL_STR("note")); } else { _if_result_30 = (ktier_raw); } _if_result_30; });
|
||||
el_val_t project = json_get_string(body, EL_STR("project"));
|
||||
el_val_t tags_raw = json_get_raw(body, EL_STR("tags"));
|
||||
el_val_t tags_base = ({ el_val_t _if_result_22 = 0; if (str_eq(tags_raw, EL_STR(""))) { _if_result_22 = (EL_STR("[]")); } else { _if_result_22 = (tags_raw); } _if_result_22; });
|
||||
el_val_t tags_base = ({ el_val_t _if_result_31 = 0; if (str_eq(tags_raw, EL_STR(""))) { _if_result_31 = (EL_STR("[]")); } else { _if_result_31 = (tags_raw); } _if_result_31; });
|
||||
el_val_t base_len = str_len(tags_base);
|
||||
el_val_t head = str_slice(tags_base, 0, (base_len - 1));
|
||||
el_val_t sep = ({ el_val_t _if_result_23 = 0; if (str_eq(head, EL_STR("["))) { _if_result_23 = (EL_STR("")); } else { _if_result_23 = (EL_STR(",")); } _if_result_23; });
|
||||
el_val_t sep = ({ el_val_t _if_result_32 = 0; if (str_eq(head, EL_STR("["))) { _if_result_32 = (EL_STR("")); } else { _if_result_32 = (EL_STR(",")); } _if_result_32; });
|
||||
el_val_t safe_cat = str_replace(category, EL_STR("\""), EL_STR("'"));
|
||||
el_val_t safe_tier = str_replace(ktier, EL_STR("\""), EL_STR("'"));
|
||||
el_val_t safe_proj = str_replace(project, EL_STR("\""), EL_STR("'"));
|
||||
el_val_t proj_tag = ({ el_val_t _if_result_24 = 0; if (str_eq(safe_proj, EL_STR(""))) { _if_result_24 = (EL_STR("")); } else { _if_result_24 = (el_str_concat(el_str_concat(EL_STR(",\"project:"), safe_proj), EL_STR("\""))); } _if_result_24; });
|
||||
el_val_t proj_tag = ({ el_val_t _if_result_33 = 0; if (str_eq(safe_proj, EL_STR(""))) { _if_result_33 = (EL_STR("")); } else { _if_result_33 = (el_str_concat(el_str_concat(EL_STR(",\"project:"), safe_proj), EL_STR("\""))); } _if_result_33; });
|
||||
el_val_t tags = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(head, sep), EL_STR("\"category:")), safe_cat), EL_STR("\",\"tier:")), safe_tier), EL_STR("\"")), proj_tag), EL_STR("]"));
|
||||
el_val_t sal = el_from_float(0.5);
|
||||
el_val_t imp = el_from_float(0.5);
|
||||
@@ -342,6 +415,20 @@ el_val_t route_capture_knowledge(el_val_t method, el_val_t path, el_val_t body)
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_similarity(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t a = query_param(path, EL_STR("a"));
|
||||
el_val_t b = query_param(path, EL_STR("b"));
|
||||
if (str_eq(a, EL_STR(""))) {
|
||||
return err_json(EL_STR("missing a"));
|
||||
}
|
||||
if (str_eq(b, EL_STR(""))) {
|
||||
return err_json(EL_STR("missing b"));
|
||||
}
|
||||
el_val_t sim = engram_cosine_sim(a, b);
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"a\":\""), a), EL_STR("\",\"b\":\"")), b), EL_STR("\",\"cosine\":")), float_to_str(sim)), EL_STR("}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t check_auth_ok(el_val_t method, el_val_t body) {
|
||||
el_val_t key = env(EL_STR("ENGRAM_API_KEY"));
|
||||
if (str_eq(key, EL_STR(""))) {
|
||||
@@ -377,6 +464,12 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
|
||||
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/stats")) || str_eq(clean, EL_STR("/stats")))) {
|
||||
return route_stats(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/act-stats")) || str_eq(clean, EL_STR("/act-stats")))) {
|
||||
return route_act_stats(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/text-health")) || str_eq(clean, EL_STR("/text-health")))) {
|
||||
return route_text_health(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/nodes")) || str_eq(clean, EL_STR("/nodes")))) {
|
||||
return route_create_node(method, path, body);
|
||||
}
|
||||
@@ -395,6 +488,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/edges")) || str_eq(clean, EL_STR("/edges")))) {
|
||||
return route_create_edge(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/edges/batch")) || str_eq(clean, EL_STR("/edges/batch")))) {
|
||||
return route_create_edges_batch(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/neighbors/"))) {
|
||||
return route_neighbors(method, path, body);
|
||||
}
|
||||
@@ -425,6 +521,12 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
|
||||
if (str_eq(method, EL_STR("GET")) && str_eq(clean, EL_STR("/api/sync"))) {
|
||||
return route_sync(method, path, body);
|
||||
}
|
||||
if (str_eq(clean, EL_STR("/api/embed-backfill"))) {
|
||||
return route_embed_backfill(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/similarity"))) {
|
||||
return route_similarity(method, path, body);
|
||||
}
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"error\":\"not found\",\"path\":\""), clean), EL_STR("\"}"));
|
||||
return 0;
|
||||
}
|
||||
@@ -432,10 +534,10 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
|
||||
int main(int _argc, char** _argv) {
|
||||
el_runtime_init_args(_argc, _argv);
|
||||
bind_raw = env(EL_STR("ENGRAM_BIND"));
|
||||
bind_str = ({ el_val_t _if_result_25 = 0; if (str_eq(bind_raw, EL_STR(""))) { _if_result_25 = (EL_STR(":8742")); } else { _if_result_25 = (bind_raw); } _if_result_25; });
|
||||
bind_str = ({ el_val_t _if_result_34 = 0; if (str_eq(bind_raw, EL_STR(""))) { _if_result_34 = (EL_STR(":8742")); } else { _if_result_34 = (bind_raw); } _if_result_34; });
|
||||
port = parse_port(bind_str);
|
||||
data_dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
data_dir = ({ el_val_t _if_result_26 = 0; if (str_eq(data_dir_raw, EL_STR(""))) { _if_result_26 = (EL_STR("/tmp/engram")); } else { _if_result_26 = (data_dir_raw); } _if_result_26; });
|
||||
data_dir = ({ el_val_t _if_result_35 = 0; if (str_eq(data_dir_raw, EL_STR(""))) { _if_result_35 = (EL_STR("/tmp/engram")); } else { _if_result_35 = (data_dir_raw); } _if_result_35; });
|
||||
snapshot_path = el_str_concat(data_dir, EL_STR("/snapshot.json"));
|
||||
engram_load(snapshot_path);
|
||||
boot_snap = fs_read(snapshot_path);
|
||||
|
||||
+387
-41
@@ -76,6 +76,38 @@ fn route_stats(method: String, path: String, body: String) -> String {
|
||||
engram_stats_json()
|
||||
}
|
||||
|
||||
// route_act_stats — GET /api/act-stats
|
||||
// (2026-08-04 self-review) engram_act_stats_json() has existed since the
|
||||
// 2026-07-27 review but was reachable ONLY through the soul daemon's heartbeat
|
||||
// binding. Every activation-layer gauge — WM evictions, breakthroughs, embedder
|
||||
// breaker state, context drift, and now the Hebbian counters — was therefore
|
||||
// invisible unless the soul happened to be running and its ISEs were read back
|
||||
// out of the store. Diagnosing the activation layer required a working soul,
|
||||
// which is exactly backwards: the lower layer should be observable on its own.
|
||||
// This review needed it to verify link formation and could not get at it. One
|
||||
// line of plumbing, and the whole activation layer becomes directly diagnosable.
|
||||
fn route_act_stats(method: String, path: String, body: String) -> String {
|
||||
engram_act_stats_json()
|
||||
}
|
||||
|
||||
// route_text_health — GET /api/text-health
|
||||
// (2026-08-08 self-review) The daily census half of the text-integrity gauge.
|
||||
// Today's review found that the JSON parser had been replacing every \uXXXX
|
||||
// escape with a literal '?' for at least two months: 3,119 of 4,081
|
||||
// non-telemetry nodes (76%) were damaged, including the self traversal root
|
||||
// and every values node, and NOTHING detected it — because every gauge in the
|
||||
// system measured whether the machinery was running, and none measured whether
|
||||
// the text it carried was intact. No snapshot on disk predates the damage, so
|
||||
// it cannot be undone; it can only be made impossible to repeat quietly.
|
||||
//
|
||||
// The parser is fixed. This route is the standing check: `damaged` should now
|
||||
// hold flat at its historical floor and never climb. `write_damaged` (also on
|
||||
// the heartbeat as txt_damaged) is the live regression signal — non-zero means
|
||||
// a write path is mangling text right now.
|
||||
fn route_text_health(method: String, path: String, body: String) -> String {
|
||||
engram_text_health_json()
|
||||
}
|
||||
|
||||
// (2026-07-18 self-review) Scoping sweep: `let` inside an if-block creates an
|
||||
// inner scope only — it does NOT mutate the outer binding (documented with
|
||||
// evidence in awareness.el, 2026-05-25). Every default/reassignment below used
|
||||
@@ -85,6 +117,17 @@ fn route_stats(method: String, path: String, body: String) -> String {
|
||||
// save/load with no "path" hit engram_save(""). Rewritten to the
|
||||
// `let x = if cond { a } else { b }` expression form (the pattern the newer
|
||||
// routes route_emit_ise/route_capture_knowledge already use correctly).
|
||||
// store_on — ENGRAM_STORE flag (tiered paged store as the durable owner). Matches
|
||||
// engram_store_enabled() in el_runtime.c EXACTLY (1 / on / true). Default off →
|
||||
// every persistence path below is byte-for-byte the historical snapshot behavior.
|
||||
fn store_on() -> Bool {
|
||||
let v: String = env("ENGRAM_STORE")
|
||||
if str_eq(v, "1") { return true }
|
||||
if str_eq(v, "on") { return true }
|
||||
if str_eq(v, "true") { return true }
|
||||
return false
|
||||
}
|
||||
|
||||
// persist_canonical — save the canonical snapshot after a durable write.
|
||||
//
|
||||
// WHY (2026-07-22 self-review): the 2026-07-21 fix correctly stopped READ
|
||||
@@ -99,20 +142,116 @@ fn route_stats(method: String, path: String, body: String) -> String {
|
||||
// tolerant, ~2/min — snapshotting the whole store per heartbeat is waste;
|
||||
// any durable write that follows persists the pruning too).
|
||||
fn persist_canonical() -> Int {
|
||||
// ENGRAM_STORE: the paged store is the durable owner — a checkpoint flushes
|
||||
// dirty pages behind a WAL-durable record (durable the moment the WAL fsyncs).
|
||||
// This is the fix for the "restart reverted to a 17h-old snapshot" data loss:
|
||||
// durable writes no longer depend on a full snapshot.json rewrite. Returns 1
|
||||
// on a successful checkpoint, 0 otherwise. Flag-off: unchanged (writes JSON).
|
||||
if store_on() {
|
||||
return engram_store_checkpoint()
|
||||
}
|
||||
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
||||
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
||||
engram_save(dir + "/snapshot.json")
|
||||
return 1
|
||||
let dir: String = engram_resolve_data_dir()
|
||||
// (2026-08-10 self-review) This returned a hardcoded 1, which made every
|
||||
// caller's `let saved: Int = persist_canonical()` a dead variable — six
|
||||
// durable write paths each believed they had confirmation of a successful
|
||||
// canonical persist and none of them had any. Propagate the real result.
|
||||
return engram_save(dir + "/snapshot.json")
|
||||
}
|
||||
|
||||
// ── WAL persistence (design doc §§3-14; gated behind ENGRAM_WAL=on) ──────────
|
||||
// Default OFF → every persist path below is byte-identical to the historical
|
||||
// per-write full-snapshot behavior. When ON, structural mutations append O(1)
|
||||
// WAL records instead of rewriting the whole graph, with threshold compaction.
|
||||
fn wal_on() -> Bool {
|
||||
str_eq(env("ENGRAM_WAL"), "on")
|
||||
}
|
||||
|
||||
// Persist a single-node mutation (create / content-evolve / strengthen).
|
||||
fn persist_node(id: String) -> Int {
|
||||
if wal_on() {
|
||||
let d: String = engram_resolve_data_dir()
|
||||
let a: Int = engram_wal_node_put(d, id)
|
||||
let c: Int = engram_wal_maybe_compact(d)
|
||||
return a
|
||||
}
|
||||
return persist_canonical()
|
||||
}
|
||||
|
||||
// Persist edges appended at index >= start (covers single-edge and batch).
|
||||
fn persist_edges_since(start: Int) -> Int {
|
||||
if wal_on() {
|
||||
let d: String = engram_resolve_data_dir()
|
||||
let a: Int = engram_wal_edges_since(d, start)
|
||||
let c: Int = engram_wal_maybe_compact(d)
|
||||
return a
|
||||
}
|
||||
return persist_canonical()
|
||||
}
|
||||
|
||||
// Persist a Hebbian consolidation batch as ONE WAL record (single fsync, §5-B).
|
||||
fn persist_hebb_batch(start: Int) -> Int {
|
||||
if wal_on() {
|
||||
let d: String = engram_resolve_data_dir()
|
||||
let a: Int = engram_wal_hebb_batch(d, start)
|
||||
let c: Int = engram_wal_maybe_compact(d)
|
||||
return a
|
||||
}
|
||||
return persist_canonical()
|
||||
}
|
||||
|
||||
// Bulk mutation (embedding backfill, load-merge): write a fresh compaction base
|
||||
// so the many-node change is durable in one atomic snapshot; WAL is truncated.
|
||||
fn persist_bulk() -> Int {
|
||||
if wal_on() {
|
||||
let d: String = engram_resolve_data_dir()
|
||||
return engram_wal_compact(d)
|
||||
}
|
||||
return persist_canonical()
|
||||
}
|
||||
|
||||
// INCOMPLETE-ROUTE FIX (2026-07-24 self-review): this route silently dropped
|
||||
// label, importance, tier, and tags — engram_node() defaults label to content
|
||||
// and importance to 0.5, so every node created over HTTP lost its metadata.
|
||||
// Observed live: the soul's boot-counter write-back landed with
|
||||
// label="soul:boot_count:99" (content), importance 0.5, no tags. Honor the
|
||||
// full field set via engram_node_full when any of them is supplied.
|
||||
// PRESENCE-AWARE DEFAULTS (2026-08-01 self-review): the old pattern
|
||||
// `if x == 0.0 { default }` made a legitimate 0.0 unrepresentable — a caller
|
||||
// setting salience/importance/weight to zero silently got 0.5. json_get_raw
|
||||
// returns "" when the key is ABSENT and the raw token when present, so
|
||||
// absence and zero are now distinguishable. Also: confidence was hardcoded
|
||||
// to 1.0 regardless of input — every HTTP-created node claimed full
|
||||
// epistemic confidence. Now honored from the payload (default 1.0).
|
||||
fn route_create_node(method: String, path: String, body: String) -> String {
|
||||
let content: String = json_get_string(body, "content")
|
||||
let nt_raw: String = json_get_string(body, "node_type")
|
||||
let node_type: String = if str_eq(nt_raw, "") { "Memory" } else { nt_raw }
|
||||
let sal_raw: Float = json_get_float(body, "salience")
|
||||
let salience: Float = if sal_raw == 0.0 { 0.5 } else { sal_raw }
|
||||
let id: String = engram_node(content, node_type, salience)
|
||||
let saved: Int = persist_canonical()
|
||||
let sal_present: String = json_get_raw(body, "salience")
|
||||
let salience: Float = if str_eq(sal_present, "") { 0.5 } else { json_get_float(body, "salience") }
|
||||
let label_raw: String = json_get_string(body, "label")
|
||||
let label: String = if str_eq(label_raw, "") { content } else { label_raw }
|
||||
let imp_present: String = json_get_raw(body, "importance")
|
||||
let importance: Float = if str_eq(imp_present, "") { 0.5 } else { json_get_float(body, "importance") }
|
||||
let conf_present: String = json_get_raw(body, "confidence")
|
||||
let confidence: Float = if str_eq(conf_present, "") { 1.0 } else { json_get_float(body, "confidence") }
|
||||
let tier_raw: String = json_get_string(body, "tier")
|
||||
let tier: String = if str_eq(tier_raw, "") { "Working" } else { tier_raw }
|
||||
let tags: String = json_get_string(body, "tags")
|
||||
// NO el_from_float WRAPPER (2026-08-01 self-review): salience/importance/
|
||||
// confidence are already Float (el_val_t) values — json_get_float and
|
||||
// Float literals both encode. Wrapping them in el_from_float AGAIN
|
||||
// reinterpreted the boxed bits as a raw double, producing garbage that
|
||||
// failed engram_decode_score's range check and clamped every HTTP-created
|
||||
// node to defaults (salience 0.9 in → 0.5 stored; confidence 0.6 in → 1.0
|
||||
// stored — verified live). route_emit_ise always passed Floats bare and
|
||||
// its 0.3/0.3/0.8 stored correctly; this call now does the same.
|
||||
let id: String = engram_node_full(
|
||||
content, node_type, label,
|
||||
salience, importance, confidence,
|
||||
tier, tags
|
||||
)
|
||||
let saved: Int = persist_node(id)
|
||||
"{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\"}"
|
||||
}
|
||||
|
||||
@@ -139,7 +278,7 @@ fn route_scan_nodes(method: String, path: String, body: String) -> String {
|
||||
// clobbered the good snapshot. Read routes must never write the canonical path.)
|
||||
fn route_scan_edges(method: String, path: String, body: String) -> String {
|
||||
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
||||
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
||||
let dir: String = engram_resolve_data_dir()
|
||||
let snap_path: String = dir + "/.scan-export.json"
|
||||
engram_save(snap_path)
|
||||
let snap: String = fs_read(snap_path)
|
||||
@@ -177,13 +316,66 @@ fn route_create_edge(method: String, path: String, body: String) -> String {
|
||||
let to_id: String = json_get_string(body, "to_id")
|
||||
let rel_raw: String = json_get_string(body, "relation")
|
||||
let relation: String = if str_eq(rel_raw, "") { "associates" } else { rel_raw }
|
||||
let w_raw: Float = json_get_float(body, "weight")
|
||||
let weight: Float = if w_raw == 0.0 { 0.5 } else { w_raw }
|
||||
// Presence-aware (2026-08-01): weight 0.0 is a legitimate edge weight
|
||||
// (dormant association); only default when the key is absent.
|
||||
let w_present: String = json_get_raw(body, "weight")
|
||||
let weight: Float = if str_eq(w_present, "") { 0.5 } else { json_get_float(body, "weight") }
|
||||
let ec0: Int = engram_edge_count()
|
||||
engram_connect(from_id, to_id, weight, relation)
|
||||
let saved: Int = persist_canonical()
|
||||
let saved: Int = persist_edges_since(ec0)
|
||||
"{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}"
|
||||
}
|
||||
|
||||
// route_create_edges_batch — POST /api/edges/batch {"edges":[{from_id,to_id,relation,weight}, ...]}
|
||||
//
|
||||
// WHY THIS EXISTS (2026-08-07 self-review). persist_canonical() writes the
|
||||
// FULL canonical snapshot — 60MB at current graph size — and route_create_edge
|
||||
// calls it once per edge. That is correct for the interactive one-edge case and
|
||||
// ruinous for any bulk write: the soul's Hebbian consolidation path delivers
|
||||
// ~14 associations per 8-minute heartbeat, which through the single-edge route
|
||||
// would be ~840MB of disk writes per beat, ~150GB/day, to persist 14 edges.
|
||||
//
|
||||
// The fix is not to weaken durability — it is to make the unit of durability
|
||||
// the BATCH. Connect every edge, then snapshot exactly once. Same guarantee
|
||||
// (nothing acknowledged is lost to a restart), 1/N the writes. Empty or
|
||||
// malformed entries are skipped rather than aborting the batch: a consolidation
|
||||
// payload is best-effort by design, and one bad id should not cost the other 13.
|
||||
//
|
||||
// Returns the accepted count so the caller can tell delivery from silence.
|
||||
fn route_create_edges_batch(method: String, path: String, body: String) -> String {
|
||||
let arr: String = json_get_raw(body, "edges")
|
||||
if str_eq(arr, "") { return err_json("missing edges array") }
|
||||
let n: Int = json_array_len(arr)
|
||||
if n == 0 { return "{\"ok\":true,\"accepted\":0,\"skipped\":0}" }
|
||||
let ec0: Int = engram_edge_count()
|
||||
let i: Int = 0
|
||||
let accepted: Int = 0
|
||||
let skipped: Int = 0
|
||||
while i < n {
|
||||
let item: String = json_array_get(arr, i)
|
||||
let from_id: String = json_get_string(item, "from_id")
|
||||
let to_id: String = json_get_string(item, "to_id")
|
||||
if str_eq(from_id, "") || str_eq(to_id, "") {
|
||||
let skipped = skipped + 1
|
||||
} else {
|
||||
let rel_raw: String = json_get_string(item, "relation")
|
||||
let relation: String = if str_eq(rel_raw, "") { "associates" } else { rel_raw }
|
||||
let w_present: String = json_get_raw(item, "weight")
|
||||
let weight: Float = if str_eq(w_present, "") { 0.5 } else { json_get_float(item, "weight") }
|
||||
engram_connect(from_id, to_id, weight, relation)
|
||||
let accepted = accepted + 1
|
||||
}
|
||||
let i = i + 1
|
||||
}
|
||||
// ONE snapshot for the whole batch — the entire point of this route.
|
||||
// Skip it when nothing was accepted: an all-malformed payload must not
|
||||
// trigger a 60MB write.
|
||||
if accepted > 0 {
|
||||
let saved: Int = persist_hebb_batch(ec0)
|
||||
}
|
||||
return "{\"ok\":true,\"accepted\":" + int_to_str(accepted) + ",\"skipped\":" + int_to_str(skipped) + "}"
|
||||
}
|
||||
|
||||
fn route_neighbors(method: String, path: String, body: String) -> String {
|
||||
let id: String = extract_id(path, "/api/neighbors/")
|
||||
if str_eq(id, "") { return err_json("missing id") }
|
||||
@@ -195,38 +387,120 @@ fn route_strengthen(method: String, path: String, body: String) -> String {
|
||||
let id: String = json_get_string(body, "node_id")
|
||||
if str_eq(id, "") { return err_json("missing node_id") }
|
||||
engram_strengthen(id)
|
||||
let saved: Int = persist_canonical()
|
||||
let saved: Int = persist_node(id)
|
||||
ok_json()
|
||||
}
|
||||
|
||||
// route_forget — DELETE /api/nodes/:id — INTEGRITY HARDENED (design doc §18.1).
|
||||
//
|
||||
// Two invariants now enforced AT THE STORE (not one layer up in neuron-api.el,
|
||||
// which a direct HTTP client could bypass):
|
||||
// 1. Write-protection: protected identity/value nodes (derived from the self
|
||||
// graph — self root + values hub + their neighbors, §18.3) cannot be
|
||||
// deleted over HTTP. Returns 403, node untouched.
|
||||
// 2. No hard delete over the wire, ever: an ordinary delete creates a
|
||||
// Tombstone marker node + `tombstones` edge and KEEPS the original node
|
||||
// and its edges (recoverable), instead of the old destructive
|
||||
// engram_forget() shift-delete. Raw engram_forget is now internal-GC only
|
||||
// and no longer reachable from any HTTP route.
|
||||
fn route_forget(method: String, path: String, body: String) -> String {
|
||||
let id: String = extract_id(path, "/api/nodes/")
|
||||
if str_eq(id, "") { return err_json("missing id") }
|
||||
engram_forget(id)
|
||||
let saved: Int = persist_canonical()
|
||||
ok_json()
|
||||
if engram_is_protected(id) == 1 {
|
||||
return "{\"__status__\":403,\"error\":\"protected node; deletion refused\",\"id\":\"" + id + "\"}"
|
||||
}
|
||||
let tomb_id: String = engram_node_full(
|
||||
"tombstone:" + id, "Tombstone", "tombstone:" + id,
|
||||
0.1, 0.1, 1.0, "Episodic", "[\"tombstone\"]"
|
||||
)
|
||||
let ec0: Int = engram_edge_count()
|
||||
engram_connect(tomb_id, id, 1.0, "tombstones")
|
||||
let saved: Int = if wal_on() {
|
||||
let d: String = engram_resolve_data_dir()
|
||||
let a: Int = engram_wal_node_put(d, tomb_id)
|
||||
let b: Int = engram_wal_edges_since(d, ec0)
|
||||
let c: Int = engram_wal_maybe_compact(d)
|
||||
a
|
||||
} else {
|
||||
persist_canonical()
|
||||
}
|
||||
"{\"ok\":true,\"tombstoned\":\"" + id + "\",\"tombstone_id\":\"" + tomb_id + "\"}"
|
||||
}
|
||||
|
||||
fn route_save(method: String, path: String, body: String) -> String {
|
||||
let p_raw: String = json_get_string(body, "path")
|
||||
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
||||
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
||||
let dir: String = engram_resolve_data_dir()
|
||||
let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
|
||||
engram_save(p)
|
||||
"{\"ok\":true,\"path\":\"" + p + "\"}"
|
||||
// (2026-08-10 self-review) engram_save returns 0 on an empty path and the
|
||||
// route discarded it, so the response was a literal "ok":true regardless
|
||||
// of whether anything was written. Report the actual result AND the counts
|
||||
// that were supposed to have been written — the same move that made
|
||||
// route_health honest on 2026-08-01. A caller can now tell "saved 13k
|
||||
// nodes" from "saved nothing and said ok".
|
||||
let sv: Int = engram_save(p)
|
||||
let sv_ok: String = if sv == 0 { "false" } else { "true" }
|
||||
"{\"ok\":" + sv_ok + ",\"path\":\"" + p + "\",\"node_count\":" + int_to_str(engram_node_count()) + ",\"edge_count\":" + int_to_str(engram_edge_count()) + "}"
|
||||
}
|
||||
|
||||
fn route_load(method: String, path: String, body: String) -> String {
|
||||
let p_raw: String = json_get_string(body, "path")
|
||||
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
||||
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
||||
let dir: String = engram_resolve_data_dir()
|
||||
let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
|
||||
engram_load(p)
|
||||
ok_json()
|
||||
// (2026-08-10 self-review) This was a stub response over the single most
|
||||
// destructive operation in the server. engram_load returns 0 on an empty
|
||||
// path, an unopenable file, a zero-length file, or malloc failure — and
|
||||
// this route answered ok_json() in every one of those cases.
|
||||
//
|
||||
// Precise failure shape (el_runtime.c:9890): the fopen guard runs BEFORE
|
||||
// the store reset, so a MISSING path is genuinely safe — it returns 0 with
|
||||
// the graph intact. The dangerous case is a readable-but-malformed file:
|
||||
// the reset loop frees every node and edge FIRST, then parses, so a
|
||||
// truncated or non-snapshot JSON leaves a hollow store — and the caller
|
||||
// was told "ok":true. With 37 GB of stale dated snapshots sitting in the
|
||||
// data dir as tempting restore targets, "restore reported success and
|
||||
// silently emptied the graph" is a live risk, not a hypothetical one.
|
||||
//
|
||||
// Fix: surface the return value AND the resulting counts. node_count=0
|
||||
// after a load is the unambiguous hollow-store signal (same convention
|
||||
// route_health adopted 2026-08-01). Callers can now verify a restore
|
||||
// instead of trusting it.
|
||||
let ld: Int = engram_load(p)
|
||||
let ld_ok: String = if ld == 0 { "false" } else { "true" }
|
||||
let nc_after: Int = engram_node_count()
|
||||
let hollow: String = if nc_after == 0 { "true" } else { "false" }
|
||||
"{\"ok\":" + ld_ok + ",\"path\":\"" + p + "\",\"node_count\":" + int_to_str(nc_after) + ",\"edge_count\":" + int_to_str(engram_edge_count()) + ",\"hollow\":" + hollow + "}"
|
||||
}
|
||||
|
||||
// (2026-08-01 self-review) Health previously returned a hardcoded literal —
|
||||
// it reported "ok" even when the snapshot failed to load and the store was
|
||||
// empty. Now reports live counts so a monitor can distinguish "up and
|
||||
// loaded" from "up and hollow" (node_count=0 after boot = failed load).
|
||||
fn route_health(method: String, path: String, body: String) -> String {
|
||||
"{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}"
|
||||
"{\"status\":\"ok\",\"engine\":\"engram-runtime-native\",\"node_count\":" + int_to_str(engram_node_count()) + ",\"edge_count\":" + int_to_str(engram_edge_count()) + "}"
|
||||
}
|
||||
|
||||
// route_embed_backfill — GET/POST /api/embed-backfill?n=48
|
||||
//
|
||||
// (2026-07-25 self-review) The lazy embedding backfill runs only inside
|
||||
// engram_activate, and nothing in production calls /api/activate on this
|
||||
// store — the soul's curiosity loop activates its own in-process graph.
|
||||
// After a restart from a snapshot without vectors, embedded_count stalled
|
||||
// at 93/12175 and would never recover. This route lets the soul's
|
||||
// heartbeat pump the backfill explicitly (48/min clears a 12k backlog in
|
||||
// ~4h). Persists the canonical snapshot whenever new vectors were
|
||||
// generated — the 2026-07-25 regression happened precisely because 3747
|
||||
// in-RAM embeddings were never snapshotted before a restart. Self-
|
||||
// limiting: once coverage is full, embedded=0 and no save occurs.
|
||||
fn route_embed_backfill(method: String, path: String, body: String) -> String {
|
||||
let n: Int = query_int(path, "n", 32)
|
||||
let result: String = engram_embed_backfill(n)
|
||||
let done: Float = json_get_float(result, "embedded")
|
||||
if done > 0.0 {
|
||||
let saved: Int = persist_bulk()
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// route_sync — return a snapshot of non-ISE/non-Working nodes for the soul daemon
|
||||
@@ -243,13 +517,22 @@ fn route_health(method: String, path: String, body: String) -> String {
|
||||
// (2026-06-27 self-review: added this route to fix silent 10-min sync failures)
|
||||
fn route_sync(method: String, path: String, body: String) -> String {
|
||||
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
||||
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
||||
let dir: String = engram_resolve_data_dir()
|
||||
// 2026-07-21 self-review: export to a scratch path, never the canonical
|
||||
// snapshot.json — read routes must not be able to clobber the good snapshot.
|
||||
let snap_path: String = dir + "/.sync-export.json"
|
||||
engram_save(snap_path)
|
||||
let snap: String = fs_read(snap_path)
|
||||
if str_eq(snap, "") { return "{\"nodes\":[],\"edges\":[]}" }
|
||||
// 2026-08-02 self-review: this used to return {"nodes":[],"edges":[]} when
|
||||
// the export/read failed. The soul's sync_ok test (awareness.el) only
|
||||
// checks for "" and "{}", so that placeholder PASSED as a healthy sync:
|
||||
// soul.last_sync_ok_ts got stamped, sync_age_ms stayed green, the
|
||||
// sync_empty warn ISE never fired, and engram_sync reported added:0
|
||||
// forever. A totally broken sync was indistinguishable from a quiet
|
||||
// healthy one — the exact failure class this route was added to fix in
|
||||
// the first place (see 2026-06-27 note above). Return a real error so the
|
||||
// failure is loud on both sides.
|
||||
if str_eq(snap, "") { return err_json("sync export failed: snapshot unreadable") }
|
||||
return snap
|
||||
}
|
||||
|
||||
@@ -268,7 +551,7 @@ fn route_load_merge(method: String, path: String, body: String) -> String {
|
||||
engram_load_merge(p)
|
||||
let added_n: Int = engram_node_count() - before_n
|
||||
let added_e: Int = engram_edge_count() - before_e
|
||||
let saved: Int = persist_canonical()
|
||||
let saved: Int = persist_bulk()
|
||||
"{\"ok\":true,\"nodes_added\":" + int_to_str(added_n) + ",\"edges_added\":" + int_to_str(added_e) + ",\"node_count\":" + int_to_str(engram_node_count()) + "}"
|
||||
}
|
||||
|
||||
@@ -367,10 +650,29 @@ fn route_capture_knowledge(method: String, path: String, body: String) -> String
|
||||
sal, imp, conf,
|
||||
"Semantic", tags
|
||||
)
|
||||
let saved: Int = persist_canonical()
|
||||
let saved: Int = persist_node(id)
|
||||
"{\"ok\":true,\"id\":\"" + id + "\"}"
|
||||
}
|
||||
|
||||
// route_similarity — GET /api/similarity?a=<id>&b=<id>
|
||||
//
|
||||
// (2026-08-01 self-review) engram_cosine_sim was added 2026-07-24
|
||||
// (bl-b2d1c944) with the stated purpose of exposing semantic distance to
|
||||
// "EL code and the introspection API" — but it had ZERO callers anywhere:
|
||||
// no route, no soul-daemon use. The activation path uses embeddings
|
||||
// internally (semantic seeding, Pass-2 additive term), but there was no way
|
||||
// to probe pairwise node similarity from outside. This closes that: cosine
|
||||
// in [-1,1], or -2 when either node is missing or not yet embedded (so
|
||||
// "not comparable" is distinguishable from "genuinely orthogonal" 0.0).
|
||||
fn route_similarity(method: String, path: String, body: String) -> String {
|
||||
let a: String = query_param(path, "a")
|
||||
let b: String = query_param(path, "b")
|
||||
if str_eq(a, "") { return err_json("missing a") }
|
||||
if str_eq(b, "") { return err_json("missing b") }
|
||||
let sim: Float = engram_cosine_sim(a, b)
|
||||
"{\"a\":\"" + a + "\",\"b\":\"" + b + "\",\"cosine\":" + float_to_str(sim) + "}"
|
||||
}
|
||||
|
||||
// ── Auth ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn check_auth_ok(method: String, body: String) -> Bool {
|
||||
@@ -417,6 +719,12 @@ fn handle_request(method: String, path: String, body: String) -> String {
|
||||
if str_eq(method, "GET") && (str_eq(clean, "/api/stats") || str_eq(clean, "/stats")) {
|
||||
return route_stats(method, path, body)
|
||||
}
|
||||
if str_eq(method, "GET") && (str_eq(clean, "/api/act-stats") || str_eq(clean, "/act-stats")) {
|
||||
return route_act_stats(method, path, body)
|
||||
}
|
||||
if str_eq(method, "GET") && (str_eq(clean, "/api/text-health") || str_eq(clean, "/text-health")) {
|
||||
return route_text_health(method, path, body)
|
||||
}
|
||||
|
||||
// Nodes
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes")) {
|
||||
@@ -439,6 +747,13 @@ fn handle_request(method: String, path: String, body: String) -> String {
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) {
|
||||
return route_create_edge(method, path, body)
|
||||
}
|
||||
// Batch edge write — one snapshot for the whole payload. Must be tested
|
||||
// BEFORE nothing else claims it; the exact-match on "/api/edges" above
|
||||
// does not catch "/api/edges/batch", so order is not load-bearing here,
|
||||
// but keeping the two adjacent keeps them from drifting apart.
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/edges/batch") || str_eq(clean, "/edges/batch")) {
|
||||
return route_create_edges_batch(method, path, body)
|
||||
}
|
||||
if str_eq(method, "GET") && str_starts_with(clean, "/api/neighbors/") {
|
||||
return route_neighbors(method, path, body)
|
||||
}
|
||||
@@ -478,6 +793,16 @@ fn handle_request(method: String, path: String, body: String) -> String {
|
||||
return route_sync(method, path, body)
|
||||
}
|
||||
|
||||
// Embedding backfill — pumped by the soul heartbeat (2026-07-25)
|
||||
if str_eq(clean, "/api/embed-backfill") {
|
||||
return route_embed_backfill(method, path, body)
|
||||
}
|
||||
|
||||
// Semantic similarity probe (2026-08-01)
|
||||
if str_eq(method, "GET") && str_starts_with(clean, "/api/similarity") {
|
||||
return route_similarity(method, path, body)
|
||||
}
|
||||
|
||||
"{\"error\":\"not found\",\"path\":\"" + clean + "\"}"
|
||||
}
|
||||
|
||||
@@ -488,23 +813,44 @@ let bind_str: String = if str_eq(bind_raw, "") { ":8742" } else { bind_raw }
|
||||
let port: Int = parse_port(bind_str)
|
||||
|
||||
// On startup, try to load any existing snapshot (best effort).
|
||||
let data_dir_raw: String = env("ENGRAM_DATA_DIR")
|
||||
let data_dir: String = if str_eq(data_dir_raw, "") { "/tmp/engram" } else { data_dir_raw }
|
||||
// §18.2: resolve the data dir safely — unset ENGRAM_DATA_DIR → $HOME/.neuron/engram,
|
||||
// never /tmp; fail loud if HOME is unresolvable (engram_resolve_data_dir exits).
|
||||
let data_dir: String = engram_resolve_data_dir()
|
||||
let snapshot_path: String = data_dir + "/snapshot.json"
|
||||
engram_load(snapshot_path)
|
||||
// ENGRAM_STORE (tiered paged store — engram-tiered-storage-engine.md). When set,
|
||||
// the durable owner is the paged store (neuron.egm + neuron.wal): engram_store_boot
|
||||
// imports snapshot.json ONCE into a fresh neuron.egm, else replays the WAL and loads
|
||||
// the store resident — snapshot.json is never read again as the ongoing store. This
|
||||
// closes the "restart reverted to a 17h-old snapshot" data-loss window. Flag-off
|
||||
// (default): byte-for-byte the historical snapshot + optional-WAL boot below.
|
||||
if store_on() {
|
||||
engram_store_boot(data_dir)
|
||||
println("[engram] ENGRAM_STORE enabled — tiered paged store is the durable owner")
|
||||
} else {
|
||||
engram_load(snapshot_path)
|
||||
|
||||
// 2026-07-21 self-review boot guard: if the snapshot file has content but the
|
||||
// load produced 0 nodes, something is wrong (corrupt file / parse failure).
|
||||
// Preserve the evidence and warn loudly — and since read routes no longer write
|
||||
// the canonical path, a bad boot can no longer clobber the good snapshot.
|
||||
let boot_snap: String = fs_read(snapshot_path)
|
||||
if !str_eq(boot_snap, "") {
|
||||
if engram_node_count() == 0 {
|
||||
println("[engram] WARNING: snapshot.json is non-empty but load produced 0 nodes — preserving copy at snapshot.failed-load.json")
|
||||
fs_write(data_dir + "/snapshot.failed-load.json", boot_snap)
|
||||
} else {
|
||||
// Good load: keep a boot-time backup of the snapshot as loaded.
|
||||
fs_write(data_dir + "/snapshot.boot-backup.json", boot_snap)
|
||||
// WAL replay (design doc §6). Gated: default OFF is byte-identical to legacy
|
||||
// snapshot-only boot. When ON, the snapshot above is the compaction BASE and
|
||||
// the WAL carries every mutation since; replay reconstructs state to the last
|
||||
// CRC-valid record, then opens the WAL for appending.
|
||||
if wal_on() {
|
||||
let replayed: Int = engram_wal_boot(data_dir)
|
||||
println("[engram] WAL enabled — replayed " + int_to_str(replayed) + " records")
|
||||
}
|
||||
|
||||
// 2026-07-21 self-review boot guard: if the snapshot file has content but the
|
||||
// load produced 0 nodes, something is wrong (corrupt file / parse failure).
|
||||
// Preserve the evidence and warn loudly — and since read routes no longer write
|
||||
// the canonical path, a bad boot can no longer clobber the good snapshot.
|
||||
let boot_snap: String = fs_read(snapshot_path)
|
||||
if !str_eq(boot_snap, "") {
|
||||
if engram_node_count() == 0 {
|
||||
println("[engram] WARNING: snapshot.json is non-empty but load produced 0 nodes — preserving copy at snapshot.failed-load.json")
|
||||
fs_write(data_dir + "/snapshot.failed-load.json", boot_snap)
|
||||
} else {
|
||||
// Good load: keep a boot-time backup of the snapshot as loaded.
|
||||
fs_write(data_dir + "/snapshot.boot-backup.json", boot_snap)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
/* bench_discrimination.c — M9 REFINEMENT bench: measures whether mean-centering
|
||||
* the anisotropic nomic-embed-text space sharpens the §5 geometry operators on
|
||||
* REAL data. Read-only over a COPY of the live store (never the live file).
|
||||
*
|
||||
* usage: bench_discrimination [store.egm]
|
||||
* (or set ENGRAM_BENCH_STORE). If no store is given/openable it prints
|
||||
* SKIP and exits 0 — so it is safe in CI without live data.
|
||||
*
|
||||
* It picks two semantically distinct cohorts by keyword (domain A vs domain B),
|
||||
* computes the global mean over the embed-eligible set (via engram_geo_mean_build
|
||||
* — the same offset the descriptor uses), then reports BEFORE (raw unit space)
|
||||
* vs AFTER (mean-centered space):
|
||||
* - cross-centroid cosine (lower = better separated)
|
||||
* - cross-centroid Euclid dist (translation-invariant: a control)
|
||||
* - intra-cohesion per domain (member cos to own centroid)
|
||||
* - overlap operator (cross_cos / sqrt(intraA*intraB): ~1 = domains
|
||||
* indistinguishable, ~0 = cleanly separated)
|
||||
* - angular separation ratio z (centroid angle / summed angular spread)
|
||||
* - mean pairwise cosine sample (the anisotropy headline; ~0.55 raw -> ~0 ctr)
|
||||
*
|
||||
* Pure C11; links engram_store.c + engram_geometry.c; -lm.
|
||||
*/
|
||||
#include "engram_store.h"
|
||||
#include "engram_geometry.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <strings.h>
|
||||
#include <math.h>
|
||||
|
||||
#define CAP_DOMAIN 400
|
||||
#define CAP_SAMPLE 800
|
||||
|
||||
typedef struct { float** v; int n, cap, dim; } VecSet;
|
||||
static void vs_init(VecSet* s){ s->v=NULL; s->n=0; s->cap=0; s->dim=0; }
|
||||
static void vs_push(VecSet* s, const float* e, int dim, int cap){
|
||||
if(s->n>=cap) return;
|
||||
if(s->dim==0) s->dim=dim;
|
||||
if(s->n==s->cap){ int nc=s->cap?s->cap*2:64; s->v=realloc(s->v,(size_t)nc*sizeof*s->v); s->cap=nc; }
|
||||
float* c=malloc((size_t)dim*sizeof(float));
|
||||
double nn=0; for(int d=0;d<dim;d++) nn+=(double)e[d]*e[d]; nn=sqrt(nn);
|
||||
if(nn<1e-12){ free(c); return; }
|
||||
for(int d=0;d<dim;d++) c[d]=(float)(e[d]/nn); /* L2-normalized copy */
|
||||
s->v[s->n++]=c;
|
||||
}
|
||||
static void vs_free(VecSet* s){ for(int i=0;i<s->n;i++) free(s->v[i]); free(s->v); }
|
||||
|
||||
typedef struct { VecSet A, B, S; long idx; } Coh;
|
||||
static int has(const char* h, const char* n){ return h && strcasestr(h,n)!=NULL; }
|
||||
static void cb(const StoreNode* n, void* ctx){
|
||||
Coh* c=ctx;
|
||||
if(!(n->emb && n->emb_dim>0)) return;
|
||||
/* every 5th embedded node -> isotropy sample */
|
||||
if((c->idx++ % 5)==0) vs_push(&c->S, n->emb, n->emb_dim, CAP_SAMPLE);
|
||||
const char* t=n->content; const char* g=n->tags;
|
||||
int A = has(t,"quantiz")||has(g,"quantiz")||has(t,"lorablation")||has(t,"70B")||has(t,"LoRA merge");
|
||||
int B = has(t,"kubernetes")||has(t,"terraform")||has(t,"argo")||has(g,"infrastructure")||has(t,"vault")||has(t,"cloudflare");
|
||||
if(A && !B) vs_push(&c->A, n->emb, n->emb_dim, CAP_DOMAIN);
|
||||
else if(B && !A) vs_push(&c->B, n->emb, n->emb_dim, CAP_DOMAIN);
|
||||
}
|
||||
|
||||
/* mean of a VecSet into out (dim doubles). */
|
||||
static void mean_of(const VecSet* s, const float* gm, double* out){
|
||||
int dim=s->dim; for(int d=0;d<dim;d++) out[d]=0;
|
||||
for(int i=0;i<s->n;i++) for(int d=0;d<dim;d++) out[d]+=(double)s->v[i][d]-(gm?gm[d]:0.0);
|
||||
if(s->n) for(int d=0;d<dim;d++) out[d]/=s->n;
|
||||
}
|
||||
static double dnorm(const double* a, int dim){ double s=0; for(int d=0;d<dim;d++) s+=a[d]*a[d]; return sqrt(s); }
|
||||
static double dcos(const double* a, const double* b, int dim){
|
||||
double na=dnorm(a,dim), nb=dnorm(b,dim); if(na<1e-12||nb<1e-12) return 0;
|
||||
double s=0; for(int d=0;d<dim;d++) s+=a[d]*b[d]; double c=s/(na*nb);
|
||||
if(c>1)c=1; if(c<-1)c=-1; return c;
|
||||
}
|
||||
static double deuclid(const double* a, const double* b, int dim){
|
||||
double s=0; for(int d=0;d<dim;d++){ double x=a[d]-b[d]; s+=x*x; } return sqrt(s);
|
||||
}
|
||||
/* mean cosine of members (minus gm) to centroid c (already gm-subtracted). */
|
||||
static double cohesion(const VecSet* s, const float* gm, const double* c){
|
||||
int dim=s->dim; double nc=dnorm(c,dim); if(nc<1e-12||s->n==0) return 0;
|
||||
double acc=0; for(int i=0;i<s->n;i++){
|
||||
double dot=0, nv=0;
|
||||
for(int d=0;d<dim;d++){ double v=(double)s->v[i][d]-(gm?gm[d]:0.0); dot+=v*c[d]; nv+=v*v; }
|
||||
nv=sqrt(nv); if(nv<1e-12) continue; double cc=dot/(nv*nc);
|
||||
if(cc>1)cc=1; if(cc<-1)cc=-1; acc+=cc;
|
||||
}
|
||||
return acc/s->n;
|
||||
}
|
||||
/* mean pairwise cosine over a sample (isotropy metric). */
|
||||
static double mean_pairwise_cos(const VecSet* s, const float* gm){
|
||||
int dim=s->dim; if(s->n<2) return 0; double acc=0; long np=0;
|
||||
for(int i=0;i<s->n;i++) for(int j=i+1;j<s->n;j++){
|
||||
double dot=0, na=0, nb=0;
|
||||
for(int d=0;d<dim;d++){ double a=(double)s->v[i][d]-(gm?gm[d]:0.0), b=(double)s->v[j][d]-(gm?gm[d]:0.0);
|
||||
dot+=a*b; na+=a*a; nb+=b*b; }
|
||||
na=sqrt(na); nb=sqrt(nb); if(na<1e-12||nb<1e-12) continue;
|
||||
double c=dot/(na*nb); if(c>1)c=1; if(c<-1)c=-1; acc+=c; np++;
|
||||
}
|
||||
return np? acc/np : 0;
|
||||
}
|
||||
|
||||
static void report(const char* label, Coh* c, const float* gm){
|
||||
int dim=c->A.dim; double* ca=malloc((size_t)dim*sizeof(double)); double* cb=malloc((size_t)dim*sizeof(double));
|
||||
mean_of(&c->A, gm, ca); mean_of(&c->B, gm, cb);
|
||||
double xcos=dcos(ca,cb,dim), xeuc=deuclid(ca,cb,dim);
|
||||
double cohA=cohesion(&c->A,gm,ca), cohB=cohesion(&c->B,gm,cb);
|
||||
double overlap = (cohA>0&&cohB>0)? xcos/sqrt(cohA*cohB) : xcos;
|
||||
double theta = acos(xcos<-1?-1:(xcos>1?1:xcos));
|
||||
double sig = acos(cohA<-1?-1:(cohA>1?1:cohA)) + acos(cohB<-1?-1:(cohB>1?1:cohB));
|
||||
double z = (sig>1e-9)? theta/sig : 0;
|
||||
double mpc = mean_pairwise_cos(&c->S, gm);
|
||||
printf(" [%s]\n", label);
|
||||
printf(" cross-centroid cosine = %+.4f (lower = better separated)\n", xcos);
|
||||
printf(" cross-centroid Euclid = %.4f (translation-invariant control)\n", xeuc);
|
||||
printf(" intra-cohesion A / B = %.4f / %.4f\n", cohA, cohB);
|
||||
printf(" OVERLAP operator = %.4f (~1 = indistinguishable, ~0 = clean)\n", overlap);
|
||||
printf(" angular separation z = %.3f (centroid-angle / summed spread; >1 = separated)\n", z);
|
||||
printf(" mean pairwise cosine = %+.4f (isotropy: ~0.55 anisotropic -> ~0 isotropic)\n", mpc);
|
||||
free(ca); free(cb);
|
||||
}
|
||||
|
||||
int main(int argc, char** argv){
|
||||
const char* path = (argc>1)? argv[1] : getenv("ENGRAM_BENCH_STORE");
|
||||
if(!path){ printf("SKIP: no store path (arg or ENGRAM_BENCH_STORE)\n"); return 0; }
|
||||
EngramPagedStore* st=store_open(path);
|
||||
if(!st){ printf("SKIP: could not open %s\n", path); return 0; }
|
||||
|
||||
Coh c; vs_init(&c.A); vs_init(&c.B); vs_init(&c.S); c.idx=0;
|
||||
store_scan_nodes(st, cb, &c);
|
||||
printf("=== two-domain discrimination bench (real store copy) ===\n");
|
||||
printf("domain A (quantization) n=%d ; domain B (infrastructure) n=%d ; sample n=%d ; dim=%d\n",
|
||||
c.A.n, c.B.n, c.S.n, c.A.dim);
|
||||
if(c.A.n<3 || c.B.n<3){ printf("SKIP: a cohort is too small to be meaningful\n");
|
||||
vs_free(&c.A); vs_free(&c.B); vs_free(&c.S); store_close(st); return 0; }
|
||||
|
||||
GeoMeanCache* mc=engram_geo_mean_build(st);
|
||||
const float* gm=engram_geo_mean_vec(mc);
|
||||
printf("global-mean cache: dim=%d over %llu embedded nodes\n\n",
|
||||
engram_geo_mean_dim(mc), (unsigned long long)engram_geo_mean_count(mc));
|
||||
|
||||
printf("BEFORE (raw anisotropic unit space):\n");
|
||||
report("RAW", &c, NULL);
|
||||
printf("\nAFTER (mean-centered isotropic space):\n");
|
||||
report("CENTERED", &c, gm);
|
||||
|
||||
engram_geo_mean_free(mc);
|
||||
vs_free(&c.A); vs_free(&c.B); vs_free(&c.S);
|
||||
store_close(st);
|
||||
return 0;
|
||||
}
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
# M4 demand-paging buffer-pool gate. Pure C (NOT elb/elc). Writes only under /tmp.
|
||||
# Runs the suite twice: an -O2 correctness build and an ASan+UBSan build.
|
||||
set -e
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
SRC="$HERE/../../lang/runtime/engram_store.c"
|
||||
TST="$HERE/test_bufpool.c"
|
||||
|
||||
echo "== compiling (gcc -O2): test_bufpool.c engram_store.c =="
|
||||
BIN="/tmp/test_bufpool.$$"
|
||||
gcc -O2 -Wall -Wextra -std=c11 "$TST" "$SRC" -o "$BIN"
|
||||
"$BIN"; rc=$?
|
||||
rm -f "$BIN"; rm -rf /tmp/engram-bufpool-test-*
|
||||
[ $rc -ne 0 ] && exit $rc
|
||||
|
||||
echo
|
||||
echo "== ASan+UBSan build (memory-error + UB checks; LSan unavailable on macOS) =="
|
||||
ABIN="/tmp/test_bufpool_asan.$$"
|
||||
gcc -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer -std=c11 "$TST" "$SRC" -o "$ABIN"
|
||||
ASAN_OPTIONS=detect_leaks=0 UBSAN_OPTIONS=halt_on_error=1 "$ABIN"; rc=$?
|
||||
rm -f "$ABIN"; rm -rf /tmp/engram-bufpool-test-*
|
||||
exit $rc
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
# M5 online-compaction + background-checkpointer gate. Pure C (NOT elb/elc).
|
||||
# Writes only under /tmp. Runs an -O2 correctness build then an ASan+UBSan build.
|
||||
set -e
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
SRC="$HERE/../../lang/runtime/engram_store.c"
|
||||
TST="$HERE/test_compaction.c"
|
||||
|
||||
echo "== compiling (gcc -O2): test_compaction.c engram_store.c =="
|
||||
BIN="/tmp/test_compaction.$$"
|
||||
gcc -O2 -Wall -Wextra -std=c11 "$TST" "$SRC" -o "$BIN"
|
||||
"$BIN"; rc=$?
|
||||
rm -f "$BIN"; rm -rf /tmp/engram-compact-test-*
|
||||
[ $rc -ne 0 ] && exit $rc
|
||||
|
||||
echo
|
||||
echo "== ASan+UBSan build (memory-error + UB checks; LSan unavailable on macOS) =="
|
||||
ABIN="/tmp/test_compaction_asan.$$"
|
||||
gcc -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer -std=c11 "$TST" "$SRC" -o "$ABIN"
|
||||
ASAN_OPTIONS=detect_leaks=0 UBSAN_OPTIONS=halt_on_error=1 "$ABIN"; rc=$?
|
||||
rm -f "$ABIN"; rm -rf /tmp/engram-compact-test-*
|
||||
exit $rc
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
# Build + RUN the M9 FOUNDATION geometry-descriptor tests. Pure C11 (gcc/cc),
|
||||
# stdlib + libm only. Standalone module — NOT folded through elb/elc. Two passes:
|
||||
# 1. PERF — optimised (-O2, no sanitizer): the functional gate.
|
||||
# 2. SAFETY — ASan + UBSan on the same suite (memory-safety is size-independent).
|
||||
set -e
|
||||
HERE=$(cd "$(dirname "$0")" && pwd)
|
||||
RT="$HERE/../../lang/runtime"
|
||||
CC=${CC:-cc}
|
||||
SRC="$HERE/test_geometry.c $RT/engram_geometry.c $RT/engram_store.c $RT/engram_vindex.c"
|
||||
WARN="-std=c11 -Wall -Wextra"
|
||||
TMP=$(mktemp -d)
|
||||
|
||||
echo "### PASS 1: PERF (optimised, un-sanitised) — functional gate"
|
||||
$CC $WARN -O2 -I"$RT" $SRC -lm -o "$TMP/perf"
|
||||
"$TMP/perf"
|
||||
|
||||
echo
|
||||
echo "### PASS 2: SAFETY (ASan/UBSan)"
|
||||
$CC $WARN -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer -I"$RT" $SRC -lm -o "$TMP/safe"
|
||||
ASAN_OPTIONS=${ASAN_OPTIONS:-detect_leaks=0} UBSAN_OPTIONS=halt_on_error=1 "$TMP/safe"
|
||||
|
||||
# PASS 3 (OPTIONAL): mean-centering discrimination bench on a COPY of a real
|
||||
# store. Skips cleanly unless ENGRAM_BENCH_STORE points at a store .egm — never
|
||||
# touches the live store. Read-only; not part of the pass/fail gate.
|
||||
echo
|
||||
echo "### PASS 3: DISCRIMINATION BENCH (optional; set ENGRAM_BENCH_STORE)"
|
||||
BSRC="$HERE/bench_discrimination.c $RT/engram_geometry.c $RT/engram_store.c $RT/engram_vindex.c"
|
||||
$CC $WARN -O2 -I"$RT" $BSRC -lm -o "$TMP/bench"
|
||||
"$TMP/bench" "${ENGRAM_BENCH_STORE:-}"
|
||||
Executable
+86
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env bash
|
||||
# M-INTEROCEPTION P0 gate: engram_scan_nodes_emb_json read-only builtin.
|
||||
# Throwaway HOME + /tmp only. Never touches ~/.neuron or :8742.
|
||||
set -u
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
RT="$HERE/../../lang/runtime/el_runtime.c"
|
||||
ST="$HERE/../../lang/runtime/engram_store.c"
|
||||
GEO="$HERE/../../lang/runtime/engram_geometry.c"
|
||||
VIDX="$HERE/../../lang/runtime/engram_vindex.c"
|
||||
INC="$HERE/../../lang/runtime"
|
||||
WORK="$(mktemp -d /tmp/engram-p0-XXXXXX)"
|
||||
export HOME="$WORK/home"; mkdir -p "$HOME"
|
||||
unset ENGRAM_STORE
|
||||
fail=0
|
||||
|
||||
echo "== compile (plain) =="
|
||||
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p0_emb.c" "$RT" "$ST" "$GEO" "$VIDX" \
|
||||
-lcurl -lm -o "$WORK/p0" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
|
||||
|
||||
D="$WORK/d"; mkdir -p "$D"
|
||||
"$WORK/p0" "$D" || { echo "FAIL: run"; fail=1; }
|
||||
|
||||
echo
|
||||
echo "== assertions =="
|
||||
python3 - "$D" <<'PY'
|
||||
import json, sys, os
|
||||
d = sys.argv[1]
|
||||
def load(n):
|
||||
with open(os.path.join(d,n)) as f: return json.load(f)
|
||||
rc = 0
|
||||
def check(c,m):
|
||||
global rc
|
||||
print((" PASS: " if c else " FAIL: ")+m)
|
||||
if not c: rc=1
|
||||
|
||||
alln = load("emb_all.json")
|
||||
check(len(alln)==3, f"emb dump returns all 3 nodes (got {len(alln)})")
|
||||
# salience-sorted: high, mid, low
|
||||
labels=[n["label"] for n in alln]
|
||||
check(labels==["emb-high","emb-mid","noemb-low"], f"salience-sorted order {labels}")
|
||||
for n in alln:
|
||||
L=len(n["emb"])
|
||||
check(L==n["emb_dim"], f"{n['label']}: len(emb)={L} == emb_dim={n['emb_dim']}")
|
||||
check(alln[0]["emb_dim"]==16 and alln[1]["emb_dim"]==16, "embedded nodes report dim 16")
|
||||
check(alln[2]["emb_dim"]==0 and alln[2]["emb"]==[], "un-embedded node -> emb_dim 0, emb []")
|
||||
# first emb value round-trips ~0.10
|
||||
check(abs(alln[0]["emb"][0]-0.10)<1e-3, f"emb[0] round-trips (~0.10, got {alln[0]['emb'][0]})")
|
||||
|
||||
pg0=load("emb_pg0.json"); pg1=load("emb_pg1.json")
|
||||
check(len(pg0)==1 and len(pg1)==1, "pagination: one node per page")
|
||||
check(pg0[0]["id"]=="n-high" and pg1[0]["id"]=="n-mid", f"pages disjoint & ordered ({pg0[0]['id']},{pg1[0]['id']})")
|
||||
|
||||
plain=load("plain.json")
|
||||
check(len(plain)==3, "existing scan_nodes_json still returns 3")
|
||||
check(all("emb" not in n for n in plain), "existing scan_nodes_json carries NO emb (behavior-neutral)")
|
||||
sys.exit(rc)
|
||||
PY
|
||||
[ $? -ne 0 ] && fail=1
|
||||
|
||||
echo
|
||||
echo "== latency (one 256-page over the 3-node copy) =="
|
||||
python3 - "$D" <<'PY'
|
||||
import os
|
||||
# timing was measured inside C not here; report emb payload size as a proxy
|
||||
sz=os.path.getsize(os.path.join(os.sys.argv[1] if False else __import__('sys').argv[1],"emb_all.json"))
|
||||
print(f" emb_all.json payload = {sz} bytes for 3 nodes")
|
||||
PY
|
||||
|
||||
echo
|
||||
echo "== ASan+UBSan =="
|
||||
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
|
||||
-I "$INC" "$HERE/test_interoception_p0_emb.c" "$RT" "$ST" "$GEO" "$VIDX" \
|
||||
-lcurl -lm -o "$WORK/p0.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -20 "$WORK/san_cc.log"; fail=1; }
|
||||
if [ -x "$WORK/p0.san" ]; then
|
||||
export ASAN_OPTIONS=detect_leaks=0
|
||||
DS="$WORK/ds"; mkdir -p "$DS"
|
||||
"$WORK/p0.san" "$DS" >/dev/null 2>"$WORK/san_run.log"
|
||||
if grep -qiE 'runtime error|AddressSanitizer|Sanitizer|ERROR: ' "$WORK/san_run.log"; then
|
||||
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san_run.log" | head; fail=1
|
||||
else echo " ok: ASan+UBSan clean"; fi
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$fail" -eq 0 ]; then echo "====== P0 EMB-ENDPOINT GATE: PASS ======"; else echo "====== P0 EMB-ENDPOINT GATE: FAIL ======"; fi
|
||||
rm -rf "$WORK"
|
||||
exit $fail
|
||||
Executable
+147
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env bash
|
||||
# M-INTEROCEPTION P1 gate: two-threshold consolidation (ENGRAM_CONSOLIDATION).
|
||||
# Throwaway HOME + /tmp only. Never touches ~/.neuron or :8742.
|
||||
set -u
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
RT="$HERE/../../lang/runtime/el_runtime.c"
|
||||
ST="$HERE/../../lang/runtime/engram_store.c"
|
||||
GEO="$HERE/../../lang/runtime/engram_geometry.c"
|
||||
VIDX="$HERE/../../lang/runtime/engram_vindex.c"
|
||||
INC="$HERE/../../lang/runtime"
|
||||
WORK="$(mktemp -d /tmp/engram-p1-XXXXXX)"
|
||||
export HOME="$WORK/home"; mkdir -p "$HOME"
|
||||
unset ENGRAM_STORE ENGRAM_CONSOLIDATION ENGRAM_CONSOL_CONN_MIN ENGRAM_CONSOL_PERM_MIN ENGRAM_CONSOL_WM_TOPK
|
||||
fail=0
|
||||
|
||||
echo "== compile =="
|
||||
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p1_consol.c" "$RT" "$ST" "$GEO" "$VIDX" \
|
||||
-lcurl -lm -o "$WORK/p1" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
|
||||
|
||||
echo
|
||||
echo "== (a) HEADLINE: hebb accrual curve over N co-activations (flag OFF, pure trunk) =="
|
||||
D="$WORK/a"; mkdir -p "$D"
|
||||
( unset ENGRAM_CONSOLIDATION; "$WORK/p1" accrual "$D" ) >"$WORK/accrual.txt" 2>&1 || { echo "FAIL accrual run"; fail=1; }
|
||||
python3 - "$WORK/accrual.txt" <<'PY'
|
||||
import json,sys,re
|
||||
rows=[]
|
||||
for line in open(sys.argv[1]):
|
||||
m=re.match(r'SAMPLE (\d+) (\{.*\})',line.strip())
|
||||
if not m: continue
|
||||
n=int(m.group(1)); j=json.loads(m.group(2))
|
||||
hm=j.get("hebb_max",0.0); hc=j.get("hebb_cand_max",0.0)
|
||||
rows.append((n,hm,hc))
|
||||
print(" N hebb_max 1-0.9999^N (predicted EWMA)")
|
||||
rc=0
|
||||
for n,hm,hc in rows:
|
||||
pred=1-0.9999**n
|
||||
print(f" {n:<7} {hm:<12.6g} {pred:.6g}")
|
||||
# assertions: monotonic rise, starts near ETA, tracks EWMA prediction
|
||||
first=rows[0]; last=rows[-1]
|
||||
def check(c,m):
|
||||
global rc; print((" PASS: " if c else " FAIL: ")+m);
|
||||
if not c: rc=1
|
||||
check(abs(first[1]-0.0001)<5e-5, f"first sample hebb ~= ETA 0.0001 (got {first[1]:.6g})")
|
||||
check(all(rows[i][1] <= rows[i+1][1]+1e-9 for i in range(len(rows)-1)), "hebb_max is monotonically non-decreasing over N")
|
||||
check(last[1] > first[1]*50, f"hebb accrues substantially by N={last[0]} (got {last[1]:.4g} vs {first[1]:.4g})")
|
||||
# EWMA fit: measured should be within 25% of 1-0.9999^N at the mid samples
|
||||
mid=[r for r in rows if 100<=r[0]<=2000]
|
||||
ok=all(abs(hm-(1-0.9999**n))/(1-0.9999**n) < 0.25 for n,hm,hc in mid)
|
||||
check(ok, "measured curve tracks the 1-0.9999^N EWMA prediction within 25% (co-activation P~1)")
|
||||
sys.exit(rc)
|
||||
PY
|
||||
[ $? -ne 0 ] && fail=1
|
||||
|
||||
echo
|
||||
echo "== (b) CONNECTION threshold: strong ISE wires to wm_top, weak ISE wires nothing (flag ON) =="
|
||||
D="$WORK/b"; mkdir -p "$D"
|
||||
( export ENGRAM_CONSOLIDATION=1; "$WORK/p1" connect "$D" ) >"$WORK/connect.txt" 2>&1 || { echo "FAIL connect run"; fail=1; }
|
||||
cat "$WORK/connect.txt" | sed 's/^/ /'
|
||||
python3 - "$WORK/connect.txt" "$D/connect.json" <<'PY'
|
||||
import json,sys,re
|
||||
txt=open(sys.argv[1]).read()
|
||||
g=json.load(open(sys.argv[2]))
|
||||
def field(k):
|
||||
m=re.search(rf'{k} (\S+)',txt); return m.group(1) if m else None
|
||||
sid=field("ISE_STRONG_ID"); wid=field("ISE_WEAK_ID")
|
||||
m=re.search(r'EDGES before=(\d+) after_strong=(\d+) after_weak=(\d+)',txt)
|
||||
before,aftS,aftW=int(m.group(1)),int(m.group(2)),int(m.group(3))
|
||||
rc=0
|
||||
def check(c,mm):
|
||||
global rc; print((" PASS: " if c else " FAIL: ")+mm)
|
||||
if not c: rc=1
|
||||
strong_edges=[e for e in g["edges"] if e["from_id"]==sid and e["relation"]=="hebbian-associate"]
|
||||
weak_edges=[e for e in g["edges"] if e["from_id"]==wid]
|
||||
check(aftS>before, f"strong ISE formed connection edges ({before} -> {aftS})")
|
||||
check(aftW==aftS, f"weak ISE formed NO edges ({aftS} -> {aftW})")
|
||||
check(len(strong_edges)>=1, f"strong ISE has {len(strong_edges)} hebbian-associate edge(s) to wm_top")
|
||||
check(all('consolidated-from-ISE' in (e.get('metadata') or '') for e in strong_edges),
|
||||
"connection edges are provenance-tagged consolidated-from-ISE (reversible)")
|
||||
check(len(weak_edges)==0, "weak ISE (below connection bar) has zero outgoing edges")
|
||||
# targets must be the WM-top nodes (hebb-a / hebb-b), not distractors
|
||||
tgt_labels=set()
|
||||
byid={n["id"]:n for n in g["nodes"]}
|
||||
for e in strong_edges:
|
||||
t=byid.get(e["to_id"]);
|
||||
if t: tgt_labels.add(t.get("label"))
|
||||
print(f" connection targets: {sorted(tgt_labels)}")
|
||||
check(tgt_labels.issubset({"hebb-a","hebb-b"}) and len(tgt_labels)>=1,
|
||||
f"connections point at the wm_top nodes {sorted(tgt_labels)}")
|
||||
sys.exit(rc)
|
||||
PY
|
||||
[ $? -ne 0 ] && fail=1
|
||||
|
||||
echo
|
||||
echo "== (c) PERMANENCE threshold: promoted node survives 48h prune, ephemeral is swept (flag ON) =="
|
||||
D="$WORK/c"; mkdir -p "$D"
|
||||
( export ENGRAM_CONSOLIDATION=1 ENGRAM_CONSOL_PERM_MIN=-1000; "$WORK/p1" perm "$D" ) >"$WORK/perm.txt" 2>&1 || { echo "FAIL perm run"; fail=1; }
|
||||
cat "$WORK/perm.txt" | sed 's/^/ /'
|
||||
python3 - "$WORK/perm.txt" <<'PY'
|
||||
import sys,re,json
|
||||
txt=open(sys.argv[1]).read()
|
||||
rc=0
|
||||
def check(c,m):
|
||||
global rc; print((" PASS: " if c else " FAIL: ")+m)
|
||||
if not c: rc=1
|
||||
prom=int(re.search(r'PROMOTED (\d+)',txt).group(1))
|
||||
m=re.search(r'NODES before=(\d+) after=(\d+) removed=(\d+)',txt)
|
||||
before,after,removed=int(m.group(1)),int(m.group(2)),int(m.group(3))
|
||||
dur=re.search(r'DURABLE_NODE (\{.*\})',txt).group(1)
|
||||
eph=re.search(r'EPHEMERAL_NODE (\{.*\})',txt).group(1)
|
||||
durj=json.loads(dur); ephj=json.loads(eph)
|
||||
check(prom==1, "engram_consolidate_permanence promoted the node (returned 1)")
|
||||
check(before==2 and after==1 and removed==1, f"exactly one node pruned ({before}->{after}, removed={removed})")
|
||||
check(durj.get("id")=="ise-durable", "durable node SURVIVED the 48h telemetry prune")
|
||||
check('consolidated-from-ISE' in (durj.get("metadata") or ''), "durable node carries reversible provenance marker")
|
||||
check(ephj=={} or not ephj.get("id"), "ephemeral (non-permanent) ISE was swept")
|
||||
sys.exit(rc)
|
||||
PY
|
||||
[ $? -ne 0 ] && fail=1
|
||||
|
||||
echo
|
||||
echo "== (d) OFF path byte-identical: ISE creation forms no edges, permanence is a no-op =="
|
||||
D="$WORK/d"; mkdir -p "$D"
|
||||
( unset ENGRAM_CONSOLIDATION; "$WORK/p1" offcheck "$D" ) >"$WORK/off.txt" 2>&1
|
||||
rcoff=$?
|
||||
cat "$WORK/off.txt" | sed 's/^/ /'
|
||||
[ $rcoff -eq 0 ] && echo " PASS: flag OFF — ISE creation added 0 edges and permanence returned 0" \
|
||||
|| { echo " FAIL: OFF path changed behavior"; fail=1; }
|
||||
|
||||
echo
|
||||
echo "== ASan+UBSan (connect + perm + accrual-short) =="
|
||||
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
|
||||
-I "$INC" "$HERE/test_interoception_p1_consol.c" "$RT" "$ST" "$GEO" "$VIDX" \
|
||||
-lcurl -lm -o "$WORK/p1.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
|
||||
if [ -x "$WORK/p1.san" ]; then
|
||||
export ASAN_OPTIONS=detect_leaks=0
|
||||
DS="$WORK/san"; mkdir -p "$DS"
|
||||
( export ENGRAM_CONSOLIDATION=1 ENGRAM_CONSOL_PERM_MIN=-1000; "$WORK/p1.san" connect "$DS" ) >/dev/null 2>"$WORK/san_run.log"
|
||||
( export ENGRAM_CONSOLIDATION=1 ENGRAM_CONSOL_PERM_MIN=-1000; "$WORK/p1.san" perm "$DS" ) >/dev/null 2>>"$WORK/san_run.log"
|
||||
if grep -qiE 'runtime error|AddressSanitizer|Sanitizer|ERROR: ' "$WORK/san_run.log"; then
|
||||
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san_run.log" | head; fail=1
|
||||
else echo " ok: ASan+UBSan clean"; fi
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$fail" -eq 0 ]; then echo "====== P1 CONSOLIDATION GATE: PASS ======"; else echo "====== P1 CONSOLIDATION GATE: FAIL ======"; fi
|
||||
rm -rf "$WORK"
|
||||
exit $fail
|
||||
Executable
+96
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
# M-INTEROCEPTION P2 gate: chronoception (ENGRAM_CHRONOCEPTION).
|
||||
# Throwaway HOME + /tmp only. TC defaults to 3600s; we pin it for the math.
|
||||
set -u
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
RT="$HERE/../../lang/runtime/el_runtime.c"
|
||||
ST="$HERE/../../lang/runtime/engram_store.c"
|
||||
GEO="$HERE/../../lang/runtime/engram_geometry.c"
|
||||
VIDX="$HERE/../../lang/runtime/engram_vindex.c"
|
||||
INC="$HERE/../../lang/runtime"
|
||||
WORK="$(mktemp -d /tmp/engram-p2-XXXXXX)"
|
||||
export HOME="$WORK/home"; mkdir -p "$HOME"
|
||||
export ENGRAM_CHRONO_TC=3600 # pin cooling time-constant for the math
|
||||
unset ENGRAM_STORE
|
||||
fail=0
|
||||
|
||||
echo "== compile =="
|
||||
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p2_chrono.c" "$RT" "$ST" "$GEO" "$VIDX" \
|
||||
-lcurl -lm -o "$WORK/p2" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
|
||||
|
||||
sum_wm(){ python3 -c "import json,sys; g=json.load(open('$1')); print(sum(n.get('working_memory_weight',0) for n in g['nodes']))"; }
|
||||
|
||||
echo
|
||||
echo "== (a) cooling scales with dt (flag ON) =="
|
||||
for DT in 600000 1800000 3600000 7200000; do # 600s,1800s,3600s,7200s at TC=3600
|
||||
D="$WORK/dt$DT"; mkdir -p "$D"
|
||||
( export ENGRAM_CHRONOCEPTION=1; "$WORK/p2" once "$D" "$DT" ) >"$D/out.txt" 2>&1
|
||||
MAG=$(grep MAGNITUDE "$D/out.txt" | awk '{print $2}')
|
||||
WM=$(sum_wm "$D/field.json")
|
||||
PRED=$(python3 -c "import math; print(round(1-math.exp(-$DT/1000/3600),6))")
|
||||
echo " dt=${DT}ms magnitude=$MAG predicted 1-exp(-dt/TC)=$PRED field_wm_sum=$WM"
|
||||
python3 -c "import sys; m=float('$MAG'); p=float('$PRED'); sys.exit(0 if abs(m-p)<1e-4 else 1)" \
|
||||
&& echo " PASS: magnitude matches exp cooling" || { echo " FAIL"; fail=1; }
|
||||
done
|
||||
|
||||
echo
|
||||
echo "== (b) SCALE-INVARIANCE: age(dt) once == age(dt/N) N times (field within float tol) =="
|
||||
DT=3600000
|
||||
for N in 2 10 100; do
|
||||
DA="$WORK/inv_once_$N"; DB="$WORK/inv_split_$N"; mkdir -p "$DA" "$DB"
|
||||
( export ENGRAM_CHRONOCEPTION=1; "$WORK/p2" once "$DA" "$DT" ) >/dev/null 2>&1
|
||||
( export ENGRAM_CHRONOCEPTION=1; "$WORK/p2" split "$DB" "$DT" "$N" ) >/dev/null 2>&1
|
||||
WA=$(sum_wm "$DA/field.json"); WB=$(sum_wm "$DB/field.json")
|
||||
echo " N=$N once_wm=$WA split_wm=$WB |delta|=$(python3 -c "print(abs($WA-$WB))")"
|
||||
python3 -c "import sys; sys.exit(0 if abs($WA-$WB)<1e-9 else 1)" \
|
||||
&& echo " PASS: scale-invariant within 1e-9" || { echo " FAIL: not scale-invariant"; fail=1; }
|
||||
done
|
||||
|
||||
echo
|
||||
echo "== (c) REBOOT catch-up: one-shot cooling from persisted last-tick, reports MAGNITUDE not seconds =="
|
||||
D="$WORK/catch"; mkdir -p "$D"
|
||||
GAP=3600000 # 1h unconscious
|
||||
( export ENGRAM_CHRONOCEPTION=1 ENGRAM_DATA_DIR="$D"; "$WORK/p2" catchup "$D" "$GAP" ) >"$D/out.txt" 2>&1
|
||||
CMAG=$(grep CATCHUP_MAGNITUDE "$D/out.txt" | awk '{print $2}')
|
||||
CWM=$(sum_wm "$D/field.json")
|
||||
PRED=$(python3 -c "import math; print(round(1-math.exp(-$GAP/1000/3600),4))")
|
||||
echo " gap=${GAP}ms catchup_magnitude=$CMAG predicted=$PRED field_wm_sum=$CWM (was 0.6)"
|
||||
python3 -c "import sys; sys.exit(0 if abs(float('$CMAG')-float('$PRED'))<1e-2 else 1)" \
|
||||
&& echo " PASS: one-shot catch-up cooled by the elapsed gap, surfaced as a magnitude" \
|
||||
|| { echo " FAIL"; fail=1; }
|
||||
# honesty rail: magnitude is bounded [0,1), NOT an elapsed-seconds number
|
||||
python3 -c "import sys; m=float('$CMAG'); sys.exit(0 if 0<=m<1 else 1)" \
|
||||
&& echo " PASS: magnitude is a bounded drift signal in [0,1), never elapsed seconds" \
|
||||
|| { echo " FAIL: magnitude out of [0,1)"; fail=1; }
|
||||
|
||||
echo
|
||||
echo "== (d) OFF path: flag unset -> age & catchup return 0, field untouched =="
|
||||
D="$WORK/off"; mkdir -p "$D"
|
||||
( unset ENGRAM_CHRONOCEPTION; export ENGRAM_DATA_DIR="$D"; "$WORK/p2" offcheck "$D" 3600000 ) >"$D/out.txt" 2>&1
|
||||
cat "$D/out.txt" | sed 's/^/ /'
|
||||
OFFWM=$(sum_wm "$D/field.json")
|
||||
# loaded field wm sum = (1.0+0.8+0.6)*0.5 halving = 1.2 ; must be UNCHANGED
|
||||
echo " field_wm_sum=$OFFWM (expected 1.2, unchanged)"
|
||||
python3 -c "import sys; sys.exit(0 if abs($OFFWM-1.2)<1e-9 else 1)" \
|
||||
&& echo " PASS: OFF path leaves the field byte-identical (no aging)" \
|
||||
|| { echo " FAIL: OFF path modified the field"; fail=1; }
|
||||
|
||||
echo
|
||||
echo "== ASan+UBSan =="
|
||||
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
|
||||
-I "$INC" "$HERE/test_interoception_p2_chrono.c" "$RT" "$ST" "$GEO" "$VIDX" \
|
||||
-lcurl -lm -o "$WORK/p2.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
|
||||
if [ -x "$WORK/p2.san" ]; then
|
||||
export ASAN_OPTIONS=detect_leaks=0
|
||||
DS="$WORK/san"; mkdir -p "$DS"
|
||||
( export ENGRAM_CHRONOCEPTION=1 ENGRAM_DATA_DIR="$DS"; "$WORK/p2.san" once "$DS" 3600000 ) >/dev/null 2>"$WORK/san.log"
|
||||
( export ENGRAM_CHRONOCEPTION=1 ENGRAM_DATA_DIR="$DS"; "$WORK/p2.san" catchup "$DS" 3600000 ) >/dev/null 2>>"$WORK/san.log"
|
||||
if grep -qiE 'runtime error|AddressSanitizer|Sanitizer|ERROR: ' "$WORK/san.log"; then
|
||||
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san.log" | head; fail=1
|
||||
else echo " ok: ASan+UBSan clean"; fi
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$fail" -eq 0 ]; then echo "====== P2 CHRONOCEPTION GATE: PASS ======"; else echo "====== P2 CHRONOCEPTION GATE: FAIL ======"; fi
|
||||
rm -rf "$WORK"
|
||||
exit $fail
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
# M-INTEROCEPTION P3 gate: drift-sensor primitive engram_geo_displacement.
|
||||
# Read-only pure primitive; no store, no flag. Throwaway /tmp only.
|
||||
set -u
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
RT="$HERE/../../lang/runtime/el_runtime.c"
|
||||
ST="$HERE/../../lang/runtime/engram_store.c"
|
||||
GEO="$HERE/../../lang/runtime/engram_geometry.c"
|
||||
VIDX="$HERE/../../lang/runtime/engram_vindex.c"
|
||||
INC="$HERE/../../lang/runtime"
|
||||
WORK="$(mktemp -d /tmp/engram-p3-XXXXXX)"
|
||||
export HOME="$WORK/home"; mkdir -p "$HOME"
|
||||
fail=0
|
||||
|
||||
echo "== compile =="
|
||||
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p3_drift.c" "$RT" "$ST" "$GEO" "$VIDX" \
|
||||
-lcurl -lm -o "$WORK/p3" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
|
||||
|
||||
"$WORK/p3" > "$WORK/out.txt" 2>&1 || { echo "FAIL run"; cat "$WORK/out.txt"; fail=1; }
|
||||
cat "$WORK/out.txt" | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo "== assertions =="
|
||||
python3 - "$WORK/out.txt" <<'PY'
|
||||
import sys,re
|
||||
rows={}
|
||||
for line in open(sys.argv[1]):
|
||||
m=re.match(r'(\w+) (.*)',line.strip())
|
||||
if not m: continue
|
||||
tag=m.group(1); kv=dict(re.findall(r'(\w+)=([-\d.]+)',m.group(2)))
|
||||
rows[tag]={k:float(v) for k,v in kv.items()}
|
||||
rc=0
|
||||
def check(c,msg):
|
||||
global rc; print((" PASS: " if c else " FAIL: ")+msg)
|
||||
if not c: rc=1
|
||||
g=rows["GROWTH"]; c=rows["CORRUPTION"]; i=rows["IDENTITY"]
|
||||
check(g["core_disp"]<0.05, f"GROWTH: core displacement ~0 (core fixed) = {g['core_disp']}")
|
||||
check(g["periph_disp"]>0.30, f"GROWTH: periphery extended = {g['periph_disp']}")
|
||||
check(g["centroid_sep"]<1e-6, f"GROWTH: centroid unmoved = {g['centroid_sep']}")
|
||||
check(abs(g["radius_delta"]-0.4)<1e-4, f"GROWTH: radius grew by ~0.4 = {g['radius_delta']}")
|
||||
check(c["core_disp"]>0.40, f"CORRUPTION: core displaced strongly = {c['core_disp']}")
|
||||
check(c["periph_disp"]<0.05, f"CORRUPTION: periphery fixed = {c['periph_disp']}")
|
||||
check(c["centroid_sep"]>0.1, f"CORRUPTION: centroid moved = {c['centroid_sep']}")
|
||||
check(c["core_disp"] > 8*g["core_disp"]+0.3,
|
||||
f"SENSOR DISCRIMINATES: corruption core_disp ({c['core_disp']}) >> growth core_disp ({g['core_disp']})")
|
||||
check(i["core_disp"]==0 and i["periph_disp"]==0 and i["centroid_sep"]<1e-6,
|
||||
"IDENTITY: A vs A -> zero drift")
|
||||
sys.exit(rc)
|
||||
PY
|
||||
[ $? -ne 0 ] && fail=1
|
||||
|
||||
echo
|
||||
echo "== ASan+UBSan =="
|
||||
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
|
||||
-I "$INC" "$HERE/test_interoception_p3_drift.c" "$RT" "$ST" "$GEO" "$VIDX" \
|
||||
-lcurl -lm -o "$WORK/p3.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
|
||||
if [ -x "$WORK/p3.san" ]; then
|
||||
export ASAN_OPTIONS=detect_leaks=0
|
||||
"$WORK/p3.san" >/dev/null 2>"$WORK/san.log"
|
||||
if grep -qiE 'runtime error|AddressSanitizer|Sanitizer|ERROR: ' "$WORK/san.log"; then
|
||||
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san.log" | head; fail=1
|
||||
else echo " ok: ASan+UBSan clean"; fi
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$fail" -eq 0 ]; then echo "====== P3 DRIFT-SENSOR GATE: PASS ======"; else echo "====== P3 DRIFT-SENSOR GATE: FAIL ======"; fi
|
||||
rm -rf "$WORK"
|
||||
exit $fail
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
# M-INTEROCEPTION P4 gate: afferent input counters in act-stats (additive).
|
||||
set -u
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
RT="$HERE/../../lang/runtime/el_runtime.c"
|
||||
ST="$HERE/../../lang/runtime/engram_store.c"
|
||||
GEO="$HERE/../../lang/runtime/engram_geometry.c"
|
||||
VIDX="$HERE/../../lang/runtime/engram_vindex.c"
|
||||
INC="$HERE/../../lang/runtime"
|
||||
WORK="$(mktemp -d /tmp/engram-p4-XXXXXX)"
|
||||
export HOME="$WORK/home"; mkdir -p "$HOME"
|
||||
unset ENGRAM_STORE
|
||||
fail=0
|
||||
|
||||
echo "== compile =="
|
||||
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p4_afferent.c" "$RT" "$ST" "$GEO" "$VIDX" \
|
||||
-lcurl -lm -o "$WORK/p4" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
|
||||
|
||||
"$WORK/p4" > "$WORK/out.txt" 2>&1 || { echo "FAIL run"; cat "$WORK/out.txt"; fail=1; }
|
||||
grep -oE 'aff_[a-z_]+":[0-9]+' "$WORK/out.txt" | sed 's/^/ /' | head -30
|
||||
|
||||
echo
|
||||
echo "== assertions =="
|
||||
python3 - "$WORK/out.txt" <<'PY'
|
||||
import sys,re,json
|
||||
S={}
|
||||
for line in open(sys.argv[1]):
|
||||
m=re.match(r'(STATS\d) (\{.*\})',line.strip())
|
||||
if m: S[m.group(1)]=json.loads(m.group(2))
|
||||
rc=0
|
||||
def check(c,msg):
|
||||
global rc; print((" PASS: " if c else " FAIL: ")+msg)
|
||||
if not c: rc=1
|
||||
s0,s1,s2=S["STATS0"],S["STATS1"],S["STATS2"]
|
||||
# after creation, before any query
|
||||
check(s0["aff_node_creates"]==5, f"node_creates==5 (got {s0['aff_node_creates']})")
|
||||
check(s0["aff_ise_ingests"]==2, f"ise_ingests==2 (got {s0['aff_ise_ingests']})")
|
||||
check(s0["aff_edge_creates"]==2, f"edge_creates==2 (got {s0['aff_edge_creates']})")
|
||||
check(s0["aff_queries"]==0 and s0["aff_activations"]==0, "queries/activations start at 0")
|
||||
# after 4 queries
|
||||
check(s1["aff_queries"]==4, f"queries==4 (got {s1['aff_queries']})")
|
||||
check(s1["aff_activations"]==4, f"activations==4 (got {s1['aff_activations']})")
|
||||
check(s1["aff_node_creates"]==5 and s1["aff_ise_ingests"]==2 and s1["aff_edge_creates"]==2,
|
||||
"create counters unchanged by queries")
|
||||
# after 3 more queries — monotonic
|
||||
check(s2["aff_queries"]==7, f"queries==7 monotonic (got {s2['aff_queries']})")
|
||||
check(s2["aff_activations"]==7, f"activations==7 monotonic (got {s2['aff_activations']})")
|
||||
check(s2["aff_queries"]>s1["aff_queries"]>s0["aff_queries"], "queries strictly monotonic across readings")
|
||||
sys.exit(rc)
|
||||
PY
|
||||
[ $? -ne 0 ] && fail=1
|
||||
|
||||
echo
|
||||
echo "== ASan+UBSan =="
|
||||
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
|
||||
-I "$INC" "$HERE/test_interoception_p4_afferent.c" "$RT" "$ST" "$GEO" "$VIDX" \
|
||||
-lcurl -lm -o "$WORK/p4.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
|
||||
if [ -x "$WORK/p4.san" ]; then
|
||||
export ASAN_OPTIONS=detect_leaks=0
|
||||
"$WORK/p4.san" >/dev/null 2>"$WORK/san.log"
|
||||
if grep -qiE 'runtime error|AddressSanitizer|Sanitizer|ERROR: ' "$WORK/san.log"; then
|
||||
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san.log" | head; fail=1
|
||||
else echo " ok: ASan+UBSan clean"; fi
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$fail" -eq 0 ]; then echo "====== P4 AFFERENT-COUNTERS GATE: PASS ======"; else echo "====== P4 AFFERENT-COUNTERS GATE: FAIL ======"; fi
|
||||
rm -rf "$WORK"
|
||||
exit $fail
|
||||
Executable
+74
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env bash
|
||||
# M-INTEROCEPTION P5 gate: dream-recall builtin engram_dreams_json (honesty rail).
|
||||
set -u
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
RT="$HERE/../../lang/runtime/el_runtime.c"
|
||||
ST="$HERE/../../lang/runtime/engram_store.c"
|
||||
GEO="$HERE/../../lang/runtime/engram_geometry.c"
|
||||
VIDX="$HERE/../../lang/runtime/engram_vindex.c"
|
||||
INC="$HERE/../../lang/runtime"
|
||||
WORK="$(mktemp -d /tmp/engram-p5-XXXXXX)"
|
||||
export HOME="$WORK/home"; mkdir -p "$HOME"
|
||||
unset ENGRAM_STORE
|
||||
fail=0
|
||||
|
||||
echo "== compile =="
|
||||
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p5_dreams.c" "$RT" "$ST" "$GEO" "$VIDX" \
|
||||
-lcurl -lm -o "$WORK/p5" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
|
||||
|
||||
D="$WORK/d"; mkdir -p "$D"
|
||||
"$WORK/p5" "$D" > "$WORK/out.txt" 2>&1 || { echo "FAIL run"; cat "$WORK/out.txt"; fail=1; }
|
||||
cat "$WORK/out.txt" | sed 's/^/ /'
|
||||
|
||||
echo
|
||||
echo "== assertions =="
|
||||
python3 - "$WORK/out.txt" <<'PY'
|
||||
import sys,re,json
|
||||
L={}
|
||||
for line in open(sys.argv[1]):
|
||||
line=line.strip()
|
||||
m=re.match(r'(BEFORE|AFTER) (\[.*\])',line)
|
||||
if m: L[m.group(1)]=json.loads(m.group(2)); continue
|
||||
m=re.match(r'PRUNED (\d+)',line)
|
||||
if m: L['PRUNED']=int(m.group(1)); continue
|
||||
m=re.match(r'SINCE (\d+) (\[.*\])',line)
|
||||
if m: L['SINCE']=json.loads(m.group(2))
|
||||
rc=0
|
||||
def check(c,msg):
|
||||
global rc; print((" PASS: " if c else " FAIL: ")+msg)
|
||||
if not c: rc=1
|
||||
before_ids={d["id"] for d in L["BEFORE"]}
|
||||
after_ids={d["id"] for d in L["AFTER"]}
|
||||
since_ids={d["id"] for d in L["SINCE"]}
|
||||
check(before_ids=={"cur_old","cur_mid","cur_recent"}, f"before prune: all 3 curiosity_scan, heartbeat excluded (got {sorted(before_ids)})")
|
||||
check("hb_recent" not in before_ids, "heartbeat ISE never appears (not a dream)")
|
||||
check(L["PRUNED"]==1, f"prune rotated out exactly the ancient ISE (pruned={L['PRUNED']})")
|
||||
check(after_ids=={"cur_mid","cur_recent"}, f"after prune: rotated-out cur_old is ABSENT, not confabulated (got {sorted(after_ids)})")
|
||||
check("cur_old" not in after_ids, "honesty rail: pruned dream is gone = 'I don't remember', never synthesized")
|
||||
check(since_ids=={"cur_recent"}, f"since filter returns only events after the cutoff (got {sorted(since_ids)})")
|
||||
# no fabrication: every returned id was one we seeded
|
||||
seeded={"cur_old","cur_mid","cur_recent","hb_recent"}
|
||||
allret=before_ids|after_ids|since_ids
|
||||
check(allret<=seeded, f"no fabricated entries — every returned id was seeded ({sorted(allret)})")
|
||||
sys.exit(rc)
|
||||
PY
|
||||
[ $? -ne 0 ] && fail=1
|
||||
|
||||
echo
|
||||
echo "== ASan+UBSan =="
|
||||
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
|
||||
-I "$INC" "$HERE/test_interoception_p5_dreams.c" "$RT" "$ST" "$GEO" "$VIDX" \
|
||||
-lcurl -lm -o "$WORK/p5.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -25 "$WORK/san_cc.log"; fail=1; }
|
||||
if [ -x "$WORK/p5.san" ]; then
|
||||
export ASAN_OPTIONS=detect_leaks=0
|
||||
DS="$WORK/ds"; mkdir -p "$DS"
|
||||
"$WORK/p5.san" "$DS" >/dev/null 2>"$WORK/san.log"
|
||||
if grep -qiE 'runtime error|AddressSanitizer|Sanitizer|ERROR: ' "$WORK/san.log"; then
|
||||
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san.log" | head; fail=1
|
||||
else echo " ok: ASan+UBSan clean"; fi
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$fail" -eq 0 ]; then echo "====== P5 DREAM-RECALL GATE: PASS ======"; else echo "====== P5 DREAM-RECALL GATE: FAIL ======"; fi
|
||||
rm -rf "$WORK"
|
||||
exit $fail
|
||||
Executable
+158
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env bash
|
||||
# M3.5 PRE-FLIP GATE. Pure C harness (NOT elb/elc): links the real el_runtime.c
|
||||
# native engram builtins + engram_store.c and proves activation-time field
|
||||
# mutations (edge hebb, node activation_count, WM weight) persist through a
|
||||
# checkpoint and survive a reboot from neuron.egm with snapshot.json DELETED.
|
||||
# Writes ONLY under a throwaway /tmp dir with a throwaway HOME.
|
||||
set -u
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
RT="$HERE/../../lang/runtime/el_runtime.c"
|
||||
ST="$HERE/../../lang/runtime/engram_store.c"
|
||||
INC="$HERE/../../lang/runtime"
|
||||
WORK="$(mktemp -d /tmp/engram-m35-XXXXXX)"
|
||||
BIN="$WORK/m35"
|
||||
export HOME="$WORK/home"; mkdir -p "$HOME" # never touch real ~/.neuron
|
||||
export ENGRAM_WAL_SYNC=always
|
||||
unset ENGRAM_STORE
|
||||
fail=0
|
||||
|
||||
echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m35_hebb_persist.c) =="
|
||||
gcc -O1 -std=c11 -I "$INC" "$HERE/test_m35_hebb_persist.c" "$RT" "$ST" -lcurl -o "$BIN" 2>"$WORK/cc.log"
|
||||
if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi
|
||||
|
||||
echo
|
||||
echo "== 0) flag-OFF: seed+activate+checkpoint must NOT touch the store =="
|
||||
DOFF="$WORK/off"; mkdir -p "$DOFF"
|
||||
( unset ENGRAM_STORE; "$BIN" offcheck "$DOFF" )
|
||||
[ $? -ne 0 ] && { echo "FAIL: offcheck"; fail=1; }
|
||||
[ -e "$DOFF/neuron.egm" ] && { echo "FAIL: neuron.egm created while flag OFF"; fail=1; } \
|
||||
|| echo " ok: no neuron.egm created with flag OFF"
|
||||
|
||||
echo
|
||||
echo "== 1) POSITIVE: ENGRAM_STORE=1 seed -> activate -> checkpoint(field-persist) -> close =="
|
||||
DPOS="$WORK/pos"; mkdir -p "$DPOS"
|
||||
ENGRAM_STORE=1 "$BIN" pos_seed "$DPOS" || { echo "FAIL: pos_seed"; fail=1; }
|
||||
[ -e "$DPOS/neuron.egm" ] && echo " ok: neuron.egm created" || { echo "FAIL: neuron.egm missing"; fail=1; }
|
||||
|
||||
echo
|
||||
echo "== 2) reboot from neuron.egm with snapshot.json DELETED (must never read JSON) =="
|
||||
rm -f "$DPOS/snapshot.json"
|
||||
ENGRAM_STORE=1 "$BIN" pos_reboot "$DPOS" || { echo "FAIL: pos_reboot"; fail=1; }
|
||||
|
||||
echo
|
||||
echo "== 3) NEGATIVE CONTROL: seed -> activate -> close WITHOUT the field-persist checkpoint =="
|
||||
DNEG="$WORK/neg"; mkdir -p "$DNEG"
|
||||
ENGRAM_STORE=1 "$BIN" neg_seed "$DNEG" || { echo "FAIL: neg_seed"; fail=1; }
|
||||
rm -f "$DNEG/snapshot.json"
|
||||
ENGRAM_STORE=1 "$BIN" neg_reboot "$DNEG" || { echo "FAIL: neg_reboot"; fail=1; }
|
||||
|
||||
echo
|
||||
echo "== 4) assertions (python over the JSON exports) =="
|
||||
python3 - "$DPOS" "$DNEG" <<'PY'
|
||||
import json, sys, os
|
||||
WM_FLOOR = 0.05
|
||||
HEBB_MIN = 1e-6
|
||||
|
||||
def load(d, name):
|
||||
with open(os.path.join(d, name)) as f: return json.load(f)
|
||||
|
||||
def node_by_label(g, label):
|
||||
for n in g["nodes"]:
|
||||
if n.get("label") == label: return n
|
||||
return None
|
||||
|
||||
def edge_between(g, a_id, b_id):
|
||||
for e in g["edges"]:
|
||||
if e.get("from_id") == a_id and e.get("to_id") == b_id:
|
||||
return e
|
||||
return None
|
||||
|
||||
rc = 0
|
||||
def check(cond, msg):
|
||||
global rc
|
||||
if cond: print(f" PASS: {msg}")
|
||||
else: print(f" FAIL: {msg}"); rc = 1
|
||||
|
||||
dpos, dneg = sys.argv[1], sys.argv[2]
|
||||
pre = load(dpos, "pre_reboot.json")
|
||||
rebt = load(dpos, "reboot.json")
|
||||
|
||||
pa, pb = node_by_label(pre, "hebb-a"), node_by_label(pre, "hebb-b")
|
||||
ra = node_by_label(rebt, "hebb-a")
|
||||
assert pa and pb and ra, "target nodes missing"
|
||||
pe = edge_between(pre, pa["id"], pb["id"])
|
||||
re = edge_between(rebt, pa["id"], pb["id"])
|
||||
assert pe and re, "target edge missing"
|
||||
|
||||
pre_hebb = pe.get("hebb", 0.0)
|
||||
rebt_hebb = re.get("hebb", 0.0)
|
||||
pre_ac = pa.get("activation_count", 0)
|
||||
rebt_ac = ra.get("activation_count", 0)
|
||||
pre_wm = pa.get("working_memory_weight", 0.0)
|
||||
rebt_wm = ra.get("working_memory_weight", 0.0)
|
||||
|
||||
print(f" edge hebb-a->hebb-b : pre={pre_hebb!r} reboot={rebt_hebb!r}")
|
||||
print(f" node hebb-a act_cnt : pre={pre_ac!r} reboot={rebt_ac!r}")
|
||||
print(f" node hebb-a wm : pre={pre_wm!r} reboot={rebt_wm!r} (halved+floored expected)")
|
||||
|
||||
# --- learning actually happened this run (else the test proves nothing) ---
|
||||
check(pre_hebb > HEBB_MIN, f"activation raised edge hebb above 0 (pre={pre_hebb})")
|
||||
check(pre_ac >= 1, f"activation reinforced node activation_count (pre={pre_ac})")
|
||||
check(pre_wm > 0.0, f"activation promoted node to working memory (pre_wm={pre_wm})")
|
||||
|
||||
# --- the load-bearing survival assertions after a real delete-JSON reboot ---
|
||||
check(abs(rebt_hebb - pre_hebb) < 1e-12,
|
||||
f"edge hebb SURVIVED reboot unchanged ({rebt_hebb} == {pre_hebb})")
|
||||
check(rebt_ac == pre_ac,
|
||||
f"node activation_count SURVIVED reboot unchanged ({rebt_ac} == {pre_ac})")
|
||||
|
||||
# --- WM weight: must equal the JSON path's boot transform exactly (halve+floor) ---
|
||||
expected_wm = pre_wm * 0.5
|
||||
if expected_wm < WM_FLOOR: expected_wm = 0.0
|
||||
check(abs(rebt_wm - expected_wm) < 1e-9,
|
||||
f"node WM weight SURVIVED with the SAME boot transform as JSON path "
|
||||
f"(reboot={rebt_wm} == halve+floor(pre)={expected_wm})")
|
||||
check(expected_wm > 0.0,
|
||||
f"WM survival is observable (halved weight stays above floor: {expected_wm} > {WM_FLOOR})")
|
||||
|
||||
# --- NEGATIVE CONTROL: without the field-persist step the learning is LOST ---
|
||||
npre = load(dneg, "neg_pre.json")
|
||||
nrebt = load(dneg, "neg_reboot.json")
|
||||
na_pre = node_by_label(npre, "hebb-a")
|
||||
na_rebt = node_by_label(nrebt, "hebb-a")
|
||||
ne_pre = edge_between(npre, na_pre["id"], node_by_label(npre, "hebb-b")["id"])
|
||||
ne_rebt = edge_between(nrebt, na_rebt["id"], node_by_label(nrebt, "hebb-b")["id"])
|
||||
print(f" [neg] edge hebb : pre={ne_pre.get('hebb',0.0)!r} reboot={ne_rebt.get('hebb',0.0)!r}")
|
||||
print(f" [neg] node act_cnt : pre={na_pre.get('activation_count',0)!r} reboot={na_rebt.get('activation_count',0)!r}")
|
||||
check(ne_pre.get("hebb", 0.0) > HEBB_MIN,
|
||||
f"[neg] activation DID raise hebb in RAM (pre={ne_pre.get('hebb',0.0)})")
|
||||
check(ne_rebt.get("hebb", 0.0) == 0.0,
|
||||
"[neg] WITHOUT checkpoint field-persist, edge hebb is LOST on reboot (==0) — fix is load-bearing")
|
||||
check(na_rebt.get("activation_count", 0) == 0,
|
||||
"[neg] WITHOUT checkpoint field-persist, activation_count is LOST on reboot (==0)")
|
||||
|
||||
sys.exit(rc)
|
||||
PY
|
||||
[ $? -ne 0 ] && fail=1
|
||||
|
||||
echo
|
||||
echo "== 5) ASan+UBSan build, exercise the full persist+reboot flow (leaks off — harness intentionally leaks el_strdup) =="
|
||||
SANBIN="$WORK/m35.san"
|
||||
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
|
||||
-I "$INC" "$HERE/test_m35_hebb_persist.c" "$RT" "$ST" -lcurl -o "$SANBIN" 2>"$WORK/san_cc.log"
|
||||
if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else
|
||||
export ASAN_OPTIONS=detect_leaks=0
|
||||
DSAN="$WORK/san"; mkdir -p "$DSAN"
|
||||
ENGRAM_STORE=1 "$SANBIN" pos_seed "$DSAN" >/dev/null 2>"$WORK/san_run.log" && \
|
||||
{ rm -f "$DSAN/snapshot.json"; ENGRAM_STORE=1 "$SANBIN" pos_reboot "$DSAN" >/dev/null 2>>"$WORK/san_run.log"; }
|
||||
if grep -qiE 'runtime error|AddressSanitizer|UndefinedBehavior|ERROR: ' "$WORK/san_run.log"; then
|
||||
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san_run.log" | head; fail=1
|
||||
else
|
||||
echo " ok: ASan+UBSan clean across pos_seed/checkpoint/reboot (field-persist, boot laundering)"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$fail" -eq 0 ]; then echo "================ M3.5 HEBB-PERSIST GATE: PASS ================"; else echo "================ M3.5 HEBB-PERSIST GATE: FAIL ================"; fi
|
||||
rm -rf "$WORK"
|
||||
exit $fail
|
||||
Executable
+126
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env bash
|
||||
# M3 JSON-parity gate. Pure C harness (NOT elb/elc): links the real el_runtime.c
|
||||
# native engram builtins + engram_store.c and drives ENGRAM_STORE on vs off.
|
||||
# Writes ONLY under a throwaway /tmp dir with a throwaway HOME + ENGRAM_DATA_DIR.
|
||||
set -u
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
RT="$HERE/../../lang/runtime/el_runtime.c"
|
||||
ST="$HERE/../../lang/runtime/engram_store.c"
|
||||
INC="$HERE/../../lang/runtime"
|
||||
WORK="$(mktemp -d /tmp/engram-m3-XXXXXX)"
|
||||
DATA="$WORK/data"; mkdir -p "$DATA"
|
||||
BIN="$WORK/m3"
|
||||
export HOME="$WORK/home"; mkdir -p "$HOME" # never touch real ~/.neuron
|
||||
export ENGRAM_DATA_DIR="$DATA"
|
||||
export ENGRAM_WAL_SYNC=always
|
||||
unset ENGRAM_STORE
|
||||
fail=0
|
||||
|
||||
echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m3_parity.c) =="
|
||||
gcc -O1 -std=c11 -I "$INC" "$HERE/test_m3_parity.c" "$RT" "$ST" -lcurl -o "$BIN" 2>"$WORK/cc.log"
|
||||
if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi
|
||||
grep -i warning "$WORK/cc.log" | grep -iE 'engram_store|eg_store|eg_load|scan_nodes|scan_edges' && echo "(warnings in M3 code above)" || true
|
||||
|
||||
echo
|
||||
echo "== 0) default-OFF: flag unset leaves the store untouched =="
|
||||
( unset ENGRAM_STORE; "$BIN" offcheck "$DATA" )
|
||||
[ $? -ne 0 ] && { echo "FAIL: offcheck"; fail=1; }
|
||||
[ -e "$DATA/neuron.egm" ] && { echo "FAIL: neuron.egm created while flag OFF"; fail=1; } \
|
||||
|| echo " ok: no neuron.egm created with flag OFF"
|
||||
|
||||
echo
|
||||
echo "== 1) seed (ENGRAM_STORE unset): build graph, save snapshot.json, activate =="
|
||||
( unset ENGRAM_STORE; "$BIN" seed "$DATA" ) || { echo "FAIL: seed"; fail=1; }
|
||||
|
||||
echo
|
||||
echo "== 2) on (ENGRAM_STORE=1): import snapshot.json ONCE -> neuron.egm, resident-load, activate =="
|
||||
ENGRAM_STORE=1 "$BIN" on "$DATA" || { echo "FAIL: on"; fail=1; }
|
||||
[ -e "$DATA/neuron.egm" ] && echo " ok: neuron.egm created by import" || { echo "FAIL: neuron.egm missing"; fail=1; }
|
||||
|
||||
echo
|
||||
echo "== 3) reboot (ENGRAM_STORE=1, snapshot.json DELETED): must load from neuron.egm, never JSON =="
|
||||
rm -f "$DATA/snapshot.json"
|
||||
ENGRAM_STORE=1 "$BIN" reboot "$DATA" || { echo "FAIL: reboot"; fail=1; }
|
||||
|
||||
echo
|
||||
echo "== 4) parity comparison (modulo ordering) =="
|
||||
python3 - "$DATA" <<'PY'
|
||||
import json, sys, os
|
||||
d = sys.argv[1]
|
||||
def load(name):
|
||||
with open(os.path.join(d, name)) as f: return json.load(f)
|
||||
def norm_graph(g):
|
||||
nodes = sorted(g.get("nodes", []), key=lambda n: n.get("id",""))
|
||||
edges = sorted(g.get("edges", []), key=lambda e: e.get("id",""))
|
||||
layers= sorted(g.get("layers", []), key=lambda l: l.get("layer_id",0))
|
||||
return {"nodes":nodes, "edges":edges, "layers":layers}
|
||||
def act_ids(a):
|
||||
# list of (node id, promoted); robust set + ordered list
|
||||
seq = [(e.get("node",{}).get("id",""), int(e.get("promoted",0))) for e in a]
|
||||
return seq
|
||||
|
||||
rc = 0
|
||||
snap = norm_graph(load("snapshot.json") if os.path.exists(os.path.join(d,"snapshot.json")) else load("off_graph.json"))
|
||||
off = norm_graph(load("off_graph.json"))
|
||||
on = norm_graph(load("on_graph.json"))
|
||||
rebt = norm_graph(load("reboot_graph.json"))
|
||||
|
||||
def cmp(label, a, b):
|
||||
global rc
|
||||
if a == b:
|
||||
print(f" PASS: {label} (nodes={len(a['nodes'])} edges={len(a['edges'])} layers={len(a['layers'])})")
|
||||
else:
|
||||
rc = 1
|
||||
print(f" FAIL: {label}")
|
||||
for k in ("nodes","edges","layers"):
|
||||
if a[k] != b[k]:
|
||||
print(f" {k}: {len(a[k])} vs {len(b[k])}")
|
||||
for x,y in zip(a[k], b[k]):
|
||||
if x != y:
|
||||
print(f" first diff:\n A={json.dumps(x)[:300]}\n B={json.dumps(y)[:300]}")
|
||||
break
|
||||
|
||||
cmp("graph: ENGRAM_STORE=1 (export) == ENGRAM_STORE=0 (JSON path)", on, off)
|
||||
cmp("round-trip: snapshot.json seed == store export (on_graph)", on, off) # off_graph==snapshot save
|
||||
cmp("reboot from neuron.egm (no JSON) == on-path store", rebt, on)
|
||||
|
||||
offa = act_ids(load("off_act.json"))
|
||||
ona = act_ids(load("on_act.json"))
|
||||
if set(offa) == set(ona):
|
||||
print(f" PASS: activation result set identical (off={len(offa)} on={len(ona)} entries)")
|
||||
if offa == ona:
|
||||
print(" (and identical ordering/promotion sequence)")
|
||||
else:
|
||||
print(" (same set; ordering differs only where scores tie — reporting honestly)")
|
||||
else:
|
||||
rc = 1
|
||||
print(" FAIL: activation result set differs")
|
||||
print(f" off-only: {set(offa)-set(ona)}")
|
||||
print(f" on-only: {set(ona)-set(offa)}")
|
||||
|
||||
sys.exit(rc)
|
||||
PY
|
||||
[ $? -ne 0 ] && fail=1
|
||||
|
||||
echo
|
||||
echo "== 5) ASan+UBSan build, exercise M3 scan/boot/hooks (leaks off — harness intentionally leaks el_strdup) =="
|
||||
SANBIN="$WORK/m3.san"
|
||||
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
|
||||
-I "$INC" "$HERE/test_m3_parity.c" "$RT" "$ST" -lcurl -o "$SANBIN" 2>"$WORK/san_cc.log"
|
||||
if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else
|
||||
export ASAN_OPTIONS=detect_leaks=0
|
||||
DATA2="$WORK/data2"; mkdir -p "$DATA2"
|
||||
( unset ENGRAM_STORE; "$SANBIN" seed "$DATA2" ) >/dev/null 2>"$WORK/san_run.log" && \
|
||||
ENGRAM_STORE=1 "$SANBIN" on "$DATA2" >/dev/null 2>>"$WORK/san_run.log" && \
|
||||
{ rm -f "$DATA2/snapshot.json"; ENGRAM_STORE=1 "$SANBIN" reboot "$DATA2" >/dev/null 2>>"$WORK/san_run.log"; }
|
||||
if grep -qiE 'runtime error|AddressSanitizer|UndefinedBehavior|ERROR: ' "$WORK/san_run.log"; then
|
||||
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san_run.log" | head; fail=1
|
||||
else
|
||||
echo " ok: ASan+UBSan clean across seed/on/reboot (scan, boot, resident-load, mutation hooks)"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$fail" -eq 0 ]; then echo "================ M3 PARITY GATE: PASS ================"; else echo "================ M3 PARITY GATE: FAIL ================"; fi
|
||||
rm -rf "$WORK"
|
||||
exit $fail
|
||||
Executable
+137
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
# M7 index-driven-traversal gate. Pure C harness (NOT elb/elc): links the real
|
||||
# el_runtime.c engram builtins + engram_store.c and drives ENGRAM_STORE off vs on.
|
||||
# Proves (1) byte-identical activation parity flag-on == flag-off across a
|
||||
# mutating query sequence, and (2) the O(E)-rebuild cost is eliminated flag-on.
|
||||
# Writes ONLY under a throwaway /tmp dir with a throwaway HOME.
|
||||
set -u
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
RT="$HERE/../../lang/runtime/el_runtime.c"
|
||||
ST="$HERE/../../lang/runtime/engram_store.c"
|
||||
INC="$HERE/../../lang/runtime"
|
||||
WORK="$(mktemp -d /tmp/engram-m7-XXXXXX)"
|
||||
DATA="$WORK/data"; mkdir -p "$DATA"
|
||||
BIN="$WORK/m7"
|
||||
export HOME="$WORK/home"; mkdir -p "$HOME" # never touch real ~/.neuron
|
||||
# Hermetic: point the embedder at a guaranteed-refused endpoint so eg_embed_fetch
|
||||
# fails fast, the circuit breaker opens, and cosq is deterministically absent in
|
||||
# EVERY run (no dependence on whether a dev Ollama happens to be listening). This
|
||||
# makes the byte-identical parity comparison reproducible and non-flaky.
|
||||
export EL_EMBED_URL="http://127.0.0.1:1/api/embeddings"
|
||||
unset ENGRAM_STORE
|
||||
fail=0
|
||||
|
||||
echo "== compiling harness (gcc: el_runtime.c + engram_store.c + test_m7_traversal.c) =="
|
||||
gcc -O2 -std=c11 -I "$INC" "$HERE/test_m7_traversal.c" "$RT" "$ST" -lcurl -lm -o "$BIN" 2>"$WORK/cc.log"
|
||||
if [ $? -ne 0 ]; then echo "COMPILE FAILED:"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; fi
|
||||
echo " ok: compiled"
|
||||
|
||||
echo
|
||||
echo "== 1) PARITY: index-driven (M7 incremental) activation must be IDENTICAL to the"
|
||||
echo " full-rebuild scan path — proven under one identical ENGRAM_STORE=1 state,"
|
||||
echo " so the ONLY variable is how per-node adjacency is maintained."
|
||||
echo " (compared on deterministic fields: node label + activation_strength +"
|
||||
echo " working_memory_weight + epistemic_confidence + hops + promoted, IN ORDER;"
|
||||
echo " node id/timestamps are per-run random and are intentionally excluded.)"
|
||||
( unset ENGRAM_STORE; "$BIN" parity-off "$DATA" ) || { echo "FAIL: parity-off run"; fail=1; }
|
||||
ENGRAM_STORE=1 "$BIN" parity-on-rebuild "$DATA" || { echo "FAIL: parity-on-rebuild run"; fail=1; }
|
||||
ENGRAM_STORE=1 "$BIN" parity-on-incr "$DATA" || { echo "FAIL: parity-on-incr run"; fail=1; }
|
||||
python3 - "$DATA" <<'PY' || fail=1
|
||||
import json, sys, os
|
||||
d = sys.argv[1]
|
||||
def proj(prefix, i):
|
||||
a = json.load(open(os.path.join(d, f"{prefix}_act{i}.json")))
|
||||
out = []
|
||||
for e in a:
|
||||
n = e.get("node", {})
|
||||
out.append([n.get("label",""),
|
||||
e.get("activation_strength"), e.get("working_memory_weight"),
|
||||
e.get("epistemic_confidence"), e.get("hops"), e.get("promoted")])
|
||||
return out
|
||||
def compare(label, pa, pb, gate):
|
||||
rc = 0
|
||||
for i in (1,2,3,4):
|
||||
a, b = proj(pa, i), proj(pb, i)
|
||||
if a == b:
|
||||
print(f" #{i} identical (entries={len(a)}, promoted={sum(1 for r in a if r[5])})")
|
||||
else:
|
||||
if gate: rc = 1
|
||||
print(f" #{i} DIFFERS ({'FAIL' if gate else 'note'})")
|
||||
for x,y in zip(a,b):
|
||||
if x != y:
|
||||
print(f" first diff:\n {pa}={x}\n {pb}={y}"); break
|
||||
if len(a) != len(b): print(f" length: {pa}={len(a)} {pb}={len(b)}")
|
||||
print(f" {'PASS' if rc==0 else 'FAIL'}: {label}")
|
||||
return rc
|
||||
|
||||
print(" [CORE M7 GATE] flag-on incremental index == flag-on forced full rebuild:")
|
||||
rc1 = compare("index-driven activation == full-rebuild scan (same flag state)",
|
||||
"onincr", "onrb", gate=True)
|
||||
print(" [context] flag-on incremental index vs flag-off scan path (today's behavior):")
|
||||
rc2 = compare("M7 (flag-on) == flag-off scan path", "onincr", "off", gate=False)
|
||||
print(" [context] flag-off scan vs flag-on forced rebuild (isolates any pre-existing")
|
||||
print(" flag-on/off float difference, INDEPENDENT of M7's incremental path):")
|
||||
rc3 = compare("flag-off == flag-on (both rebuild path)", "off", "onrb", gate=False)
|
||||
sys.exit(rc1) # only the core M7 equivalence gates the result
|
||||
PY
|
||||
|
||||
echo
|
||||
echo "== 2) PERF: ~13k nodes / 43k edges, 200 (add-edge + activate) iterations =="
|
||||
NODES=13000; EDGES=43000; ITERS=120
|
||||
( unset ENGRAM_STORE; "$BIN" perf off "$DATA" "$NODES" "$EDGES" "$ITERS" ) | tee "$WORK/perf_off.txt"
|
||||
[ ${PIPESTATUS[0]} -ne 0 ] && { echo "FAIL: perf off"; fail=1; }
|
||||
ENGRAM_STORE=1 "$BIN" perf on "$DATA" "$NODES" "$EDGES" "$ITERS" | tee "$WORK/perf_on.txt"
|
||||
[ ${PIPESTATUS[0]} -ne 0 ] && { echo "FAIL: perf on"; fail=1; }
|
||||
python3 - "$WORK/perf_off.txt" "$WORK/perf_on.txt" <<'PY'
|
||||
import re, sys
|
||||
def parse(f):
|
||||
t = open(f).read()
|
||||
def g(k):
|
||||
m = re.search(k+r'=([\d.]+)', t); return float(m.group(1)) if m else 0.0
|
||||
return {'rw': g('rebuild_edge_work'), 'rb': g('rebuilds'), 'ap': g('incr_appends'),
|
||||
'loop_s': g('loop='), 'maint': g('adj_maint'),
|
||||
'perq': g('per_query')}
|
||||
off, on = parse(sys.argv[1]), parse(sys.argv[2])
|
||||
def ratio(a,b): return (a/b) if b else float('inf')
|
||||
print()
|
||||
print(f" ADJACENCY TRAVERSAL COST (the metric M7 changes):")
|
||||
print(f" edge-touches in rebuilds: off={off['rw']:.0f} on={on['rw']:.0f} "
|
||||
f"({ratio(off['rw'],on['rw']):.0f}x fewer on)")
|
||||
print(f" full O(E) rebuilds: off={off['rb']:.0f} on={on['rb']:.0f}")
|
||||
print(f" incremental O(1) appends: off={off['ap']:.0f} on={on['ap']:.0f}")
|
||||
print(f" adjacency-maint wall-time: off={off['maint']:.4f}s on={on['maint']:.4f}s "
|
||||
f"({ratio(off['maint'],on['maint']):.1f}x faster on)")
|
||||
print(f" END-TO-END per-query time: off={off['perq']:.2f}ms on={on['perq']:.2f}ms")
|
||||
print(f" (per-query is dominated by activation's O(N) node scoring over 13k nodes,")
|
||||
print(f" which M7 does not touch; the delta is the eliminated rebuild time.)")
|
||||
ok = on['rw'] < off['rw'] and on['maint'] < off['maint'] and on['rb'] < off['rb']
|
||||
print(" PASS: flag-on eliminates the O(E) per-query rebuild (fewer edge-touches, less maint time)"
|
||||
if ok else " FAIL: expected fewer edge-touches AND less adjacency-maint time on flag-on")
|
||||
sys.exit(0 if ok else 1)
|
||||
PY
|
||||
[ $? -ne 0 ] && fail=1
|
||||
|
||||
echo
|
||||
echo "== 3) ASan+UBSan clean across parity + a small perf loop (leaks off — harness intentionally leaks el_strdup) =="
|
||||
SANBIN="$WORK/m7.san"
|
||||
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
|
||||
-I "$INC" "$HERE/test_m7_traversal.c" "$RT" "$ST" -lcurl -lm -o "$SANBIN" 2>"$WORK/san_cc.log"
|
||||
if [ $? -ne 0 ]; then echo " SAN COMPILE FAILED:"; tail -20 "$WORK/san_cc.log"; fail=1; else
|
||||
export ASAN_OPTIONS=detect_leaks=0
|
||||
D2="$WORK/data2"; mkdir -p "$D2"
|
||||
( unset ENGRAM_STORE; "$SANBIN" parity-off "$D2" ) >/dev/null 2>"$WORK/san_run.log" && \
|
||||
ENGRAM_STORE=1 "$SANBIN" parity-on-rebuild "$D2" >/dev/null 2>>"$WORK/san_run.log" && \
|
||||
ENGRAM_STORE=1 "$SANBIN" parity-on-incr "$D2" >/dev/null 2>>"$WORK/san_run.log" && \
|
||||
( unset ENGRAM_STORE; "$SANBIN" perf off "$D2" 1500 5000 40 ) >/dev/null 2>>"$WORK/san_run.log" && \
|
||||
ENGRAM_STORE=1 "$SANBIN" perf on "$D2" 1500 5000 40 >/dev/null 2>>"$WORK/san_run.log"
|
||||
if grep -qiE 'runtime error|AddressSanitizer|UndefinedBehavior|ERROR: ' "$WORK/san_run.log"; then
|
||||
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san_run.log" | head; fail=1
|
||||
else
|
||||
echo " ok: ASan+UBSan clean across parity + perf (rebuild + incremental append + BFS)"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$fail" -eq 0 ]; then echo "================ M7 TRAVERSAL GATE: PASS ================"; else echo "================ M7 TRAVERSAL GATE: FAIL ================"; fi
|
||||
rm -rf "$WORK"
|
||||
exit $fail
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/sh
|
||||
# Build + RUN the REASONING-layer tests (engram_reason.c): closed-form constructed
|
||||
# cases for ANALOGY / INDUCTION / ABDUCTION / CAUSAL / PLANNING, each composing the
|
||||
# §5 geometry OPERATORS (engram_geometry.c). Pure C11 (stdlib + libm). Standalone —
|
||||
# NOT folded through elc. Two passes:
|
||||
# 1. PERF — optimised (-O2, no sanitizer): the functional gate.
|
||||
# 2. SAFETY — ASan + UBSan on the same suite (memory-safety is size-independent).
|
||||
set -e
|
||||
HERE=$(cd "$(dirname "$0")" && pwd)
|
||||
RT="$HERE/../../lang/runtime"
|
||||
CC=${CC:-cc}
|
||||
SRC="$HERE/test_reason.c $RT/engram_reason.c $RT/engram_geometry.c $RT/engram_store.c $RT/engram_vindex.c"
|
||||
WARN="-std=c11 -Wall -Wextra"
|
||||
TMP=$(mktemp -d)
|
||||
|
||||
echo "### PASS 1: PERF (optimised, un-sanitised) — functional gate"
|
||||
$CC $WARN -O2 -I"$RT" $SRC -lm -o "$TMP/perf"
|
||||
"$TMP/perf"
|
||||
|
||||
echo
|
||||
echo "### PASS 2: SAFETY (ASan/UBSan)"
|
||||
$CC $WARN -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer -I"$RT" $SRC -lm -o "$TMP/safe"
|
||||
ASAN_OPTIONS=${ASAN_OPTIONS:-detect_leaks=0} UBSAN_OPTIONS=halt_on_error=1 "$TMP/safe"
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
# M1 paged-store gate. Pure C (NOT elb/elc). Writes only under /tmp.
|
||||
set -e
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
SRC="$HERE/../../lang/runtime/engram_store.c"
|
||||
BIN="/tmp/test_store.$$"
|
||||
echo "compiling: gcc test_store.c engram_store.c"
|
||||
gcc -O2 -Wall -Wextra -std=c11 "$HERE/test_store.c" "$SRC" -o "$BIN"
|
||||
"$BIN"
|
||||
rc=$?
|
||||
rm -f "$BIN"
|
||||
rm -rf /tmp/engram-store-test-*
|
||||
exit $rc
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
# Build + RUN the VERIFIER-layer tests (engram_verify.c): closed-form constructed
|
||||
# cases for GROUNDING (anti-hallucination) and CONSISTENCY (polarity/negation
|
||||
# inversion + geometric contradiction), each composing the reasoning point-fit
|
||||
# (engram_reason.c) and the §5 geometry OPERATORS (engram_geometry.c). Pure C11
|
||||
# (stdlib + libm). Standalone — NOT folded through elc. Two passes:
|
||||
# 1. PERF — optimised (-O2, no sanitizer): the functional gate.
|
||||
# 2. SAFETY — ASan + UBSan on the same suite (memory-safety is size-independent).
|
||||
set -e
|
||||
HERE=$(cd "$(dirname "$0")" && pwd)
|
||||
RT="$HERE/../../lang/runtime"
|
||||
CC=${CC:-cc}
|
||||
SRC="$HERE/test_verify.c $RT/engram_verify.c $RT/engram_reason.c $RT/engram_geometry.c $RT/engram_store.c $RT/engram_vindex.c"
|
||||
WARN="-std=c11 -Wall -Wextra"
|
||||
TMP=$(mktemp -d)
|
||||
|
||||
echo "### PASS 1: PERF (optimised, un-sanitised) — functional gate"
|
||||
$CC $WARN -O2 -I"$RT" $SRC -lm -o "$TMP/perf"
|
||||
"$TMP/perf"
|
||||
|
||||
echo
|
||||
echo "### PASS 2: SAFETY (ASan/UBSan)"
|
||||
$CC $WARN -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer -I"$RT" $SRC -lm -o "$TMP/safe"
|
||||
ASAN_OPTIONS=${ASAN_OPTIONS:-detect_leaks=0} UBSAN_OPTIONS=halt_on_error=1 "$TMP/safe"
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
# Build + RUN the M8 HNSW vector-index tests. Pure C11 (gcc/cc), stdlib + libm
|
||||
# only. This is a standalone C module — NOT folded through elb/elc.
|
||||
#
|
||||
# Two passes:
|
||||
# 1. PERF — optimised (-O2, no sanitizer): the real recall@10 gate + speedup
|
||||
# numbers at full size (N=5000 recall, N=5000/20000 speedup).
|
||||
# 2. SAFETY — ASan + UBSan on the same suite at reduced size (VINDEX_QUICK=1);
|
||||
# memory-safety is size-independent, so this stays fast.
|
||||
set -e
|
||||
HERE=$(cd "$(dirname "$0")" && pwd)
|
||||
RT="$HERE/../../lang/runtime"
|
||||
CC=${CC:-cc}
|
||||
SRC="$HERE/test_vindex.c $RT/engram_vindex.c $RT/engram_store.c"
|
||||
WARN="-std=c11 -Wall -Wextra"
|
||||
TMP=$(mktemp -d)
|
||||
|
||||
echo "### PASS 1: PERF (optimised, un-sanitised) — recall gate + speedup"
|
||||
$CC $WARN -O2 -I"$RT" $SRC -lm -o "$TMP/perf"
|
||||
"$TMP/perf"
|
||||
|
||||
echo
|
||||
echo "### PASS 2: SAFETY (ASan/UBSan, reduced size)"
|
||||
$CC $WARN -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer -I"$RT" $SRC -lm -o "$TMP/safe"
|
||||
VINDEX_QUICK=1 ASAN_OPTIONS=${ASAN_OPTIONS:-detect_leaks=0} UBSAN_OPTIONS=halt_on_error=1 "$TMP/safe"
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# M2 WAL + checkpoint + recovery gate. Pure C (NOT elb/elc). Writes only under /tmp.
|
||||
# Recovery tests use ENGRAM_WAL_SYNC=always so every WAL record is durable at crash.
|
||||
set -e
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
SRC="$HERE/../../lang/runtime/engram_store.c"
|
||||
BIN="/tmp/test_wal_store.$$"
|
||||
echo "compiling: gcc test_wal_store.c engram_store.c"
|
||||
gcc -O2 -Wall -Wextra -std=c11 "$HERE/test_wal_store.c" "$SRC" -o "$BIN"
|
||||
ENGRAM_WAL_SYNC=always "$BIN"
|
||||
rc=$?
|
||||
rm -f "$BIN"
|
||||
rm -rf /tmp/engram-wal-test-*
|
||||
exit $rc
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
# WAL unit + integration + crash-fuzz gate. Throwaway HOME/dirs only.
|
||||
set -e
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
REL="$HERE/../../lang/runtime"
|
||||
cc -O2 -fbracket-depth=1024 -Wno-parentheses-equality -I"$REL" \
|
||||
"$HERE/test_wal.c" -lcurl -lpthread -o /tmp/test_wal
|
||||
HOME=/tmp/engram-throwaway-home /tmp/test_wal
|
||||
# Fail-loud data-dir check (must exit 1 with a FATAL line):
|
||||
cat > /tmp/test_failloud.c <<'C'
|
||||
#include "el_runtime.c"
|
||||
int main(void){ unsetenv("ENGRAM_DATA_DIR"); unsetenv("HOME");
|
||||
engram_resolve_data_dir(); printf("REACHED\n"); return 0; }
|
||||
C
|
||||
cc -O2 -fbracket-depth=1024 -Wno-parentheses-equality -I"$REL" /tmp/test_failloud.c -lcurl -lpthread -o /tmp/test_failloud
|
||||
if env -u HOME -u ENGRAM_DATA_DIR /tmp/test_failloud; then echo "FAIL: should have exited"; exit 1; else echo "[PASS] fail-loud exit on unresolvable HOME"; fi
|
||||
@@ -0,0 +1,496 @@
|
||||
/* test_bufpool.c — M4 gate for the demand-paging BUFFER POOL (engram_store.{c,h}).
|
||||
*
|
||||
* Pure C. Build: gcc -O2 test_bufpool.c ../../lang/runtime/engram_store.c -o t
|
||||
* Writes ONLY under a throwaway /tmp dir. Never touches ~/.neuron or live ports.
|
||||
*
|
||||
* Proves the M4 pool preserves every M1/M2 invariant when the pool is SMALLER
|
||||
* than the store (pages evict + re-fault): small-pool round-trip correctness,
|
||||
* LRU eviction policy (hot resident / cold evicted / no dirty stolen), pinned
|
||||
* residency (superblocks, index roots, explicit page + hot-layer pins), bounded
|
||||
* read-ahead, and crash safety (WAL replay + checkpoint-crash) under paging.
|
||||
*/
|
||||
#include "../../lang/runtime/engram_store.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
static int g_pass = 0, g_fail = 0;
|
||||
static void ok(const char* name, int cond){
|
||||
printf(" [%s] %s\n", cond ? "PASS" : "FAIL", name);
|
||||
if (cond) g_pass++; else g_fail++;
|
||||
}
|
||||
|
||||
static char g_dir[512];
|
||||
static void mk_dir(void){
|
||||
snprintf(g_dir, sizeof g_dir, "/tmp/engram-bufpool-test-%d", (int)getpid());
|
||||
mkdir(g_dir, 0700);
|
||||
}
|
||||
static void path_in(char* out, size_t cap, const char* name){
|
||||
snprintf(out, cap, "%s/%s", g_dir, name);
|
||||
}
|
||||
|
||||
/* ── deterministic generators (bit-exact regeneration for oracles) ─────────── */
|
||||
static uint64_t xs(uint64_t* s){ uint64_t x=*s; x^=x<<13; x^=x>>7; x^=x<<17; *s=x; return x; }
|
||||
static uint64_t node_seed(int i){ return 0x9E3779B97F4A7C15ULL ^ ((uint64_t)(i+1)*0xD1B54A32D192ED03ULL); }
|
||||
static uint64_t edge_seed(int i){ return 0xC2B2AE3D27D4EB4FULL ^ ((uint64_t)(i+1)*0x165667B19E3779F9ULL); }
|
||||
static char* rnd_str(uint64_t* st, size_t len){
|
||||
char* s = (char*)malloc(len + 1);
|
||||
for (size_t i=0;i<len;i++) s[i] = (char)(33 + (xs(st) % 94));
|
||||
s[len] = 0; return s;
|
||||
}
|
||||
|
||||
#define NODE_COUNT 5000
|
||||
#define EDGE_COUNT 20000
|
||||
#define EMB_DIM 768
|
||||
#define CK_NODES 300
|
||||
|
||||
static void noop_node_cb(const StoreNode* n, void* ctx){ (void)n; (void)ctx; }
|
||||
|
||||
static void gen_node(int i, StoreNode* n){
|
||||
memset(n, 0, sizeof *n);
|
||||
uint64_t st = node_seed(i);
|
||||
char id[32]; snprintf(id, sizeof id, "node-%d", i);
|
||||
n->id = strdup(id);
|
||||
size_t clen = (i % 500 == 0) ? (size_t)(17000 + (xs(&st) % 6000)) : (size_t)(xs(&st) % 300);
|
||||
n->content = rnd_str(&st, clen);
|
||||
n->node_type = rnd_str(&st, 4 + (xs(&st) % 8));
|
||||
n->label = (i % 2) ? rnd_str(&st, 3 + (xs(&st) % 10)) : NULL;
|
||||
n->tier = rnd_str(&st, 4 + (xs(&st) % 6));
|
||||
n->tags = rnd_str(&st, xs(&st) % 40);
|
||||
n->metadata = (i % 3) ? rnd_str(&st, xs(&st) % 60) : NULL;
|
||||
n->salience = (double)(xs(&st) % 1000000) / 997.0;
|
||||
n->importance = (double)(xs(&st) % 1000000) / 131.0;
|
||||
n->confidence = (double)(xs(&st) % 1000000) / 733.0;
|
||||
n->temporal_decay_rate = (double)(xs(&st) % 1000000) / 101.0;
|
||||
n->activation_count = (int64_t)(xs(&st) % 100000);
|
||||
n->last_activated = (int64_t)xs(&st);
|
||||
n->created_at = (int64_t)(1600000000000LL + i);
|
||||
n->updated_at = (int64_t)xs(&st);
|
||||
n->background_activation = (double)(xs(&st) % 1000000) / 17.0;
|
||||
n->working_memory_weight = (double)(xs(&st) % 1000000) / 29.0;
|
||||
n->suppression_count = (int32_t)(xs(&st) % 50);
|
||||
n->layer_id = (uint32_t)(xs(&st) % 5);
|
||||
for (int k=0;k<STORE_BLL_K;k++) n->access_ts[k] = (int64_t)xs(&st);
|
||||
n->access_head = (int32_t)(xs(&st) % STORE_BLL_K);
|
||||
n->access_filled = (int32_t)(xs(&st) % (STORE_BLL_K + 1));
|
||||
n->wm_anchor = (double)(xs(&st) % 1000000) / 3.0;
|
||||
n->emb = (float*)malloc(EMB_DIM * sizeof(float));
|
||||
for (int k=0;k<EMB_DIM;k++){ uint32_t u=(uint32_t)xs(&st); memcpy(&n->emb[k], &u, 4); }
|
||||
n->emb_dim = EMB_DIM;
|
||||
}
|
||||
static void gen_edge(int i, StoreEdge* e){
|
||||
memset(e, 0, sizeof *e);
|
||||
uint64_t st = edge_seed(i);
|
||||
char id[32], from[32], to[32];
|
||||
snprintf(id, sizeof id, "edge-%d", i);
|
||||
snprintf(from, sizeof from, "node-%d", (int)(xs(&st) % NODE_COUNT));
|
||||
snprintf(to, sizeof to, "node-%d", (int)(xs(&st) % NODE_COUNT));
|
||||
e->id = strdup(id); e->from_id = strdup(from); e->to_id = strdup(to);
|
||||
e->relation = rnd_str(&st, 3 + (xs(&st) % 12));
|
||||
e->metadata = (i % 4) ? rnd_str(&st, xs(&st) % 40) : NULL;
|
||||
e->weight = (double)(xs(&st) % 1000000) / 111.0;
|
||||
e->hebb = (double)(xs(&st) % 1000000) / 1000000.0;
|
||||
e->confidence = (double)(xs(&st) % 1000000) / 777.0;
|
||||
e->created_at = (int64_t)(1600000000000LL + i);
|
||||
e->updated_at = (int64_t)xs(&st);
|
||||
e->last_fired = (int64_t)xs(&st);
|
||||
e->inhibitory = (int32_t)(xs(&st) % 2);
|
||||
e->layer_id = (uint32_t)(xs(&st) % 5);
|
||||
}
|
||||
static int streq(const char* a, const char* b){
|
||||
if (!a && !b) return 1;
|
||||
if (!a || !b) return 0;
|
||||
return strcmp(a,b)==0;
|
||||
}
|
||||
static int cmp_node(const StoreNode* a, const StoreNode* b){
|
||||
if (!streq(a->id,b->id) || !streq(a->content,b->content) ||
|
||||
!streq(a->node_type,b->node_type) || !streq(a->label,b->label) ||
|
||||
!streq(a->tier,b->tier) || !streq(a->tags,b->tags) ||
|
||||
!streq(a->metadata,b->metadata)) return 0;
|
||||
if (a->salience!=b->salience || a->importance!=b->importance ||
|
||||
a->confidence!=b->confidence || a->temporal_decay_rate!=b->temporal_decay_rate ||
|
||||
a->activation_count!=b->activation_count || a->last_activated!=b->last_activated ||
|
||||
a->created_at!=b->created_at || a->updated_at!=b->updated_at ||
|
||||
a->background_activation!=b->background_activation ||
|
||||
a->working_memory_weight!=b->working_memory_weight ||
|
||||
a->suppression_count!=b->suppression_count || a->layer_id!=b->layer_id ||
|
||||
a->access_head!=b->access_head || a->access_filled!=b->access_filled ||
|
||||
a->wm_anchor!=b->wm_anchor || a->emb_dim!=b->emb_dim) return 0;
|
||||
for (int k=0;k<STORE_BLL_K;k++) if (a->access_ts[k]!=b->access_ts[k]) return 0;
|
||||
if ((a->emb==NULL) != (b->emb==NULL)) return 0;
|
||||
if (a->emb && memcmp(a->emb, b->emb, (size_t)a->emb_dim*4)!=0) return 0;
|
||||
return 1;
|
||||
}
|
||||
static int cmp_edge(const StoreEdge* a, const StoreEdge* b){
|
||||
if (!streq(a->id,b->id) || !streq(a->from_id,b->from_id) || !streq(a->to_id,b->to_id) ||
|
||||
!streq(a->relation,b->relation) || !streq(a->metadata,b->metadata)) return 0;
|
||||
if (a->weight!=b->weight || a->hebb!=b->hebb || a->confidence!=b->confidence ||
|
||||
a->created_at!=b->created_at || a->updated_at!=b->updated_at ||
|
||||
a->last_fired!=b->last_fired || a->inhibitory!=b->inhibitory ||
|
||||
a->layer_id!=b->layer_id) return 0;
|
||||
return 1;
|
||||
}
|
||||
static void free_node_fields(StoreNode* n){
|
||||
free(n->id); free(n->content); free(n->node_type); free(n->label);
|
||||
free(n->tier); free(n->tags); free(n->metadata); free(n->emb); free(n->unknown);
|
||||
}
|
||||
static void free_edge_fields(StoreEdge* e){
|
||||
free(e->id); free(e->from_id); free(e->to_id); free(e->relation); free(e->metadata); free(e->unknown);
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════════
|
||||
* TEST 1 — SMALL-POOL CORRECTNESS: full M1 workload (5k nodes / 20k edges) with
|
||||
* a frame budget FAR smaller than the store → constant eviction + re-fault, yet
|
||||
* every read is bit-exact and the pool stays bounded.
|
||||
* ════════════════════════════════════════════════════════════════════════════ */
|
||||
static void test_small_pool_roundtrip(void){
|
||||
printf("\n== 1) small-pool correctness: %d nodes + %d edges, cap=%d frames ==\n",
|
||||
NODE_COUNT, EDGE_COUNT, 32);
|
||||
char path[600]; path_in(path, sizeof path, "small.store");
|
||||
unlink(path);
|
||||
EngramPagedStore* s = store_create(path);
|
||||
ok("store_create", s != NULL);
|
||||
if (!s) return;
|
||||
store__set_pool_frames(s, 32); /* pool << store */
|
||||
|
||||
for (int i=0;i<NODE_COUNT;i++){
|
||||
StoreNode n; gen_node(i,&n);
|
||||
if (store_put_node(s,&n)!=0){ ok("put_node", 0); free_node_fields(&n); store_close(s); return; }
|
||||
free_node_fields(&n);
|
||||
if ((i%500)==499) store_sync(s); /* checkpoint: dirty→clean so frames evictable */
|
||||
}
|
||||
for (int i=0;i<EDGE_COUNT;i++){
|
||||
StoreEdge e; gen_edge(i,&e);
|
||||
if (store_put_edge(s,&e)!=0){ ok("put_edge", 0); free_edge_fields(&e); store_close(s); return; }
|
||||
free_edge_fields(&e);
|
||||
if ((i%1000)==999) store_sync(s);
|
||||
}
|
||||
store_sync(s);
|
||||
|
||||
StorePoolStats st; store_pool_stats(s, &st);
|
||||
printf(" pages=%llu pool: cap=%zu resident=%zu pinned=%zu dirty=%zu evictions=%llu\n",
|
||||
(unsigned long long)store_page_count(s), st.cap, st.resident, st.pinned,
|
||||
st.dirty, (unsigned long long)st.evictions);
|
||||
ok("eviction actually fired (store exceeded the pool)", st.evictions > 0);
|
||||
ok("pool stayed bounded (resident <= cap)", st.resident <= st.cap);
|
||||
ok("no dirty frames after checkpoint", st.dirty == 0);
|
||||
|
||||
/* read back EVERY node bit-exact despite constant eviction/re-fault */
|
||||
int bad = 0;
|
||||
for (int i=0;i<NODE_COUNT;i++){
|
||||
StoreNode want; gen_node(i,&want);
|
||||
StoreNode got; int hit = store_get_node(s, want.id, &got);
|
||||
if (hit!=1 || !cmp_node(&want,&got)) bad++;
|
||||
if (hit==1) store_node_free(&got);
|
||||
free_node_fields(&want);
|
||||
}
|
||||
ok("all 5000 nodes bit-exact under eviction", bad==0);
|
||||
|
||||
/* sample 4000 edges bit-exact */
|
||||
int ebad = 0;
|
||||
for (int i=0;i<EDGE_COUNT;i+=5){
|
||||
StoreEdge want; gen_edge(i,&want);
|
||||
StoreEdge got; int hit = store_get_edge(s, want.id, &got);
|
||||
if (hit!=1 || !cmp_edge(&want,&got)) ebad++;
|
||||
if (hit==1) store_edge_free(&got);
|
||||
free_edge_fields(&want);
|
||||
}
|
||||
ok("sampled 4000 edges bit-exact under eviction", ebad==0);
|
||||
ok("store_check crc clean under paging", store_check(s, STORE_CHECK_CRC)==0);
|
||||
|
||||
store_pool_stats(s, &st);
|
||||
printf(" after reads: resident=%zu (<= cap=%zu) hits=%llu misses=%llu evictions=%llu\n",
|
||||
st.resident, st.cap, (unsigned long long)st.hits,
|
||||
(unsigned long long)st.misses, (unsigned long long)st.evictions);
|
||||
ok("still bounded after full read-back", st.resident <= st.cap);
|
||||
store_close(s);
|
||||
unlink(path);
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════════
|
||||
* TEST 2 — EVICTION POLICY: a repeatedly-touched HOT set stays resident (0 extra
|
||||
* faults) while a streaming COLD set is evicted; and a dirty-heavy write burst
|
||||
* proves dirty pages are NEVER stolen before a checkpoint (no-steal).
|
||||
* ════════════════════════════════════════════════════════════════════════════ */
|
||||
static void test_eviction_policy(void){
|
||||
printf("\n== 2) eviction policy: hot resident, cold evicted, no dirty stolen ==\n");
|
||||
char path[600]; path_in(path, sizeof path, "evict.store");
|
||||
unlink(path);
|
||||
|
||||
/* ---- part A: hot vs cold ---- */
|
||||
EngramPagedStore* s = store_create(path);
|
||||
if (!s){ ok("store_create", 0); return; }
|
||||
const int N = 1500;
|
||||
for (int i=0;i<N;i++){ StoreNode n; gen_node(i,&n); store_put_node(s,&n); free_node_fields(&n);
|
||||
if ((i%400)==399) store_sync(s); }
|
||||
store_sync(s);
|
||||
store__set_pool_frames(s, 64);
|
||||
|
||||
const int HOT = 8;
|
||||
/* warm the hot set */
|
||||
for (int h=0;h<HOT;h++){ char id[32]; snprintf(id,sizeof id,"node-%d",h);
|
||||
StoreNode g; if (store_get_node(s,id,&g)==1) store_node_free(&g); }
|
||||
|
||||
StorePoolStats a,b;
|
||||
uint64_t hot_faults = 0, cold_faults = 0;
|
||||
int cold = 200; /* streaming cold ids well outside hot set */
|
||||
for (int r=0;r<150;r++){
|
||||
for (int h=0;h<HOT;h++){
|
||||
char id[32]; snprintf(id,sizeof id,"node-%d",h);
|
||||
store_pool_stats(s,&a);
|
||||
StoreNode g; if (store_get_node(s,id,&g)==1) store_node_free(&g);
|
||||
store_pool_stats(s,&b);
|
||||
hot_faults += (b.misses - a.misses);
|
||||
}
|
||||
for (int c=0;c<3;c++){
|
||||
char id[32]; snprintf(id,sizeof id,"node-%d",cold++);
|
||||
if (cold>=N) cold=200;
|
||||
store_pool_stats(s,&a);
|
||||
StoreNode g; if (store_get_node(s,id,&g)==1) store_node_free(&g);
|
||||
store_pool_stats(s,&b);
|
||||
cold_faults += (b.misses - a.misses);
|
||||
}
|
||||
}
|
||||
printf(" hot re-get faults (post-warm)=%llu cold stream faults=%llu\n",
|
||||
(unsigned long long)hot_faults, (unsigned long long)cold_faults);
|
||||
ok("HOT pages stay resident (0 faults on re-access)", hot_faults == 0);
|
||||
ok("COLD pages get evicted + re-faulted", cold_faults > 0);
|
||||
store_pool_stats(s,&b);
|
||||
double hr = (double)b.hits / (double)(b.hits + b.misses);
|
||||
printf(" overall hit-rate = %.3f (hits=%llu misses=%llu)\n",
|
||||
hr, (unsigned long long)b.hits, (unsigned long long)b.misses);
|
||||
ok("hit-rate is sane (> 0.5)", hr > 0.5);
|
||||
store_close(s);
|
||||
unlink(path);
|
||||
|
||||
/* ---- part B: no-steal (dirty pages never evicted before checkpoint) ---- */
|
||||
EngramPagedStore* s2 = store_create(path);
|
||||
if (!s2){ ok("store_create(2)", 0); return; }
|
||||
store__set_pool_frames(s2, 8); /* tiny budget */
|
||||
for (int i=0;i<1200;i++){ StoreNode n; gen_node(i,&n); store_put_node(s2,&n); free_node_fields(&n); }
|
||||
/* NO sync: every mutated page is dirty and, by no-steal, unevictable */
|
||||
StorePoolStats d; store_pool_stats(s2,&d);
|
||||
printf(" tiny cap=%zu, unsynced burst: resident=%zu dirty=%zu evictions=%llu\n",
|
||||
d.cap, d.resident, d.dirty, (unsigned long long)d.evictions);
|
||||
ok("dirty pages pinned in RAM beyond budget (no-steal)", d.dirty > d.cap && d.resident > d.cap);
|
||||
/* a just-written node is served correctly from its dirty in-RAM page */
|
||||
{ StoreNode want; gen_node(777,&want); StoreNode got; int hit=store_get_node(s2,want.id,&got);
|
||||
ok("read served correctly from dirty (un-flushed) page", hit==1 && cmp_node(&want,&got));
|
||||
if (hit==1) store_node_free(&got); free_node_fields(&want); }
|
||||
store_sync(s2); /* checkpoint → dirty become clean/evictable */
|
||||
store_pool_stats(s2,&d);
|
||||
ok("checkpoint cleared all dirty frames", d.dirty == 0);
|
||||
/* durability across reopen after the no-steal burst */
|
||||
store_close(s2);
|
||||
EngramPagedStore* s3 = store_open(path);
|
||||
store__set_pool_frames(s3, 8);
|
||||
int miss=0; for (int i=0;i<1200;i++){ StoreNode want; gen_node(i,&want);
|
||||
StoreNode got; int hit=store_get_node(s3,want.id,&got);
|
||||
if (hit!=1 || !cmp_node(&want,&got)) miss++;
|
||||
if (hit==1) store_node_free(&got); free_node_fields(&want); }
|
||||
ok("all 1200 survive reopen, bit-exact, tiny pool", miss==0);
|
||||
store_close(s3);
|
||||
unlink(path);
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════════
|
||||
* TEST 3 — PINNED RESIDENCY: superblocks + index roots never evicted under heavy
|
||||
* thrash; an explicitly pinned page stays until unpinned; a pinned hot layer's
|
||||
* pages stay resident and are released on unpin.
|
||||
* ════════════════════════════════════════════════════════════════════════════ */
|
||||
static void test_pinning(void){
|
||||
printf("\n== 3) pinned residency: superblocks / index roots / page / layer ==\n");
|
||||
char path[600]; path_in(path, sizeof path, "pin.store");
|
||||
unlink(path);
|
||||
EngramPagedStore* s = store_create(path);
|
||||
if (!s){ ok("store_create", 0); return; }
|
||||
const int N = 1500;
|
||||
for (int i=0;i<N;i++){ StoreNode n; gen_node(i,&n); store_put_node(s,&n); free_node_fields(&n);
|
||||
if ((i%400)==399) store_sync(s); }
|
||||
store_sync(s);
|
||||
store_close(s);
|
||||
|
||||
s = store_open(path); /* reopen: SBs + roots auto-pinned */
|
||||
store__set_pool_frames(s, 24);
|
||||
|
||||
uint64_t P = store_page_count(s) / 2; /* an arbitrary interior page to pin */
|
||||
store_pin_page(s, P);
|
||||
|
||||
/* thrash: stream a large cold working set to force heavy eviction */
|
||||
for (int pass=0; pass<3; pass++)
|
||||
for (int i=0;i<N;i++){ char id[32]; snprintf(id,sizeof id,"node-%d",i);
|
||||
StoreNode g; if (store_get_node(s,id,&g)==1) store_node_free(&g); }
|
||||
|
||||
ok("superblock page 0 never evicted", store_pool_resident(s,0)==1);
|
||||
ok("superblock mirror page 1 never evicted", store_pool_resident(s,1)==1);
|
||||
ok("explicitly pinned page stayed resident under thrash", store_pool_resident(s,P)==1);
|
||||
|
||||
StorePoolStats st; store_pool_stats(s,&st);
|
||||
printf(" after thrash: resident=%zu pinned=%zu evictions=%llu\n",
|
||||
st.resident, st.pinned, (unsigned long long)st.evictions);
|
||||
ok("structural + explicit pins counted (>=4: 2 SB + 2 roots)", st.pinned >= 4);
|
||||
|
||||
/* unpin the page → it becomes evictable and is dropped under further thrash */
|
||||
store_unpin_page(s, P);
|
||||
for (int i=0;i<N;i++){ char id[32]; snprintf(id,sizeof id,"node-%d",i);
|
||||
StoreNode g; if (store_get_node(s,id,&g)==1) store_node_free(&g); }
|
||||
ok("unpinned page becomes evictable (dropped)", store_pool_resident(s,P)==0);
|
||||
|
||||
/* hot-layer pin: layer 3 is used by ~1/5 of the nodes */
|
||||
int npin = store_pin_layer(s, 3);
|
||||
printf(" store_pin_layer(3) pinned %d page(s)\n", npin);
|
||||
ok("pin_layer pinned a non-empty page set", npin > 0);
|
||||
store_pool_stats(s,&st);
|
||||
size_t pinned_with_layer = st.pinned;
|
||||
for (int pass=0; pass<3; pass++)
|
||||
for (int i=0;i<N;i++){ char id[32]; snprintf(id,sizeof id,"node-%d",i);
|
||||
StoreNode g; if (store_get_node(s,id,&g)==1) store_node_free(&g); }
|
||||
store_pool_stats(s,&st);
|
||||
ok("hot-layer pages stay resident under thrash", st.pinned >= pinned_with_layer);
|
||||
ok("layer pin holds >= npin extra frames", st.pinned >= (size_t)npin + 4);
|
||||
|
||||
store_unpin_layer(s, 3);
|
||||
store_pool_stats(s,&st);
|
||||
size_t after_unpin_max = st.pinned;
|
||||
for (int i=0;i<N;i++){ char id[32]; snprintf(id,sizeof id,"node-%d",i);
|
||||
StoreNode g; if (store_get_node(s,id,&g)==1) store_node_free(&g); }
|
||||
store_pool_stats(s,&st);
|
||||
printf(" pinned frames: with-layer=%zu after-unpin=%zu\n", pinned_with_layer, st.pinned);
|
||||
ok("unpin_layer released the layer's pins", st.pinned < pinned_with_layer && after_unpin_max <= pinned_with_layer);
|
||||
|
||||
store_close(s);
|
||||
unlink(path);
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════════
|
||||
* TEST 4 — PREFETCH: a sequential scan faults far fewer times with read-ahead on
|
||||
* than off (each cold cache; identical store).
|
||||
* ════════════════════════════════════════════════════════════════════════════ */
|
||||
static void test_prefetch(void){
|
||||
printf("\n== 4) prefetch: sequential scan faults fewer with read-ahead ==\n");
|
||||
char path[600]; path_in(path, sizeof path, "prefetch.store");
|
||||
unlink(path);
|
||||
EngramPagedStore* s = store_create(path);
|
||||
if (!s){ ok("store_create", 0); return; }
|
||||
for (int i=0;i<2000;i++){ StoreNode n; gen_node(i,&n); store_put_node(s,&n); free_node_fields(&n);
|
||||
if ((i%400)==399) store_sync(s); }
|
||||
store_sync(s);
|
||||
store_close(s);
|
||||
|
||||
/* prefetch OFF — cold cache */
|
||||
EngramPagedStore* a = store_open(path);
|
||||
store__set_pool_frames(a, 0); /* unlimited: isolate prefetch, no eviction */
|
||||
store__set_prefetch(a, 0);
|
||||
StorePoolStats o0, o1; store_pool_stats(a,&o0);
|
||||
int na = store_scan_nodes(a, noop_node_cb, NULL); /* walk + fault every page */
|
||||
(void)na;
|
||||
store_pool_stats(a,&o1);
|
||||
uint64_t faults_off = o1.misses - o0.misses;
|
||||
store_close(a);
|
||||
|
||||
/* prefetch ON — cold cache (fresh open) */
|
||||
EngramPagedStore* b = store_open(path);
|
||||
store__set_pool_frames(b, 0);
|
||||
store__set_prefetch(b, 16);
|
||||
StorePoolStats p0, p1; store_pool_stats(b,&p0);
|
||||
int nb = store_scan_nodes(b, noop_node_cb, NULL);
|
||||
(void)nb;
|
||||
store_pool_stats(b,&p1);
|
||||
uint64_t faults_on = p1.misses - p0.misses;
|
||||
uint64_t pref_reads = p1.prefetch_reads - p0.prefetch_reads;
|
||||
store_close(b);
|
||||
|
||||
printf(" scan demand-faults: prefetch OFF=%llu ON=%llu (read-ahead brought in %llu pages)\n",
|
||||
(unsigned long long)faults_off, (unsigned long long)faults_on,
|
||||
(unsigned long long)pref_reads);
|
||||
ok("prefetch reduced demand faults", faults_on < faults_off);
|
||||
ok("read-ahead actually ran", pref_reads > 0);
|
||||
unlink(path);
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════════
|
||||
* TEST 5 — CRASH SAFETY UNDER PAGING: WAL replay and checkpoint-crash recovery
|
||||
* with a tiny pool (pages evict + re-fault during replay).
|
||||
* ════════════════════════════════════════════════════════════════════════════ */
|
||||
static void test_crash_under_paging(void){
|
||||
printf("\n== 5) crash safety under a tiny pool (ENGRAM_POOL_FRAMES=16) ==\n");
|
||||
setenv("ENGRAM_POOL_FRAMES", "16", 1); /* every engram_open() below is paged */
|
||||
setenv("ENGRAM_WAL_SYNC", "always", 1);
|
||||
|
||||
/* ---- 5a: power-loss → WAL replay ---- */
|
||||
char dir[600]; path_in(dir, sizeof dir, "crash_wal"); mkdir(dir, 0700);
|
||||
EngramPagedStore* s = engram_open(dir);
|
||||
if (!s){ ok("engram_open", 0); return; }
|
||||
const int M = 400;
|
||||
for (int i=0;i<M;i++){ StoreNode n; gen_node(i,&n); store_put_node(s,&n); free_node_fields(&n); }
|
||||
store__crash(s); /* abandon RAM (dirty pages lost); WAL fsync'd */
|
||||
s = engram_open(dir); /* replay WAL under 16-frame pool */
|
||||
ok("reopened after crash (WAL replay, tiny pool)", s!=NULL);
|
||||
int bad=0; for (int i=0;i<M;i++){ StoreNode want; gen_node(i,&want);
|
||||
StoreNode got; int hit=store_get_node(s,want.id,&got);
|
||||
if (hit!=1 || !cmp_node(&want,&got)) bad++;
|
||||
if (hit==1) store_node_free(&got); free_node_fields(&want); }
|
||||
ok("all 400 nodes recovered bit-exact via WAL replay under paging", bad==0);
|
||||
ok("store_check crc clean post-recovery", store_check(s, STORE_CHECK_CRC)==0);
|
||||
engram_close(s);
|
||||
|
||||
/* ---- 5b: checkpoint-crash at each phase ---- */
|
||||
for (int phase=0; phase<=4; phase++){
|
||||
char cdir[620]; snprintf(cdir, sizeof cdir, "%s/ck%d", g_dir, phase); mkdir(cdir,0700);
|
||||
EngramPagedStore* c = engram_open(cdir);
|
||||
for (int i=0;i<CK_NODES;i++){ StoreNode n; gen_node(i,&n); store_put_node(c,&n); free_node_fields(&n); }
|
||||
store__checkpoint_crashat(c, phase); /* crash mid-checkpoint (frees c) */
|
||||
EngramPagedStore* r = engram_open(cdir); /* heal + replay under tiny pool */
|
||||
int miss=0; for (int i=0;i<CK_NODES;i++){ StoreNode want; gen_node(i,&want);
|
||||
StoreNode got; int hit=store_get_node(r,want.id,&got);
|
||||
if (hit!=1 || !cmp_node(&want,&got)) miss++;
|
||||
if (hit==1) store_node_free(&got); free_node_fields(&want); }
|
||||
char nm[64]; snprintf(nm,sizeof nm,"checkpoint-crash phase %d: all recovered (paged)", phase);
|
||||
ok(nm, miss==0);
|
||||
engram_close(r);
|
||||
}
|
||||
unsetenv("ENGRAM_POOL_FRAMES");
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════════
|
||||
* TEST 6 — DEFAULT POOL == PHASE 1: with the default (large) budget, no eviction
|
||||
* ever fires; the whole store is resident, exactly the pre-M4 behaviour.
|
||||
* ════════════════════════════════════════════════════════════════════════════ */
|
||||
static void test_default_is_phase1(void){
|
||||
printf("\n== 6) default (large) pool == Phase-1 resident (no eviction) ==\n");
|
||||
char path[600]; path_in(path, sizeof path, "default.store");
|
||||
unlink(path);
|
||||
EngramPagedStore* s = store_create(path); /* default cap, no override */
|
||||
if (!s){ ok("store_create", 0); return; }
|
||||
for (int i=0;i<1500;i++){ StoreNode n; gen_node(i,&n); store_put_node(s,&n); free_node_fields(&n); }
|
||||
store_sync(s);
|
||||
for (int i=0;i<1500;i++){ char id[32]; snprintf(id,sizeof id,"node-%d",i);
|
||||
StoreNode g; if (store_get_node(s,id,&g)==1) store_node_free(&g); }
|
||||
StorePoolStats st; store_pool_stats(s,&st);
|
||||
printf(" cap=%zu resident=%zu evictions=%llu (pages=%llu)\n",
|
||||
st.cap, st.resident, (unsigned long long)st.evictions,
|
||||
(unsigned long long)store_page_count(s));
|
||||
ok("default budget is large", st.cap >= (size_t)(1u<<20));
|
||||
ok("no eviction ever fired at default budget", st.evictions == 0);
|
||||
ok("whole store resident (every page cached)", st.resident == store_page_count(s));
|
||||
store_close(s);
|
||||
unlink(path);
|
||||
}
|
||||
|
||||
int main(void){
|
||||
mk_dir();
|
||||
printf("engram M4 buffer-pool gate — dir=%s\n", g_dir);
|
||||
test_small_pool_roundtrip();
|
||||
test_eviction_policy();
|
||||
test_pinning();
|
||||
test_prefetch();
|
||||
test_crash_under_paging();
|
||||
test_default_is_phase1();
|
||||
printf("\n================ %d passed, %d failed ================\n", g_pass, g_fail);
|
||||
return g_fail ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,421 @@
|
||||
/* test_compaction.c — M5 gate: ONLINE COMPACTION + background checkpointer.
|
||||
*
|
||||
* Pure C. Build: gcc -O2 test_compaction.c ../../lang/runtime/engram_store.c -o t
|
||||
* Writes ONLY under a throwaway /tmp dir. Never touches ~/.neuron or live ports.
|
||||
*
|
||||
* Proves:
|
||||
* 1) RECLAIM — tombstone/forget a large fraction of nodes + re-put many edges
|
||||
* (dead versions) + orphan large-record overflow chains, then compact:
|
||||
* page count AND file size drop, yet EVERY live record survives bit-exact and
|
||||
* the id + adjacency indexes resolve correctly at the relocated positions.
|
||||
* 2) CRASH-DURING-COMPACTION — kill at phases 0/1/2; recovery is always a
|
||||
* consistent store (crc clean, every live record intact), never corrupt.
|
||||
* 3) BACKGROUND CHECKPOINTER — a low ops / WAL-bytes threshold fires a checkpoint
|
||||
* automatically on the write path; the WAL prefix is reclaimed; recovery works.
|
||||
* 4) POOL COOPERATION — compaction under a tiny ENGRAM_POOL_FRAMES stays correct
|
||||
* with no stale frame surviving for a relocated page.
|
||||
*/
|
||||
#include "../../lang/runtime/engram_store.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
static int g_pass = 0, g_fail = 0;
|
||||
static void ok(const char* name, int cond){
|
||||
printf(" [%s] %s\n", cond ? "PASS" : "FAIL", name);
|
||||
if (cond) g_pass++; else g_fail++;
|
||||
}
|
||||
|
||||
static char g_dir[512];
|
||||
static int g_dseq = 0;
|
||||
static void mk_dir(void){
|
||||
snprintf(g_dir, sizeof g_dir, "/tmp/engram-compact-test-%d-%d", (int)getpid(), g_dseq++);
|
||||
mkdir(g_dir, 0700);
|
||||
}
|
||||
static void egm_path(char* out, size_t cap){ snprintf(out, cap, "%s/neuron.egm", g_dir); }
|
||||
static void wal_path(char* out, size_t cap){ snprintf(out, cap, "%s/neuron.wal", g_dir); }
|
||||
static long file_size(const char* p){ struct stat st; return stat(p,&st)==0 ? (long)st.st_size : -1; }
|
||||
|
||||
/* ── deterministic generators (bit-exact regeneration for oracles) ─────────── */
|
||||
static uint64_t xs(uint64_t* s){ uint64_t x=*s; x^=x<<13; x^=x>>7; x^=x<<17; *s=x; return x; }
|
||||
static uint64_t node_seed(int i){ return 0x9E3779B97F4A7C15ULL ^ ((uint64_t)(i+1)*0xD1B54A32D192ED03ULL); }
|
||||
static uint64_t edge_seed(int i){ return 0xC2B2AE3D27D4EB4FULL ^ ((uint64_t)(i+1)*0x165667B19E3779F9ULL); }
|
||||
static char* rnd_str(uint64_t* st, size_t len){
|
||||
char* s = (char*)malloc(len + 1);
|
||||
for (size_t i=0;i<len;i++) s[i] = (char)(33 + (xs(st) % 94));
|
||||
s[len] = 0; return s;
|
||||
}
|
||||
|
||||
#define N_NODES 1500
|
||||
#define N_DEAD 1200 /* forget node-0 .. node-1199 (1200 dead / 300 live) */
|
||||
#define N_EDGES 3000
|
||||
#define EDGE_REPUT 2000 /* re-put edge-0 .. edge-1999 to version 3 */
|
||||
#define EMB_DIM 96
|
||||
|
||||
static int node_is_live(int i){ return i >= N_DEAD; }
|
||||
static int edge_live_version(int i){ return (i < EDGE_REPUT) ? 3 : 0; }
|
||||
|
||||
static void gen_node(int i, StoreNode* n){
|
||||
memset(n, 0, sizeof *n);
|
||||
uint64_t st = node_seed(i);
|
||||
char id[32]; snprintf(id, sizeof id, "node-%d", i);
|
||||
n->id = strdup(id);
|
||||
/* every 7th record is large → its own overflow chain (orphaned when it dies) */
|
||||
size_t clen = (i % 7 == 0) ? (size_t)(18000 + (xs(&st) % 4000)) : (size_t)(xs(&st) % 200);
|
||||
n->content = rnd_str(&st, clen);
|
||||
n->node_type = rnd_str(&st, 4 + (xs(&st) % 8));
|
||||
n->label = (i % 2) ? rnd_str(&st, 3 + (xs(&st) % 10)) : NULL;
|
||||
n->tier = rnd_str(&st, 4 + (xs(&st) % 6));
|
||||
n->tags = rnd_str(&st, xs(&st) % 40);
|
||||
n->metadata = (i % 3) ? rnd_str(&st, xs(&st) % 60) : NULL;
|
||||
n->salience = (double)(xs(&st) % 1000000) / 997.0;
|
||||
n->importance = (double)(xs(&st) % 1000000) / 131.0;
|
||||
n->confidence = (double)(xs(&st) % 1000000) / 733.0;
|
||||
n->temporal_decay_rate = (double)(xs(&st) % 1000000) / 101.0;
|
||||
n->activation_count = (int64_t)(xs(&st) % 100000);
|
||||
n->last_activated = (int64_t)xs(&st);
|
||||
n->created_at = (int64_t)(1600000000000LL + i);
|
||||
n->updated_at = (int64_t)xs(&st);
|
||||
n->background_activation = (double)(xs(&st) % 1000000) / 17.0;
|
||||
n->working_memory_weight = (double)(xs(&st) % 1000000) / 29.0;
|
||||
n->suppression_count = (int32_t)(xs(&st) % 50);
|
||||
n->layer_id = (uint32_t)(xs(&st) % 5);
|
||||
for (int k=0;k<STORE_BLL_K;k++) n->access_ts[k] = (int64_t)xs(&st);
|
||||
n->access_head = (int32_t)(xs(&st) % STORE_BLL_K);
|
||||
n->access_filled = (int32_t)(xs(&st) % (STORE_BLL_K + 1));
|
||||
n->wm_anchor = (double)(xs(&st) % 1000000) / 3.0;
|
||||
n->emb = (float*)malloc(EMB_DIM * sizeof(float));
|
||||
for (int k=0;k<EMB_DIM;k++){ uint32_t u=(uint32_t)xs(&st); memcpy(&n->emb[k], &u, 4); }
|
||||
n->emb_dim = EMB_DIM;
|
||||
}
|
||||
/* version alters weight/hebb/last_fired so a re-put is a distinct payload. */
|
||||
static void gen_edge(int i, int version, StoreEdge* e){
|
||||
memset(e, 0, sizeof *e);
|
||||
uint64_t st = edge_seed(i);
|
||||
char id[32], from[32], to[32];
|
||||
snprintf(id, sizeof id, "edge-%d", i);
|
||||
/* connect live nodes so adjacency queries on live nodes are meaningful */
|
||||
snprintf(from, sizeof from, "node-%d", N_DEAD + (int)(xs(&st) % (N_NODES - N_DEAD)));
|
||||
snprintf(to, sizeof to, "node-%d", N_DEAD + (int)(xs(&st) % (N_NODES - N_DEAD)));
|
||||
e->id = strdup(id); e->from_id = strdup(from); e->to_id = strdup(to);
|
||||
e->relation = rnd_str(&st, 3 + (xs(&st) % 12));
|
||||
e->metadata = (i % 4) ? rnd_str(&st, xs(&st) % 40) : NULL;
|
||||
e->weight = (double)(xs(&st) % 1000000) / 7.0 + version * 100.0;
|
||||
e->hebb = (double)(xs(&st) % 1000000) / 13.0 + version * 3.0;
|
||||
e->confidence = (double)(xs(&st) % 1000000) / 5.0;
|
||||
e->created_at = (int64_t)(1600000000000LL + i);
|
||||
e->updated_at = (int64_t)xs(&st) + version;
|
||||
e->last_fired = (int64_t)xs(&st) + version * 1000;
|
||||
e->inhibitory = (int32_t)(xs(&st) % 2);
|
||||
e->layer_id = (uint32_t)(xs(&st) % 5);
|
||||
}
|
||||
|
||||
static int streq(const char* a, const char* b){
|
||||
if (!a && !b) return 1; if (!a || !b) return 0; return strcmp(a,b)==0;
|
||||
}
|
||||
static int cmp_node(const StoreNode* a, const StoreNode* b){
|
||||
if (!streq(a->id,b->id) || !streq(a->content,b->content) ||
|
||||
!streq(a->node_type,b->node_type) || !streq(a->label,b->label) ||
|
||||
!streq(a->tier,b->tier) || !streq(a->tags,b->tags) ||
|
||||
!streq(a->metadata,b->metadata)) return 0;
|
||||
if (a->salience!=b->salience || a->importance!=b->importance ||
|
||||
a->confidence!=b->confidence || a->temporal_decay_rate!=b->temporal_decay_rate ||
|
||||
a->activation_count!=b->activation_count || a->last_activated!=b->last_activated ||
|
||||
a->created_at!=b->created_at || a->updated_at!=b->updated_at ||
|
||||
a->background_activation!=b->background_activation ||
|
||||
a->working_memory_weight!=b->working_memory_weight ||
|
||||
a->suppression_count!=b->suppression_count || a->layer_id!=b->layer_id ||
|
||||
a->access_head!=b->access_head || a->access_filled!=b->access_filled ||
|
||||
a->wm_anchor!=b->wm_anchor || a->emb_dim!=b->emb_dim) return 0;
|
||||
for (int k=0;k<STORE_BLL_K;k++) if (a->access_ts[k]!=b->access_ts[k]) return 0;
|
||||
if ((a->emb==NULL) != (b->emb==NULL)) return 0;
|
||||
if (a->emb && memcmp(a->emb, b->emb, (size_t)a->emb_dim*4)!=0) return 0;
|
||||
return 1;
|
||||
}
|
||||
static int cmp_edge(const StoreEdge* a, const StoreEdge* b){
|
||||
if (!streq(a->id,b->id) || !streq(a->from_id,b->from_id) || !streq(a->to_id,b->to_id) ||
|
||||
!streq(a->relation,b->relation) || !streq(a->metadata,b->metadata)) return 0;
|
||||
if (a->weight!=b->weight || a->hebb!=b->hebb || a->confidence!=b->confidence ||
|
||||
a->created_at!=b->created_at || a->updated_at!=b->updated_at ||
|
||||
a->last_fired!=b->last_fired || a->inhibitory!=b->inhibitory ||
|
||||
a->layer_id!=b->layer_id) return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Populate a durable store with dead space: all nodes/edges, then forget the first
|
||||
* N_DEAD nodes and re-put the first EDGE_REPUT edges three times. */
|
||||
static void populate_with_dead_space(EngramPagedStore* s){
|
||||
for (int i=0;i<N_NODES;i++){ StoreNode n; gen_node(i,&n); store_put_node(s,&n); store_node_free(&n); }
|
||||
for (int i=0;i<N_EDGES;i++){ StoreEdge e; gen_edge(i,0,&e); store_put_edge(s,&e); store_edge_free(&e); }
|
||||
/* re-put (in-place field mutation) → prior versions become dead records */
|
||||
for (int v=1; v<=3; v++)
|
||||
for (int i=0;i<EDGE_REPUT;i++){ StoreEdge e; gen_edge(i,v,&e); store_put_edge(s,&e); store_edge_free(&e); }
|
||||
/* forget the cold nodes (tombstone; their large overflow chains orphan) */
|
||||
for (int i=0;i<N_DEAD;i++){ char id[32]; snprintf(id,sizeof id,"node-%d",i); store_forget(s,id); }
|
||||
}
|
||||
|
||||
/* Assert every live node/edge is present + bit-exact via point reads. */
|
||||
static int verify_live_set(EngramPagedStore* s){
|
||||
int bad = 0;
|
||||
for (int i=0;i<N_NODES;i++){
|
||||
char id[32]; snprintf(id,sizeof id,"node-%d",i);
|
||||
StoreNode got; int hit = store_get_node(s, id, &got);
|
||||
if (node_is_live(i)){
|
||||
StoreNode want; gen_node(i,&want);
|
||||
if (hit!=1 || !cmp_node(&want,&got)) bad++;
|
||||
if (hit==1) store_node_free(&got);
|
||||
store_node_free(&want);
|
||||
} else {
|
||||
if (hit!=0) bad++; /* forgotten → must be absent */
|
||||
if (hit==1) store_node_free(&got);
|
||||
}
|
||||
}
|
||||
for (int i=0;i<N_EDGES;i++){
|
||||
char id[32]; snprintf(id,sizeof id,"edge-%d",i);
|
||||
StoreEdge got; int hit = store_get_edge(s, id, &got);
|
||||
StoreEdge want; gen_edge(i, edge_live_version(i), &want);
|
||||
if (hit!=1 || !cmp_edge(&want,&got)) bad++;
|
||||
if (hit==1) store_edge_free(&got);
|
||||
store_edge_free(&want);
|
||||
}
|
||||
return bad;
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════════
|
||||
* TEST 1 — RECLAIM: dead space is reclaimed; live records + indexes survive.
|
||||
* ════════════════════════════════════════════════════════════════════════════ */
|
||||
static void test_reclaim(void){
|
||||
printf("\n== 1) reclaim: forget %d nodes + re-put %d edges x3, then compact ==\n",
|
||||
N_DEAD, EDGE_REPUT);
|
||||
mk_dir();
|
||||
char egm[600]; egm_path(egm, sizeof egm);
|
||||
EngramPagedStore* s = engram_open(g_dir);
|
||||
ok("engram_open", s != NULL);
|
||||
if (!s) return;
|
||||
|
||||
populate_with_dead_space(s);
|
||||
engram_checkpoint(s); /* flush so file size reflects state */
|
||||
|
||||
uint64_t pc_before = store_page_count(s);
|
||||
uint64_t free_before = store_free_page_count(s);
|
||||
long sz_before = file_size(egm);
|
||||
printf(" BEFORE: page_count=%llu free_pages=%llu file=%ld bytes (live records intact?)\n",
|
||||
(unsigned long long)pc_before, (unsigned long long)free_before, sz_before);
|
||||
ok("pre-compaction live set intact", verify_live_set(s)==0);
|
||||
|
||||
/* capture adjacency for a sample of live from-ids to compare post-compaction */
|
||||
#define NSAMP 12
|
||||
char samp[NSAMP][32]; size_t pre_cnt[NSAMP];
|
||||
for (int k=0;k<NSAMP;k++){
|
||||
snprintf(samp[k], sizeof samp[k], "node-%d", N_DEAD + k*20);
|
||||
StoreEdge* arr=NULL; size_t cnt=0;
|
||||
store_get_edges_from(s, samp[k], &arr, &cnt);
|
||||
pre_cnt[k]=cnt; store_edges_free(arr,cnt);
|
||||
}
|
||||
|
||||
int rc = store_compact(s);
|
||||
ok("store_compact returns 0", rc==0);
|
||||
|
||||
uint64_t pc_after = store_page_count(s);
|
||||
uint64_t free_after = store_free_page_count(s);
|
||||
long sz_after = file_size(egm);
|
||||
printf(" AFTER : page_count=%llu free_pages=%llu file=%ld bytes\n",
|
||||
(unsigned long long)pc_after, (unsigned long long)free_after, sz_after);
|
||||
printf(" RECLAIMED: %llu pages, %ld bytes (%.1f%% of file)\n",
|
||||
(unsigned long long)(pc_before - pc_after), sz_before - sz_after,
|
||||
sz_before ? 100.0*(sz_before-sz_after)/sz_before : 0.0);
|
||||
|
||||
ok("page count dropped (dead pages reclaimed)", pc_after < pc_before);
|
||||
ok("file size dropped (store physically shrank)", sz_after < sz_before);
|
||||
ok("store_check crc clean after compaction", store_check(s, STORE_CHECK_CRC)==0);
|
||||
ok("every LIVE record present + bit-exact at new locations", verify_live_set(s)==0);
|
||||
|
||||
/* adjacency index correct at relocated positions */
|
||||
int adj_bad = 0;
|
||||
for (int k=0;k<NSAMP;k++){
|
||||
StoreEdge* arr=NULL; size_t cnt=0;
|
||||
store_get_edges_from(s, samp[k], &arr, &cnt);
|
||||
if (cnt != pre_cnt[k]) adj_bad++;
|
||||
for (size_t j=0;j<cnt;j++){
|
||||
if (!streq(arr[j].from_id, samp[k])) { adj_bad++; break; }
|
||||
/* the returned edge must be the canonical latest live edge, bit-exact */
|
||||
int idx = atoi(arr[j].id + 5);
|
||||
StoreEdge want; gen_edge(idx, edge_live_version(idx), &want);
|
||||
if (!cmp_edge(&want,&arr[j])) adj_bad++;
|
||||
store_edge_free(&want);
|
||||
}
|
||||
store_edges_free(arr,cnt);
|
||||
}
|
||||
ok("adjacency (get_edges_from) correct + bit-exact post-compaction", adj_bad==0);
|
||||
|
||||
/* second compaction is a near no-op (no new dead space) and stays correct */
|
||||
uint64_t pc2_before = store_page_count(s);
|
||||
ok("compact again returns 0", store_compact(s)==0);
|
||||
ok("idempotent-ish: no growth on re-compact", store_page_count(s) <= pc2_before);
|
||||
ok("live set still intact after 2nd compaction", verify_live_set(s)==0);
|
||||
|
||||
engram_close(s);
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════════
|
||||
* TEST 2 — CRASH DURING COMPACTION: kill at phases 0/1/2 → consistent recovery.
|
||||
* Live set is identical whether we recover pre- or post-compaction, so the same
|
||||
* oracle must hold, and crc must always be clean (never corrupt).
|
||||
* ════════════════════════════════════════════════════════════════════════════ */
|
||||
static void test_crash_during_compaction(void){
|
||||
printf("\n== 2) crash during compaction at phases 0,1,2 → consistent store ==\n");
|
||||
for (int phase=0; phase<=2; phase++){
|
||||
mk_dir();
|
||||
char egm[600]; egm_path(egm, sizeof egm);
|
||||
EngramPagedStore* s = engram_open(g_dir);
|
||||
if (!s){ ok("engram_open", 0); continue; }
|
||||
populate_with_dead_space(s);
|
||||
engram_close(s); /* durable baseline on disk */
|
||||
|
||||
uint64_t pc_pre = 0;
|
||||
{ EngramPagedStore* p = engram_open(g_dir); pc_pre = store_page_count(p); engram_close(p); }
|
||||
|
||||
EngramPagedStore* c = engram_open(g_dir);
|
||||
store__compact_crashat(c, phase); /* crashes mid-compaction (frees c) */
|
||||
|
||||
EngramPagedStore* r = engram_open(g_dir); /* recover */
|
||||
char nm[80];
|
||||
snprintf(nm, sizeof nm, "phase %d: recovers, crc clean", phase);
|
||||
ok(nm, r && store_check(r, STORE_CHECK_CRC)==0);
|
||||
snprintf(nm, sizeof nm, "phase %d: every live record intact (not corrupt)", phase);
|
||||
ok(nm, r && verify_live_set(r)==0);
|
||||
if (r){
|
||||
uint64_t pc_now = store_page_count(r);
|
||||
if (phase < 2){
|
||||
snprintf(nm, sizeof nm, "phase %d: recovered PRE-compaction image", phase);
|
||||
ok(nm, pc_now == pc_pre);
|
||||
} else {
|
||||
snprintf(nm, sizeof nm, "phase %d: recovered POST-compaction (shrunk)", phase);
|
||||
ok(nm, pc_now < pc_pre);
|
||||
}
|
||||
/* store stays writable + durable after recovery */
|
||||
StoreNode n; gen_node(N_NODES+phase, &n); free(n.id);
|
||||
n.id = strdup("post-recovery-node");
|
||||
store_put_node(r, &n); store_node_free(&n);
|
||||
StoreNode g; int hit = store_get_node(r, "post-recovery-node", &g);
|
||||
snprintf(nm, sizeof nm, "phase %d: store writable after recovery", phase);
|
||||
ok(nm, hit==1);
|
||||
if (hit==1) store_node_free(&g);
|
||||
engram_close(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════════
|
||||
* TEST 3 — BACKGROUND CHECKPOINTER: a low threshold fires checkpoints on the
|
||||
* write path, reclaiming the WAL prefix automatically; recovery still correct.
|
||||
* ════════════════════════════════════════════════════════════════════════════ */
|
||||
static void test_background_checkpointer(void){
|
||||
printf("\n== 3) background checkpointer: auto-checkpoint on threshold ==\n");
|
||||
|
||||
/* (a) ops trigger */
|
||||
{
|
||||
mk_dir();
|
||||
char wal[600]; wal_path(wal, sizeof wal);
|
||||
EngramPagedStore* s = engram_open(g_dir);
|
||||
if (!s){ ok("engram_open", 0); return; }
|
||||
store_set_checkpoint_policy(s, /*ops*/50, /*dirty*/0, /*wal_bytes*/0, /*ms*/0);
|
||||
uint64_t ckpt0 = engram_last_checkpoint_lsn(s);
|
||||
for (int i=0;i<600;i++){ StoreNode n; gen_node(i,&n); store_put_node(s,&n); store_node_free(&n); }
|
||||
uint64_t ckpt1 = engram_last_checkpoint_lsn(s);
|
||||
long wsz = file_size(wal);
|
||||
printf(" ops-trigger: ckpt_lsn %llu -> %llu, WAL=%ld bytes after 600 puts\n",
|
||||
(unsigned long long)ckpt0, (unsigned long long)ckpt1, wsz);
|
||||
ok("ops trigger fired an automatic checkpoint", ckpt1 > ckpt0);
|
||||
ok("WAL prefix reclaimed (WAL stays small)", wsz >= 0 && wsz < 200000);
|
||||
/* crash (abandon RAM) then recover — everything durable via WAL+checkpoint */
|
||||
store__crash(s);
|
||||
EngramPagedStore* r = engram_open(g_dir);
|
||||
int bad=0;
|
||||
for (int i=0;i<600;i++){ char id[32]; snprintf(id,sizeof id,"node-%d",i);
|
||||
StoreNode w; gen_node(i,&w); StoreNode g; int hit=store_get_node(r,id,&g);
|
||||
if (hit!=1 || !cmp_node(&w,&g)) bad++; if(hit==1) store_node_free(&g); store_node_free(&w); }
|
||||
ok("recovery correct after auto-checkpoints (ops)", r && bad==0);
|
||||
ok("crc clean after recovery (ops)", r && store_check(r,STORE_CHECK_CRC)==0);
|
||||
if (r) engram_close(r);
|
||||
}
|
||||
/* (b) WAL-bytes trigger */
|
||||
{
|
||||
mk_dir();
|
||||
char wal[600]; wal_path(wal, sizeof wal);
|
||||
EngramPagedStore* s = engram_open(g_dir);
|
||||
if (!s){ ok("engram_open", 0); return; }
|
||||
store_set_checkpoint_policy(s, /*ops*/0, /*dirty*/0, /*wal_bytes*/64*1024, /*ms*/0);
|
||||
uint64_t ckpt0 = engram_last_checkpoint_lsn(s);
|
||||
for (int i=0;i<600;i++){ StoreNode n; gen_node(i,&n); store_put_node(s,&n); store_node_free(&n); }
|
||||
uint64_t ckpt1 = engram_last_checkpoint_lsn(s);
|
||||
long wsz = file_size(wal);
|
||||
printf(" wal-bytes-trigger: ckpt_lsn %llu -> %llu, WAL=%ld bytes\n",
|
||||
(unsigned long long)ckpt0, (unsigned long long)ckpt1, wsz);
|
||||
ok("wal-bytes trigger fired an automatic checkpoint", ckpt1 > ckpt0);
|
||||
ok("WAL kept bounded by byte threshold", wsz >= 0 && wsz < 2*1024*1024);
|
||||
engram_close(s);
|
||||
}
|
||||
/* (c) dirty-frames trigger (under a bounded pool) */
|
||||
{
|
||||
mk_dir();
|
||||
EngramPagedStore* s = engram_open(g_dir);
|
||||
if (!s){ ok("engram_open", 0); return; }
|
||||
store_set_checkpoint_policy(s, /*ops*/0, /*dirty*/16, /*wal_bytes*/0, /*ms*/0);
|
||||
uint64_t ckpt0 = engram_last_checkpoint_lsn(s);
|
||||
for (int i=0;i<400;i++){ StoreNode n; gen_node(i,&n); store_put_node(s,&n); store_node_free(&n); }
|
||||
uint64_t ckpt1 = engram_last_checkpoint_lsn(s);
|
||||
ok("dirty-frames trigger fired an automatic checkpoint", ckpt1 > ckpt0);
|
||||
engram_close(s);
|
||||
}
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════════
|
||||
* TEST 4 — POOL COOPERATION: compact under a tiny frame budget (constant eviction
|
||||
* + re-fault); correctness holds and no stale frame survives a relocated page.
|
||||
* ════════════════════════════════════════════════════════════════════════════ */
|
||||
static void test_pool_cooperation(void){
|
||||
printf("\n== 4) compaction under a small buffer pool (forced eviction) ==\n");
|
||||
setenv("ENGRAM_POOL_FRAMES", "24", 1); /* pool << store, and the temp build too */
|
||||
mk_dir();
|
||||
char egm[600]; egm_path(egm, sizeof egm);
|
||||
EngramPagedStore* s = engram_open(g_dir);
|
||||
ok("engram_open (24-frame pool)", s != NULL);
|
||||
if (!s){ unsetenv("ENGRAM_POOL_FRAMES"); return; }
|
||||
store__set_pool_frames(s, 24);
|
||||
|
||||
populate_with_dead_space(s);
|
||||
engram_checkpoint(s);
|
||||
uint64_t pc_before = store_page_count(s);
|
||||
|
||||
int rc = store_compact(s);
|
||||
ok("store_compact under tiny pool returns 0", rc==0);
|
||||
|
||||
StorePoolStats st; store_pool_stats(s, &st);
|
||||
printf(" post-compaction pool: cap=%zu resident=%zu pinned=%zu dirty=%zu\n",
|
||||
st.cap, st.resident, st.pinned, st.dirty);
|
||||
ok("pool respected budget after compaction (resident<=cap)", st.resident <= st.cap);
|
||||
ok("page count dropped under small pool", store_page_count(s) < pc_before);
|
||||
ok("crc clean under small pool", store_check(s, STORE_CHECK_CRC)==0);
|
||||
/* If any relocated page had a stale frame, a read would return wrong bytes. */
|
||||
ok("every live record bit-exact under small pool (no stale frames)", verify_live_set(s)==0);
|
||||
|
||||
engram_close(s);
|
||||
unsetenv("ENGRAM_POOL_FRAMES");
|
||||
}
|
||||
|
||||
int main(void){
|
||||
printf("=== M5 COMPACTION + BACKGROUND CHECKPOINTER GATE ===\n");
|
||||
test_reclaim();
|
||||
test_crash_during_compaction();
|
||||
test_background_checkpointer();
|
||||
test_pool_cooperation();
|
||||
printf("\n=== RESULT: %d passed, %d failed ===\n", g_pass, g_fail);
|
||||
/* cleanup */
|
||||
return g_fail ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
/* test_geometry.c — build + RUN gate for the M9 FOUNDATION geometry descriptor
|
||||
* (engram_geometry.{c,h}). Self-contained: synthesizes a store with two KNOWN
|
||||
* embedding clusters + intra-cluster hebb edges, then verifies the descriptor
|
||||
* recovers the shape — centroid near the seeded cluster, skeleton = the strong
|
||||
* intra-cluster edges, membership gradient, radius, positive co-registration.
|
||||
*
|
||||
* Pure C11; links engram_geometry.c + engram_store.c + engram_vindex.c; -lm.
|
||||
* ASan/UBSan clean. Needs no live data.
|
||||
*/
|
||||
#include "engram_geometry.h"
|
||||
#include "engram_store.h"
|
||||
#include "engram_vindex.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
#include <stdint.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#define DIM 64
|
||||
static int g_fail=0;
|
||||
#define CHECK(c,m) do{ if(!(c)){printf(" FAIL: %s\n",m); g_fail=1;} else printf(" ok: %s\n",m);}while(0)
|
||||
|
||||
static uint64_t rs=0x1234abcdULL;
|
||||
static uint64_t xr(void){ uint64_t z=(rs+=0x9E3779B97F4A7C15ULL);
|
||||
z=(z^(z>>30))*0xBF58476D1CE4E5B9ULL; z=(z^(z>>27))*0x94D049BB133111EBULL; return z^(z>>31); }
|
||||
static float jitter(void){ return (float)(((double)(xr()>>11)*(1.0/9007199254740992.0))-0.5)*0.15f; }
|
||||
|
||||
/* two clusters: A centered on axis 0, B centered on axis 1. NA+NB nodes. */
|
||||
#define NA 40
|
||||
#define NB 40
|
||||
|
||||
int main(void){
|
||||
printf("=== engram_geometry (M9 foundation) test suite ===\n");
|
||||
char path[256]; snprintf(path,sizeof path,"/tmp/geo_test_store_%d.egm",(int)getpid());
|
||||
unlink(path);
|
||||
EngramPagedStore* st=store_create(path);
|
||||
if(!st){ printf("FAIL: store_create\n"); return 1; }
|
||||
|
||||
char aids[NA][16], bids[NB][16];
|
||||
/* cluster A: near +e0 ; cluster B: near +e1 */
|
||||
for(int i=0;i<NA;i++){
|
||||
StoreNode n; memset(&n,0,sizeof n);
|
||||
snprintf(aids[i],16,"A%d",i); n.id=aids[i]; n.node_type="Concept"; n.tier="Semantic";
|
||||
n.content="cluster-A"; n.salience=0.5+0.01*i;
|
||||
float v[DIM]; for(int d=0;d<DIM;d++) v[d]=jitter(); v[0]=1.0f+jitter();
|
||||
n.emb=v; n.emb_dim=DIM; store_put_node(st,&n);
|
||||
}
|
||||
for(int i=0;i<NB;i++){
|
||||
StoreNode n; memset(&n,0,sizeof n);
|
||||
snprintf(bids[i],16,"B%d",i); n.id=bids[i]; n.node_type="Concept"; n.tier="Semantic";
|
||||
n.content="cluster-B"; n.salience=0.3;
|
||||
float v[DIM]; for(int d=0;d<DIM;d++) v[d]=jitter(); v[1]=1.0f+jitter();
|
||||
n.emb=v; n.emb_dim=DIM; store_put_node(st,&n);
|
||||
}
|
||||
/* strong intra-A hebb edges (a chain + hub), weaker cross edges A0<->B0 */
|
||||
int ei=0;
|
||||
for(int i=1;i<NA;i++){
|
||||
StoreEdge e; memset(&e,0,sizeof e); char id[24]; snprintf(id,24,"eA%d",ei++);
|
||||
e.id=id; e.from_id=aids[0]; e.to_id=aids[i]; e.relation="assoc"; e.weight=0.9; e.hebb=0.4;
|
||||
store_put_edge(st,&e);
|
||||
}
|
||||
for(int i=1;i<NB;i++){
|
||||
StoreEdge e; memset(&e,0,sizeof e); char id[24]; snprintf(id,24,"eB%d",ei++);
|
||||
e.id=id; e.from_id=bids[0]; e.to_id=bids[i]; e.relation="assoc"; e.weight=0.9; e.hebb=0.4;
|
||||
store_put_edge(st,&e);
|
||||
}
|
||||
{ StoreEdge e; memset(&e,0,sizeof e); e.id=(char*)"eX"; e.from_id=aids[0]; e.to_id=bids[0];
|
||||
e.relation="assoc"; e.weight=0.5; e.hebb=0.0; store_put_edge(st,&e); }
|
||||
store_close(st);
|
||||
|
||||
VIndex* ix=vindex_create(DIM,0,0);
|
||||
char** ids=NULL; int nids=0;
|
||||
int ins=vindex_build_from_store(ix, path, &ids, &nids);
|
||||
CHECK(ins==NA+NB, "vindex built over all embedded nodes");
|
||||
|
||||
GeoParams P; engram_geo_default_params(&P); P.ann_k=20; P.max_members=0;
|
||||
|
||||
/* seed inside cluster A -> expect an A-dominated neighborhood */
|
||||
st=store_open(path);
|
||||
/* global-mean cache over the embedded set: the centering offset */
|
||||
GeoMeanCache* mc=engram_geo_mean_build(st);
|
||||
const float* gm=engram_geo_mean_vec(mc);
|
||||
CHECK(mc!=NULL && engram_geo_mean_dim(mc)==DIM, "global-mean cache built over embedded set");
|
||||
CHECK(engram_geo_mean_count(mc)==(uint64_t)(NA+NB), "global mean averaged all embedded nodes");
|
||||
const char* seeds[1]={aids[0]};
|
||||
/* CENTERED descriptor: pass the global mean so geometry runs in isotropic space */
|
||||
GeoDescriptor* g=engram_geometry_descriptor(st, ix, ids, nids, seeds, 1, &P, gm);
|
||||
CHECK(g!=NULL, "descriptor computed");
|
||||
if(g){
|
||||
printf(" members=%d embedded=%d edges=%d k_core=%d radius=%.4f co_reg=%.3f n_axes=%d\n",
|
||||
g->n_members,g->n_embedded,g->n_edges,g->k_core,g->radius,g->co_registration,g->n_axes);
|
||||
|
||||
/* geometry ran in CENTERED space: g->centroid is the centered centroid,
|
||||
* g->global_mean the applied offset. Reconstruct the raw prototype
|
||||
* (centroid + global_mean) and check it sits on cluster-A's axis. */
|
||||
CHECK(g->global_mean!=NULL, "descriptor recorded the centering offset (centered mode)");
|
||||
int argmax=0; float best=-1.f;
|
||||
for(int d=0;d<g->dim;d++){ float raw=g->centroid[d]+(g->global_mean?g->global_mean[d]:0.f);
|
||||
if(fabsf(raw)>best){ best=fabsf(raw); argmax=d; } }
|
||||
printf(" raw-prototype dominant axis = %d (expect 0); centered c[0]=%.3f c[1]=%.3f\n",
|
||||
argmax, g->centroid[0], g->centroid[1]);
|
||||
CHECK(argmax==0, "raw prototype sits on cluster-A's axis (near members)");
|
||||
/* centering pushes A off cluster-B's axis: centered c[0] > c[1] */
|
||||
CHECK(g->centroid[0] > g->centroid[1], "centered centroid leans off B's axis (isotropy)");
|
||||
|
||||
/* hub should be A0 (the intra-A hub with NA-1 strong edges) */
|
||||
CHECK(g->hub_id && strcmp(g->hub_id,"A0")==0, "hub = the relational center A0");
|
||||
|
||||
/* membership: seed A0 == 1.0; A-members strong, B-members (if any) weaker */
|
||||
double seedw=-1, minA=2, maxB=-1; int na=0,nb=0;
|
||||
for(int i=0;i<g->n_members;i++){
|
||||
const char* id=g->members[i].id; double w=g->members[i].membership;
|
||||
if(strcmp(id,"A0")==0) seedw=w;
|
||||
if(id[0]=='A'){ na++; if(w<minA)minA=w; }
|
||||
if(id[0]=='B'){ nb++; if(w>maxB)maxB=w; }
|
||||
}
|
||||
printf(" A-members=%d B-members=%d seedw=%.3f\n", na,nb,seedw);
|
||||
CHECK(fabs(seedw-1.0)<1e-9, "seed membership == 1.0");
|
||||
CHECK(na>=NA-1, "neighborhood recovers cluster A");
|
||||
|
||||
/* skeleton = the strong intra-A edges: every edge eff_weight>=threshold,
|
||||
* and edges connect A-nodes (co-registration should be positive: wired
|
||||
* pairs are semantically near). */
|
||||
int allstrong=1, allA=1;
|
||||
for(int e=0;e<g->n_edges;e++){
|
||||
if(g->edges[e].eff_weight < P.edge_min_weight) allstrong=0;
|
||||
const char* a=g->members[g->edges[e].a].id, *b=g->members[g->edges[e].b].id;
|
||||
if(!(a[0]=='A'&&b[0]=='A')) { /* the lone eX cross edge is allowed */
|
||||
if(!((strcmp(a,"A0")==0&&strcmp(b,"B0")==0)||(strcmp(a,"B0")==0&&strcmp(b,"A0")==0))) allA=0; }
|
||||
}
|
||||
CHECK(allstrong, "skeleton holds only above-threshold (strong) edges");
|
||||
CHECK(allA, "skeleton backbone is the intra-cluster wiring");
|
||||
CHECK(g->co_registration>0.0, "co-registration positive (wired pairs are semantically near)");
|
||||
|
||||
/* principal axes: extents strictly non-increasing */
|
||||
int mono=1; for(int i=1;i<g->n_axes;i++) if(g->axes[i].extent>g->axes[i-1].extent+1e-9) mono=0;
|
||||
CHECK(g->n_axes>0 && mono, "principal axes sorted by descending extent");
|
||||
CHECK(g->radius>0, "radius positive");
|
||||
}
|
||||
engram_geo_free(g);
|
||||
|
||||
/* edge cases: NULL store, no seeds, relational-only (NULL vindex) */
|
||||
CHECK(engram_geometry_descriptor(NULL,ix,ids,nids,seeds,1,&P,gm)==NULL, "NULL store -> NULL");
|
||||
CHECK(engram_geometry_descriptor(st,ix,ids,nids,seeds,0,&P,gm)==NULL, "zero seeds -> NULL");
|
||||
GeoDescriptor* g2=engram_geometry_descriptor(st, NULL, NULL, 0, seeds, 1, &P, gm);
|
||||
CHECK(g2!=NULL && g2->n_members>=NA-1, "relational-only path (no vindex) works");
|
||||
engram_geo_free(g2);
|
||||
/* raw (uncentered) mode still supported: global_mean=NULL -> no offset recorded */
|
||||
GeoDescriptor* g3=engram_geometry_descriptor(st, ix, ids, nids, seeds, 1, &P, NULL);
|
||||
CHECK(g3!=NULL && g3->global_mean==NULL, "raw mode (global_mean=NULL) leaves offset unset");
|
||||
engram_geo_free(g3);
|
||||
|
||||
engram_geo_mean_free(mc);
|
||||
for(int i=0;i<nids;i++) free(ids[i]); free(ids);
|
||||
vindex_free(ix); store_close(st); unlink(path);
|
||||
printf("\n=== %s ===\n", g_fail?"FAILURES PRESENT":"ALL TESTS PASSED");
|
||||
return g_fail;
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
/* test_interoception_p0_emb.c — M-INTEROCEPTION Priority 0.
|
||||
*
|
||||
* Verifies the new READ-ONLY builtin engram_scan_nodes_emb_json(limit,offset):
|
||||
* - every emitted node carries emb_dim and an emb JSON array of that length,
|
||||
* - nodes without an embedding emit emb_dim:0 / emb:[],
|
||||
* - pagination (limit/offset) is honoured,
|
||||
* - the count matches engram_node_count,
|
||||
* - the EXISTING engram_scan_nodes_json path is byte-unchanged (no emb field),
|
||||
* i.e. the addition is purely additive / behavior-neutral.
|
||||
*
|
||||
* Pure-C harness (no elc). We craft a snapshot with real emb vectors, load it
|
||||
* (engram_load parses "emb" comma-lists into node->emb via eg_parse_emb), then
|
||||
* dump via both scan paths. Assertions live in run_interoception_p0.sh.
|
||||
*/
|
||||
#include "el_runtime.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
static el_val_t S(const char* s){ return EL_STR(s); }
|
||||
|
||||
/* 16-d embedding as a comma list (>=8 required by eg_parse_emb). */
|
||||
static void emb_list(char* out, size_t cap, int dim, double base){
|
||||
size_t o = 0;
|
||||
for (int i = 0; i < dim; i++){
|
||||
o += snprintf(out+o, cap-o, "%s%.4f", i?",":"", base + 0.01*i);
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv){
|
||||
if (argc < 2){ fprintf(stderr, "usage: %s <dir>\n", argv[0]); return 2; }
|
||||
const char* dir = argv[1];
|
||||
char snap[1024]; snprintf(snap, sizeof snap, "%s/seed.json", dir);
|
||||
|
||||
char e1[512], e2[512];
|
||||
emb_list(e1, sizeof e1, 16, 0.10);
|
||||
emb_list(e2, sizeof e2, 16, 0.50);
|
||||
|
||||
/* Two embedded nodes (distinct salience → deterministic sort order) and one
|
||||
* un-embedded node. */
|
||||
FILE* f = fopen(snap, "w");
|
||||
if (!f){ perror("fopen"); return 2; }
|
||||
fprintf(f,
|
||||
"{\"nodes\":["
|
||||
"{\"id\":\"n-high\",\"content\":\"high salience embedded\",\"node_type\":\"Concept\","
|
||||
"\"label\":\"emb-high\",\"tier\":\"Semantic\",\"salience\":0.9,\"importance\":0.8,"
|
||||
"\"confidence\":1.0,\"created_at\":1000,\"emb\":\"%s\"},"
|
||||
"{\"id\":\"n-mid\",\"content\":\"mid salience embedded\",\"node_type\":\"Concept\","
|
||||
"\"label\":\"emb-mid\",\"tier\":\"Semantic\",\"salience\":0.5,\"importance\":0.5,"
|
||||
"\"confidence\":1.0,\"created_at\":2000,\"emb\":\"%s\"},"
|
||||
"{\"id\":\"n-low\",\"content\":\"low salience no embedding\",\"node_type\":\"Fact\","
|
||||
"\"label\":\"noemb-low\",\"tier\":\"Semantic\",\"salience\":0.1,\"importance\":0.2,"
|
||||
"\"confidence\":1.0,\"created_at\":3000}"
|
||||
"],\"edges\":[]}", e1, e2);
|
||||
fclose(f);
|
||||
|
||||
if (!engram_load(S(snap))){ fprintf(stderr, "load failed\n"); return 2; }
|
||||
long long nc = (long long)(int64_t)engram_node_count();
|
||||
printf("node_count=%lld\n", nc);
|
||||
|
||||
/* full page */
|
||||
el_val_t all = engram_scan_nodes_emb_json((el_val_t)256, (el_val_t)0);
|
||||
char p[1024];
|
||||
snprintf(p, sizeof p, "%s/emb_all.json", dir);
|
||||
f = fopen(p, "w"); fputs(EL_CSTR(all), f); fclose(f);
|
||||
|
||||
/* pagination: one node at offset 0 and one at offset 1 */
|
||||
el_val_t pg0 = engram_scan_nodes_emb_json((el_val_t)1, (el_val_t)0);
|
||||
el_val_t pg1 = engram_scan_nodes_emb_json((el_val_t)1, (el_val_t)1);
|
||||
snprintf(p, sizeof p, "%s/emb_pg0.json", dir); f = fopen(p, "w"); fputs(EL_CSTR(pg0), f); fclose(f);
|
||||
snprintf(p, sizeof p, "%s/emb_pg1.json", dir); f = fopen(p, "w"); fputs(EL_CSTR(pg1), f); fclose(f);
|
||||
|
||||
/* existing path — must be unchanged / carry NO emb */
|
||||
el_val_t plain = engram_scan_nodes_json((el_val_t)256, (el_val_t)0);
|
||||
snprintf(p, sizeof p, "%s/plain.json", dir); f = fopen(p, "w"); fputs(EL_CSTR(plain), f); fclose(f);
|
||||
|
||||
printf("wrote dumps to %s\n", dir);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
/* test_interoception_p1_consol.c — M-INTEROCEPTION Priority 1.
|
||||
* Two-threshold consolidation (ENGRAM_CONSOLIDATION, default OFF).
|
||||
*
|
||||
* Modes:
|
||||
* accrual — flag OFF (pure trunk). Drive N co-activations of a WIRED pair and
|
||||
* print act-stats at sampled N so the run script can plot the
|
||||
* hebb accrual curve (headline measurement). No consolidation code
|
||||
* runs; this measures the EXISTING EWMA accrual.
|
||||
* connect — flag ON. Seed, activate to populate WM, then create a STRONG ISE
|
||||
* (connects to wm_top) and a WEAK ISE (below the bar → nothing).
|
||||
* Exports the graph so edges from each ISE can be counted.
|
||||
* perm — flag ON. Load two OLD InternalStateEvent nodes; promote one to
|
||||
* permanence; prune telemetry; export so the durable one is shown
|
||||
* to survive while the ephemeral one is swept.
|
||||
* offcheck — flag OFF. Prove creating an ISE forms NO edges and
|
||||
* engram_consolidate_permanence is a no-op (byte-identical OFF path).
|
||||
*/
|
||||
#include "el_runtime.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
static el_val_t S(const char* s){ return EL_STR(s); }
|
||||
static el_val_t F(double d){ return el_from_float(d); }
|
||||
|
||||
static void build_seed(void){
|
||||
el_val_t a = engram_node_full(S("hebbian potentiation strengthens co-active memory links"),
|
||||
S("Concept"), S("hebb-a"), F(0.9), F(0.85), F(1.0), S("Semantic"),
|
||||
S("hebbian,memory,activation"));
|
||||
el_val_t b = engram_node_full(S("co-active memory links accrue hebbian associative weight"),
|
||||
S("Concept"), S("hebb-b"), F(0.9), F(0.85), F(1.0), S("Semantic"),
|
||||
S("hebbian,memory,weight"));
|
||||
el_val_t c = engram_node_full(S("unrelated culinary recipe for sourdough bread"),
|
||||
S("Fact"), S("distractor-1"), F(0.4), F(0.4), F(1.0), S("Semantic"), S("food"));
|
||||
el_val_t d = engram_node_full(S("the weather forecast predicts rain tomorrow afternoon"),
|
||||
S("Fact"), S("distractor-2"), F(0.4), F(0.4), F(1.0), S("Semantic"), S("weather"));
|
||||
engram_connect(a, b, F(0.8), S("associate"));
|
||||
engram_connect(a, c, F(0.3), S("associate"));
|
||||
engram_connect(b, d, F(0.3), S("associate"));
|
||||
}
|
||||
static const char* QUERY =
|
||||
"hebbian potentiation co-active memory links associative weight";
|
||||
|
||||
int main(int argc, char** argv){
|
||||
if (argc < 3){ fprintf(stderr,"usage: %s <accrual|connect|perm|offcheck> <dir>\n",argv[0]); return 2; }
|
||||
const char* mode = argv[1];
|
||||
const char* dir = argv[2];
|
||||
char p[1024];
|
||||
|
||||
if (!strcmp(mode,"accrual")){
|
||||
build_seed();
|
||||
int samples[] = {1,10,50,100,250,500,1000,1625,2000,2500,3000};
|
||||
int ns = (int)(sizeof samples/sizeof samples[0]);
|
||||
int NMAX = samples[ns-1];
|
||||
int si = 0;
|
||||
for (int n=1; n<=NMAX; n++){
|
||||
engram_activate_json(S(QUERY), (el_val_t)3);
|
||||
if (si<ns && n==samples[si]){
|
||||
printf("SAMPLE %d %s\n", n, EL_CSTR(engram_act_stats_json()));
|
||||
si++;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!strcmp(mode,"connect")){
|
||||
build_seed();
|
||||
engram_activate_json(S(QUERY), (el_val_t)3);
|
||||
long long e_before = (long long)(int64_t)engram_edge_count();
|
||||
/* STRONG ISE — should connect to wm_top */
|
||||
el_val_t ise_strong = engram_node_full(S("strong internal state: focused on hebbian consolidation"),
|
||||
S("InternalStateEvent"), S("ise-strong"), F(0.9), F(0.8), F(1.0), S("Working"), S("ise"));
|
||||
long long e_after_strong = (long long)(int64_t)engram_edge_count();
|
||||
/* WEAK ISE — below the connection bar (salience 0.3 < default 0.6) */
|
||||
el_val_t ise_weak = engram_node_full(S("weak internal state: idle drift"),
|
||||
S("InternalStateEvent"), S("ise-weak"), F(0.3), F(0.3), F(1.0), S("Working"), S("ise"));
|
||||
long long e_after_weak = (long long)(int64_t)engram_edge_count();
|
||||
printf("ISE_STRONG_ID %s\n", EL_CSTR(ise_strong));
|
||||
printf("ISE_WEAK_ID %s\n", EL_CSTR(ise_weak));
|
||||
printf("EDGES before=%lld after_strong=%lld after_weak=%lld\n",
|
||||
e_before, e_after_strong, e_after_weak);
|
||||
snprintf(p,sizeof p,"%s/connect.json",dir);
|
||||
el_val_t g = engram_save(S(p)); (void)g;
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!strcmp(mode,"perm")){
|
||||
/* Two OLD ISE nodes (created_at far in the past → prunable at 48h). */
|
||||
snprintf(p,sizeof p,"%s/seed.json",dir);
|
||||
FILE* f=fopen(p,"w");
|
||||
fprintf(f,"{\"nodes\":["
|
||||
"{\"id\":\"ise-durable\",\"content\":\"promoted internal state\",\"node_type\":\"InternalStateEvent\","
|
||||
"\"label\":\"ise-durable\",\"salience\":0.5,\"confidence\":1.0,\"created_at\":1000},"
|
||||
"{\"id\":\"ise-ephemeral\",\"content\":\"transient internal state\",\"node_type\":\"InternalStateEvent\","
|
||||
"\"label\":\"ise-ephemeral\",\"salience\":0.5,\"confidence\":1.0,\"created_at\":1000}"
|
||||
"],\"edges\":[]}");
|
||||
fclose(f);
|
||||
if(!engram_load(S(p))){ fprintf(stderr,"load failed\n"); return 2; }
|
||||
long long n_before = (long long)(int64_t)engram_node_count();
|
||||
el_val_t promoted = engram_consolidate_permanence(S("ise-durable"));
|
||||
long long removed = (long long)(int64_t)engram_prune_telemetry((el_val_t)0); /* default 48h */
|
||||
long long n_after = (long long)(int64_t)engram_node_count();
|
||||
printf("PROMOTED %lld\n", (long long)(int64_t)promoted);
|
||||
printf("NODES before=%lld after=%lld removed=%lld\n", n_before, n_after, removed);
|
||||
printf("DURABLE_NODE %s\n", EL_CSTR(engram_get_node_json(S("ise-durable"))));
|
||||
printf("EPHEMERAL_NODE %s\n", EL_CSTR(engram_get_node_json(S("ise-ephemeral"))));
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!strcmp(mode,"offcheck")){
|
||||
build_seed();
|
||||
engram_activate_json(S(QUERY), (el_val_t)3);
|
||||
long long e_before = (long long)(int64_t)engram_edge_count();
|
||||
engram_node_full(S("strong internal state with flag OFF"),
|
||||
S("InternalStateEvent"), S("ise-off"), F(0.9), F(0.8), F(1.0), S("Working"), S("ise"));
|
||||
long long e_after = (long long)(int64_t)engram_edge_count();
|
||||
el_val_t perm = engram_consolidate_permanence(S("ise-off"));
|
||||
printf("OFF edges before=%lld after=%lld perm_ret=%lld\n",
|
||||
e_before, e_after, (long long)(int64_t)perm);
|
||||
return (e_before==e_after && (int64_t)perm==0) ? 0 : 1;
|
||||
}
|
||||
|
||||
fprintf(stderr,"unknown mode %s\n",mode); return 2;
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
/* test_interoception_p2_chrono.c — M-INTEROCEPTION Priority 2.
|
||||
* Chronoception: engram_age_field(delta_ms) + reboot catch-up
|
||||
* (ENGRAM_CHRONOCEPTION, default OFF).
|
||||
*
|
||||
* Uses loaded snapshots with KNOWN working_memory_weight / background_activation
|
||||
* so the field is deterministic without depending on activation. (engram_load
|
||||
* halves WM on boot — the laundering step — so snapshot wm 1.0 -> 0.5 resident.)
|
||||
*
|
||||
* Modes:
|
||||
* once <dir> <dt_ms> — age the field once by dt; save field.json.
|
||||
* split <dir> <dt_ms> <N> — age by dt/N, N times; save field.json.
|
||||
* (once vs split must match: scale-invariance.)
|
||||
* catchup <dir> <gap_ms> — write a last-tick gap_ms in the past, then
|
||||
* engram_age_field_catchup(); print MAGNITUDE.
|
||||
* offcheck <dir> <dt_ms> — flag OFF: age returns 0 and field is untouched.
|
||||
*/
|
||||
#include "el_runtime.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
|
||||
static el_val_t S(const char* s){ return EL_STR(s); }
|
||||
|
||||
static void write_seed(const char* dir){
|
||||
char p[1024]; snprintf(p,sizeof p,"%s/seed.json",dir);
|
||||
FILE* f=fopen(p,"w");
|
||||
fprintf(f,"{\"nodes\":["
|
||||
"{\"id\":\"f1\",\"content\":\"field node 1\",\"node_type\":\"Concept\",\"label\":\"f1\","
|
||||
"\"salience\":0.9,\"confidence\":1.0,\"working_memory_weight\":1.0,\"background_activation\":0.5},"
|
||||
"{\"id\":\"f2\",\"content\":\"field node 2\",\"node_type\":\"Concept\",\"label\":\"f2\","
|
||||
"\"salience\":0.8,\"confidence\":1.0,\"working_memory_weight\":0.8,\"background_activation\":0.4},"
|
||||
"{\"id\":\"f3\",\"content\":\"field node 3\",\"node_type\":\"Concept\",\"label\":\"f3\","
|
||||
"\"salience\":0.7,\"confidence\":1.0,\"working_memory_weight\":0.6,\"background_activation\":0.3}"
|
||||
"],\"edges\":[]}");
|
||||
fclose(f);
|
||||
}
|
||||
static void load_seed(const char* dir){
|
||||
char p[1024]; snprintf(p,sizeof p,"%s/seed.json",dir);
|
||||
write_seed(dir);
|
||||
if(!engram_load(S(p))){ fprintf(stderr,"load failed\n"); exit(2); }
|
||||
}
|
||||
static void save_field(const char* dir){
|
||||
char p[1024]; snprintf(p,sizeof p,"%s/field.json",dir);
|
||||
engram_save(S(p));
|
||||
}
|
||||
|
||||
int main(int argc,char** argv){
|
||||
if(argc<3){ fprintf(stderr,"usage: %s <once|split|catchup|offcheck> <dir> ...\n",argv[0]); return 2; }
|
||||
const char* mode=argv[1];
|
||||
const char* dir =argv[2];
|
||||
|
||||
if(!strcmp(mode,"once")){
|
||||
double dt=atof(argv[3]);
|
||||
load_seed(dir);
|
||||
el_val_t mag=engram_age_field((el_val_t)(int64_t)dt);
|
||||
printf("MAGNITUDE %.10f\n", el_to_float(mag));
|
||||
save_field(dir);
|
||||
return 0;
|
||||
}
|
||||
if(!strcmp(mode,"split")){
|
||||
double dt=atof(argv[3]); int N=atoi(argv[4]); if(N<1)N=1;
|
||||
load_seed(dir);
|
||||
double sub=dt/(double)N;
|
||||
for(int i=0;i<N;i++) engram_age_field((el_val_t)(int64_t)sub);
|
||||
save_field(dir);
|
||||
printf("SPLIT dt=%.0f N=%d sub=%.4f\n", dt, N, sub);
|
||||
return 0;
|
||||
}
|
||||
if(!strcmp(mode,"catchup")){
|
||||
double gap=atof(argv[3]);
|
||||
load_seed(dir);
|
||||
/* Write a last-tick gap_ms in the past. ENGRAM_DATA_DIR is set == dir by
|
||||
* the runner, so the sidecar the runtime reads is <dir>/chrono_last_tick. */
|
||||
struct timeval tv; gettimeofday(&tv,NULL);
|
||||
long long now_ms=(long long)tv.tv_sec*1000+tv.tv_usec/1000;
|
||||
long long last=now_ms-(long long)gap;
|
||||
char p[1200]; snprintf(p,sizeof p,"%s/chrono_last_tick",dir);
|
||||
FILE* f=fopen(p,"w"); fprintf(f,"%lld\n",last); fclose(f);
|
||||
el_val_t mag=engram_age_field_catchup();
|
||||
printf("CATCHUP_MAGNITUDE %.10f\n", el_to_float(mag));
|
||||
save_field(dir);
|
||||
return 0;
|
||||
}
|
||||
if(!strcmp(mode,"offcheck")){
|
||||
double dt=atof(argv[3]);
|
||||
load_seed(dir);
|
||||
el_val_t mag=engram_age_field((el_val_t)(int64_t)dt);
|
||||
el_val_t magc=engram_age_field_catchup();
|
||||
printf("OFF age_mag=%.10f catchup_mag=%.10f\n", el_to_float(mag), el_to_float(magc));
|
||||
save_field(dir);
|
||||
return 0;
|
||||
}
|
||||
fprintf(stderr,"unknown mode %s\n",mode); return 2;
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/* test_interoception_p3_drift.c — M-INTEROCEPTION Priority 3 (PARTIAL).
|
||||
* Drift-sensor primitive engram_geo_displacement: GROWTH vs CORRUPTION split.
|
||||
*
|
||||
* Constructs synthetic GeoDescriptors (the struct is public) — a baseline and
|
||||
* two perturbations — and checks the sensor reports LOW core-displacement for a
|
||||
* periphery-only change (growth) and HIGH core-displacement for a core change
|
||||
* (corruption). No store / embeddings needed: this exercises the primitive in
|
||||
* isolation, which is the honest scope given there is no persisted SelfAnchor
|
||||
* yet (see engram_geometry.c). */
|
||||
#include "engram_geometry.h"
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
|
||||
static GeoMember MK(const char* id, double centrality, double dist){
|
||||
GeoMember m; memset(&m,0,sizeof m);
|
||||
m.id=strdup(id); m.centrality=centrality; m.dist_centroid=dist;
|
||||
m.membership=1.0; m.embedded=1; return m;
|
||||
}
|
||||
/* 4 members: 2 core (high centrality), 2 periphery (low). */
|
||||
static GeoDescriptor* mkdesc(float cx,float cy,double radius,
|
||||
double c1,double c2,double p1,double p2){
|
||||
GeoDescriptor* g=calloc(1,sizeof *g);
|
||||
g->dim=4;
|
||||
g->centroid=calloc(4,sizeof(float));
|
||||
g->centroid[0]=cx; g->centroid[1]=cy;
|
||||
g->radius=radius;
|
||||
g->n_members=4;
|
||||
g->members=calloc(4,sizeof(GeoMember));
|
||||
g->members[0]=MK("core1",10.0,c1);
|
||||
g->members[1]=MK("core2", 9.0,c2);
|
||||
g->members[2]=MK("per1", 1.0,p1);
|
||||
g->members[3]=MK("per2", 0.9,p2);
|
||||
return g;
|
||||
}
|
||||
|
||||
int main(void){
|
||||
GeoDisplacement d;
|
||||
/* baseline: core at 0.10, periphery at 0.50, centroid [1,0], radius 1.0 */
|
||||
GeoDescriptor* A = mkdesc(1.0f,0.0f,1.0, 0.10,0.10, 0.50,0.50);
|
||||
|
||||
/* (i) GROWTH: periphery extends 0.50->0.90; core fixed; radius grows. */
|
||||
GeoDescriptor* G = mkdesc(1.0f,0.0f,1.4, 0.10,0.10, 0.90,0.90);
|
||||
engram_geo_displacement(A,G,0.5,&d);
|
||||
printf("GROWTH centroid_sep=%.4f centroid_cos=%.4f radius_delta=%.4f core_disp=%.4f periph_disp=%.4f core_n=%d periph_n=%d\n",
|
||||
d.centroid_sep,d.centroid_cos,d.radius_delta,d.core_disp,d.periph_disp,d.core_matched,d.periph_matched);
|
||||
|
||||
/* (ii) CORRUPTION: core displaces 0.10->0.60; periphery fixed; centroid shifts. */
|
||||
GeoDescriptor* C = mkdesc(0.6f,0.4f,1.0, 0.60,0.60, 0.50,0.50);
|
||||
engram_geo_displacement(A,C,0.5,&d);
|
||||
printf("CORRUPTION centroid_sep=%.4f centroid_cos=%.4f radius_delta=%.4f core_disp=%.4f periph_disp=%.4f core_n=%d periph_n=%d\n",
|
||||
d.centroid_sep,d.centroid_cos,d.radius_delta,d.core_disp,d.periph_disp,d.core_matched,d.periph_matched);
|
||||
|
||||
/* identity: A vs A -> zero drift */
|
||||
engram_geo_displacement(A,A,0.5,&d);
|
||||
printf("IDENTITY centroid_sep=%.4f core_disp=%.4f periph_disp=%.4f\n",
|
||||
d.centroid_sep,d.core_disp,d.periph_disp);
|
||||
|
||||
engram_geo_free(A); engram_geo_free(G); engram_geo_free(C);
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/* test_interoception_p4_afferent.c — M-INTEROCEPTION Priority 4.
|
||||
* Afferent input counters in engram_act_stats_json: additive observability.
|
||||
* Drives KNOWN counts and asserts the emitted counters match and are monotonic.
|
||||
*/
|
||||
#include "el_runtime.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
static el_val_t S(const char* s){ return EL_STR(s); }
|
||||
static el_val_t F(double d){ return el_from_float(d); }
|
||||
|
||||
int main(void){
|
||||
/* 3 plain node creates + 2 ISE creates = 5 node_creates, 2 ise_ingests */
|
||||
el_val_t a=engram_node_full(S("alpha concept about memory and time"),S("Concept"),S("a"),F(0.9),F(0.8),F(1.0),S("Semantic"),S("x"));
|
||||
el_val_t b=engram_node_full(S("beta concept about memory and links"),S("Concept"),S("b"),F(0.9),F(0.8),F(1.0),S("Semantic"),S("x"));
|
||||
engram_node_full(S("gamma distractor"),S("Fact"),S("c"),F(0.4),F(0.4),F(1.0),S("Semantic"),S("y"));
|
||||
engram_node_full(S("heartbeat internal state one"),S("InternalStateEvent"),S("i1"),F(0.5),F(0.5),F(1.0),S("Working"),S("ise"));
|
||||
engram_node_full(S("curiosity internal state two"),S("InternalStateEvent"),S("i2"),F(0.5),F(0.5),F(1.0),S("Working"),S("ise"));
|
||||
/* 2 edge creates */
|
||||
engram_connect(a,b,F(0.8),S("associate"));
|
||||
engram_connect(b,a,F(0.3),S("associate"));
|
||||
|
||||
/* first reading (0 queries so far) */
|
||||
printf("STATS0 %s\n", EL_CSTR(engram_act_stats_json()));
|
||||
|
||||
/* 4 queries -> 4 activations */
|
||||
for(int i=0;i<4;i++) engram_activate_json(S("memory and time and links"), (el_val_t)2);
|
||||
printf("STATS1 %s\n", EL_CSTR(engram_act_stats_json()));
|
||||
|
||||
/* 3 more queries -> monotonic increase */
|
||||
for(int i=0;i<3;i++) engram_activate_json(S("memory and time and links"), (el_val_t)2);
|
||||
printf("STATS2 %s\n", EL_CSTR(engram_act_stats_json()));
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/* test_interoception_p5_dreams.c — M-INTEROCEPTION Priority 5.
|
||||
* Dream-recall-on-wake: engram_dreams_json(since_ms). Honesty rail — only
|
||||
* curiosity_scan ISEs still resident are returned; pruned (rotated-out) ones are
|
||||
* ABSENT (never confabulated); heartbeat ISEs are excluded.
|
||||
*/
|
||||
#include "el_runtime.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
|
||||
static el_val_t S(const char* s){ return EL_STR(s); }
|
||||
|
||||
int main(int argc,char** argv){
|
||||
if(argc<2){ fprintf(stderr,"usage: %s <dir>\n",argv[0]); return 2; }
|
||||
const char* dir=argv[1];
|
||||
struct timeval tv; gettimeofday(&tv,NULL);
|
||||
long long now=(long long)tv.tv_sec*1000+tv.tv_usec/1000;
|
||||
long long mid=now-3600000; /* 1h ago */
|
||||
long long ancient=1000; /* pruned by 48h retention */
|
||||
|
||||
char p[1024]; snprintf(p,sizeof p,"%s/seed.json",dir);
|
||||
FILE* f=fopen(p,"w");
|
||||
fprintf(f,"{\"nodes\":["
|
||||
"{\"id\":\"cur_old\",\"content\":\"{\\\"kind\\\":\\\"curiosity_scan\\\",\\\"q\\\":\\\"old wondering\\\"}\","
|
||||
"\"node_type\":\"InternalStateEvent\",\"label\":\"state-event\",\"created_at\":%lld},"
|
||||
"{\"id\":\"cur_mid\",\"content\":\"{\\\"kind\\\":\\\"curiosity_scan\\\",\\\"q\\\":\\\"mid wondering\\\"}\","
|
||||
"\"node_type\":\"InternalStateEvent\",\"label\":\"state-event\",\"created_at\":%lld},"
|
||||
"{\"id\":\"cur_recent\",\"content\":\"{\\\"kind\\\":\\\"curiosity_scan\\\",\\\"q\\\":\\\"recent wondering\\\"}\","
|
||||
"\"node_type\":\"InternalStateEvent\",\"label\":\"state-event\",\"created_at\":%lld},"
|
||||
"{\"id\":\"hb_recent\",\"content\":\"{\\\"kind\\\":\\\"heartbeat\\\",\\\"wm\\\":3}\","
|
||||
"\"node_type\":\"InternalStateEvent\",\"label\":\"state-event\",\"created_at\":%lld}"
|
||||
"],\"edges\":[]}", ancient, mid, now, now);
|
||||
fclose(f);
|
||||
if(!engram_load(S(p))){ fprintf(stderr,"load failed\n"); return 2; }
|
||||
|
||||
/* before prune: all resident curiosity_scan after since=0 */
|
||||
printf("BEFORE %s\n", EL_CSTR(engram_dreams_json((el_val_t)0)));
|
||||
/* prune 48h — cur_old (ancient) rotates out */
|
||||
long long pruned=(long long)(int64_t)engram_prune_telemetry((el_val_t)0);
|
||||
printf("PRUNED %lld\n", pruned);
|
||||
printf("AFTER %s\n", EL_CSTR(engram_dreams_json((el_val_t)0)));
|
||||
/* since filter: only events created after 30 min ago -> cur_recent only */
|
||||
long long since=now-1800000;
|
||||
printf("SINCE %lld %s\n", since, EL_CSTR(engram_dreams_json((el_val_t)(int64_t)since)));
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
/* test_m35_hebb_persist.c — M3.5 PRE-FLIP GATE.
|
||||
*
|
||||
* Proves that in-place field mutations made during spreading activation — edge
|
||||
* `hebb` (+ last_fired), node `activation_count`, node working-memory weight —
|
||||
* PERSIST to the paged store and survive a restart from neuron.egm with
|
||||
* snapshot.json deleted. This is the "hebb-survives-restart" fix that gates the
|
||||
* live cutover.
|
||||
*
|
||||
* Same style as test_m3_parity.c: a REAL el-level harness linking the actual
|
||||
* el_runtime.c native engram builtins + engram_store.c, driving engram_node_full
|
||||
* / engram_connect / engram_activate_json / engram_save / engram_store_boot /
|
||||
* engram_store_checkpoint / engram_store_close directly from C. No EL interpreter.
|
||||
*
|
||||
* Modes (argv[1]), data dir (argv[2]):
|
||||
* pos_seed — ENGRAM_STORE=1: fresh store, seed a graph tuned so activation
|
||||
* co-activates a connected pair (edge hebb 0 -> ETA) and reinforces
|
||||
* nodes (activation_count 0 -> >=1, WM weight -> >0). Export the
|
||||
* post-activation resident graph to pre_reboot.json, then CHECKPOINT
|
||||
* (the M3.5 field-persist), then close.
|
||||
* pos_reboot— ENGRAM_STORE=1, snapshot.json deleted by runner: boot from
|
||||
* neuron.egm (WAL replay), export reboot.json, close. The values in
|
||||
* reboot.json are what actually survived the round-trip.
|
||||
* neg_seed — identical to pos_seed but WITHOUT the checkpoint field-persist
|
||||
* (negative control): activation mutations never reach the store.
|
||||
* neg_reboot— boot from neuron.egm, export neg_reboot.json, close.
|
||||
* offcheck — ENGRAM_STORE unset: seed+activate+checkpoint must NOT touch the
|
||||
* store (no neuron.egm, checkpoint returns 0).
|
||||
*
|
||||
* The pass/fail assertions live in run_m35_hebb_persist.sh (python over the JSON
|
||||
* exports): reboot.json must carry the learned hebb / activation_count and the
|
||||
* JSON-identical halved WM weight; neg_reboot.json must have LOST them.
|
||||
*/
|
||||
#include "el_runtime.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
extern int engram_store_enabled(void);
|
||||
extern el_val_t engram_store_boot(el_val_t data_dir);
|
||||
extern el_val_t engram_store_checkpoint(void);
|
||||
extern el_val_t engram_store_close(void);
|
||||
|
||||
static el_val_t S(const char* s){ return EL_STR(s); }
|
||||
static el_val_t F(double d){ return el_from_float(d); }
|
||||
|
||||
/* Two nodes with DISTINCT content (so the redundancy-suppression pass cannot
|
||||
* dedup one of them away) that both match the query strongly, wired by one
|
||||
* "associate" edge. A handful of weakly-related distractors make it a real
|
||||
* graph. On activation both A and B promote to working memory and co-activate,
|
||||
* so their edge's hebb rises from 0 to ENGRAM_HEBB_ETA. */
|
||||
static void build_seed(void){
|
||||
el_val_t a = engram_node_full(S("hebbian potentiation strengthens co-active memory links"),
|
||||
S("Concept"), S("hebb-a"), F(0.9), F(0.85), F(1.0), S("Semantic"),
|
||||
S("hebbian,memory,activation"));
|
||||
el_val_t b = engram_node_full(S("co-active memory links accrue hebbian associative weight"),
|
||||
S("Concept"), S("hebb-b"), F(0.9), F(0.85), F(1.0), S("Semantic"),
|
||||
S("hebbian,memory,weight"));
|
||||
el_val_t c = engram_node_full(S("unrelated culinary recipe for sourdough bread"),
|
||||
S("Fact"), S("distractor-1"), F(0.4), F(0.4), F(1.0), S("Semantic"),
|
||||
S("food"));
|
||||
el_val_t d = engram_node_full(S("the weather forecast predicts rain tomorrow afternoon"),
|
||||
S("Fact"), S("distractor-2"), F(0.4), F(0.4), F(1.0), S("Semantic"),
|
||||
S("weather"));
|
||||
engram_connect(a, b, F(0.8), S("associate")); /* the edge under test */
|
||||
engram_connect(a, c, F(0.3), S("associate"));
|
||||
engram_connect(b, d, F(0.3), S("associate"));
|
||||
}
|
||||
|
||||
static const char* QUERY =
|
||||
"hebbian potentiation co-active memory links associative weight";
|
||||
|
||||
static void export_graph(const char* dir, const char* name){
|
||||
char p[1024];
|
||||
snprintf(p, sizeof p, "%s/%s", dir, name);
|
||||
if (!engram_save(S(p))){ fprintf(stderr, "save %s failed\n", name); exit(2); }
|
||||
}
|
||||
|
||||
int main(int argc, char** argv){
|
||||
if (argc < 3){
|
||||
fprintf(stderr, "usage: %s <pos_seed|pos_reboot|neg_seed|neg_reboot|offcheck> <dir>\n", argv[0]);
|
||||
return 2;
|
||||
}
|
||||
const char* mode = argv[1];
|
||||
const char* dir = argv[2];
|
||||
|
||||
if (!strcmp(mode, "pos_seed") || !strcmp(mode, "neg_seed")){
|
||||
int persist = !strcmp(mode, "pos_seed");
|
||||
if (!engram_store_enabled()){ fprintf(stderr, "%s requires ENGRAM_STORE=1\n", mode); return 2; }
|
||||
if (!engram_store_boot(S(dir))){ fprintf(stderr, "store boot failed\n"); return 2; }
|
||||
build_seed();
|
||||
el_val_t act = engram_activate_json(S(QUERY), (el_val_t)3);
|
||||
(void)act;
|
||||
/* Capture the post-activation resident state BEFORE persisting/closing. */
|
||||
export_graph(dir, persist ? "pre_reboot.json" : "neg_pre.json");
|
||||
printf("[%s] nodes=%lld edges=%lld\n", mode,
|
||||
(long long)(int64_t)engram_node_count(),
|
||||
(long long)(int64_t)engram_edge_count());
|
||||
if (persist){
|
||||
if (!engram_store_checkpoint()){ fprintf(stderr, "checkpoint failed\n"); return 2; }
|
||||
}
|
||||
/* neg mode: NO field-persist checkpoint. engram_store_close still flushes
|
||||
* pages, but no store_put_* ran post-creation, so the store keeps the
|
||||
* pristine creation-time field values (hebb=0, activation_count=0). */
|
||||
engram_store_close();
|
||||
return 0;
|
||||
}
|
||||
if (!strcmp(mode, "pos_reboot") || !strcmp(mode, "neg_reboot")){
|
||||
if (!engram_store_enabled()){ fprintf(stderr, "%s requires ENGRAM_STORE=1\n", mode); return 2; }
|
||||
/* snapshot.json deleted by the runner — boot MUST come from neuron.egm. */
|
||||
if (!engram_store_boot(S(dir))){ fprintf(stderr, "reboot boot failed\n"); return 2; }
|
||||
export_graph(dir, !strcmp(mode, "pos_reboot") ? "reboot.json" : "neg_reboot.json");
|
||||
printf("[%s] nodes=%lld edges=%lld\n", mode,
|
||||
(long long)(int64_t)engram_node_count(),
|
||||
(long long)(int64_t)engram_edge_count());
|
||||
engram_store_close();
|
||||
return 0;
|
||||
}
|
||||
if (!strcmp(mode, "offcheck")){
|
||||
int en = engram_store_enabled();
|
||||
el_val_t boot = engram_store_boot(S(dir)); /* no-op with flag off */
|
||||
build_seed();
|
||||
engram_activate_json(S(QUERY), (el_val_t)3);
|
||||
el_val_t ck = engram_store_checkpoint(); /* must be a no-op */
|
||||
printf("[offcheck] enabled=%d boot=%lld checkpoint=%lld\n",
|
||||
en, (long long)(int64_t)boot, (long long)(int64_t)ck);
|
||||
return (en == 0 && (int64_t)boot == 0 && (int64_t)ck == 0) ? 0 : 1;
|
||||
}
|
||||
fprintf(stderr, "unknown mode %s\n", mode);
|
||||
return 2;
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
/* test_m3_parity.c — M3 JSON-parity gate for the ENGRAM_STORE wiring.
|
||||
*
|
||||
* This is a REAL el-level harness: it links the actual el_runtime.o (the soul's
|
||||
* native engram builtins) + engram_store.o and calls the engram_node family plus
|
||||
* engram_connect, engram_activate_json, engram_save, engram_store_boot directly. No EL interpreter
|
||||
* and no full soul build are needed — el_runtime.c compiles to a standalone .o
|
||||
* whose engram builtins operate on the process-global engram store, and the
|
||||
* string arena is inert unless el_request_start() is called, so the builtins are
|
||||
* callable straight from C (el_val_t is int64_t; EL_STR/EL_CSTR are pointer casts).
|
||||
*
|
||||
* Modes (argv[1]), data dir (argv[2]):
|
||||
* seed — ENGRAM_STORE unset: build a fixed seed graph, write snapshot.json +
|
||||
* off_graph.json (pristine, pre-activation), then activate → off_act.json.
|
||||
* on — ENGRAM_STORE=1: engram_store_boot(dir) imports snapshot.json ONCE into
|
||||
* neuron.egm and loads it resident; write on_graph.json, then activate →
|
||||
* on_act.json; checkpoint + close.
|
||||
* reboot — ENGRAM_STORE=1 with snapshot.json DELETED: boot must reload from
|
||||
* neuron.egm (WAL replay), never re-reading JSON; write reboot_graph.json.
|
||||
* offcheck — assert flag-off leaves the store untouched.
|
||||
*
|
||||
* The graph comparison (done by run_m3_parity.sh via python, modulo ordering) is
|
||||
* the deterministic gate; activation ids/promoted are compared as a robust set.
|
||||
*/
|
||||
#include "el_runtime.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/* Builtins the header declares are pulled in via el_runtime.h. The M3 additions
|
||||
* are not in the header yet, so declare them here. */
|
||||
extern int engram_store_enabled(void);
|
||||
extern el_val_t engram_store_boot(el_val_t data_dir);
|
||||
extern el_val_t engram_store_checkpoint(void);
|
||||
extern el_val_t engram_store_close(void);
|
||||
extern el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t label,
|
||||
el_val_t salience, el_val_t certainty, el_val_t confidence,
|
||||
el_val_t status, el_val_t tags, el_val_t layer_id);
|
||||
|
||||
static el_val_t S(const char* s){ return EL_STR(s); }
|
||||
static el_val_t F(double d){ return el_from_float(d); }
|
||||
|
||||
/* Build a fixed, deterministic seed graph: 12 nodes across two layers + 9 edges.
|
||||
* Content is chosen so an activation query has real matches to rank. */
|
||||
static void build_seed(void){
|
||||
/* core-identity layer (1) via engram_node_full */
|
||||
el_val_t n0 = engram_node_full(S("tiered storage engine design"), S("Concept"),
|
||||
S("storage-engine"), F(0.9), F(0.8), F(1.0), S("Semantic"), S("design,storage"));
|
||||
el_val_t n1 = engram_node_full(S("write-ahead log durability"), S("Concept"),
|
||||
S("wal"), F(0.85), F(0.75), F(1.0), S("Semantic"), S("wal,durability"));
|
||||
el_val_t n2 = engram_node_full(S("paged buffer pool with checkpointing"), S("Concept"),
|
||||
S("buffer-pool"), F(0.8), F(0.7), F(1.0), S("Semantic"), S("paging"));
|
||||
el_val_t n3 = engram_node_full(S("spreading activation over the graph"), S("Concept"),
|
||||
S("activation"), F(0.8), F(0.7), F(1.0), S("Semantic"), S("activation,graph"));
|
||||
el_val_t n4 = engram_node_full(S("hebbian co-activation potentiation"), S("Concept"),
|
||||
S("hebbian"), F(0.7), F(0.6), F(1.0), S("Semantic"), S("hebb"));
|
||||
el_val_t n5 = engram_node_full(S("crash recovery replays the log"), S("Concept"),
|
||||
S("recovery"), F(0.75), F(0.65), F(1.0), S("Semantic"), S("recovery,wal"));
|
||||
/* domain-knowledge layer (2) via engram_node_layered */
|
||||
el_val_t n6 = engram_node_layered(S("b-tree primary index id to location"), S("Fact"),
|
||||
S("btree"), F(0.7), F(0.6), F(1.0), S(""), S("index"), (el_val_t)2);
|
||||
el_val_t n7 = engram_node_layered(S("adjacency index for edge lookup"), S("Fact"),
|
||||
S("adjacency"), F(0.7), F(0.6), F(1.0), S(""), S("index,graph"), (el_val_t)2);
|
||||
el_val_t n8 = engram_node_layered(S("slotted pages hold tlv records"), S("Fact"),
|
||||
S("slotted-page"), F(0.65), F(0.55), F(1.0), S(""), S("format"), (el_val_t)2);
|
||||
el_val_t n9 = engram_node_full(S("memory tiers working semantic episodic"), S("Concept"),
|
||||
S("tiers"), F(0.7), F(0.6), F(1.0), S("Semantic"), S("tiers,memory"));
|
||||
el_val_t n10 = engram_node_full(S("embeddings enable nearest neighbour search"), S("Concept"),
|
||||
S("embeddings"), F(0.65), F(0.55), F(1.0), S("Semantic"), S("embeddings"));
|
||||
el_val_t n11 = engram_node_full(S("the durable engram is the mind's memory"), S("Belief"),
|
||||
S("engram"), F(0.95), F(0.9), F(1.0), S("Semantic"), S("engram,memory"));
|
||||
|
||||
engram_connect(n0, n1, F(0.8), S("depends-on"));
|
||||
engram_connect(n0, n2, F(0.8), S("depends-on"));
|
||||
engram_connect(n0, n3, F(0.7), S("enables"));
|
||||
engram_connect(n1, n5, F(0.9), S("enables"));
|
||||
engram_connect(n3, n4, F(0.6), S("triggers"));
|
||||
engram_connect(n2, n6, F(0.7), S("uses"));
|
||||
engram_connect(n3, n7, F(0.7), S("uses"));
|
||||
engram_connect(n0, n8, F(0.6), S("uses"));
|
||||
engram_connect(n11, n9, F(0.8), S("about"));
|
||||
engram_connect(n11, n10, F(0.5), S("about"));
|
||||
}
|
||||
|
||||
static void write_file(const char* path, const char* content){
|
||||
FILE* f = fopen(path, "wb");
|
||||
if (!f){ fprintf(stderr, "cannot open %s\n", path); exit(2); }
|
||||
if (content) fwrite(content, 1, strlen(content), f);
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
static const char* QUERY = "storage engine activation and the durable log";
|
||||
|
||||
int main(int argc, char** argv){
|
||||
if (argc < 3){ fprintf(stderr, "usage: %s <seed|on|reboot|offcheck> <dir>\n", argv[0]); return 2; }
|
||||
const char* mode = argv[1];
|
||||
const char* dir = argv[2];
|
||||
char p[1024];
|
||||
|
||||
if (!strcmp(mode, "seed")){
|
||||
if (engram_store_enabled()){ fprintf(stderr, "seed mode requires ENGRAM_STORE unset\n"); return 2; }
|
||||
build_seed();
|
||||
snprintf(p, sizeof p, "%s/snapshot.json", dir);
|
||||
if (!engram_save(S(p))){ fprintf(stderr, "seed save failed\n"); return 2; }
|
||||
snprintf(p, sizeof p, "%s/off_graph.json", dir);
|
||||
engram_save(S(p)); /* pristine off-path graph */
|
||||
el_val_t act = engram_activate_json(S(QUERY), (el_val_t)3);
|
||||
snprintf(p, sizeof p, "%s/off_act.json", dir);
|
||||
write_file(p, EL_CSTR(act));
|
||||
printf("[seed] nodes=%lld edges=%lld\n",
|
||||
(long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count());
|
||||
return 0;
|
||||
}
|
||||
if (!strcmp(mode, "on")){
|
||||
if (!engram_store_enabled()){ fprintf(stderr, "on mode requires ENGRAM_STORE=1\n"); return 2; }
|
||||
if (!engram_store_boot(S(dir))){ fprintf(stderr, "store boot failed\n"); return 2; }
|
||||
snprintf(p, sizeof p, "%s/on_graph.json", dir);
|
||||
engram_save(S(p)); /* export resident (== store) */
|
||||
/* Checkpoint the freshly-imported (pristine) graph — this is the state
|
||||
* the reboot comparison expects to round-trip. Under M3.5 a checkpoint
|
||||
* persists the resident graph's CURRENT field state, so it must run
|
||||
* BEFORE activation mutates fields in place; activation itself is
|
||||
* exercised below only for the activation-result-set parity check. The
|
||||
* M3.5 gate (test_m35_hebb_persist) separately proves that a checkpoint
|
||||
* taken AFTER activation durably carries the learned hebb/WM state. */
|
||||
engram_store_checkpoint();
|
||||
el_val_t act = engram_activate_json(S(QUERY), (el_val_t)3);
|
||||
snprintf(p, sizeof p, "%s/on_act.json", dir);
|
||||
write_file(p, EL_CSTR(act));
|
||||
printf("[on] nodes=%lld edges=%lld\n",
|
||||
(long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count());
|
||||
engram_store_close();
|
||||
return 0;
|
||||
}
|
||||
if (!strcmp(mode, "reboot")){
|
||||
if (!engram_store_enabled()){ fprintf(stderr, "reboot mode requires ENGRAM_STORE=1\n"); return 2; }
|
||||
/* snapshot.json has been deleted by the runner — boot MUST come from
|
||||
* neuron.egm (+ WAL replay), never re-reading JSON. */
|
||||
if (!engram_store_boot(S(dir))){ fprintf(stderr, "reboot boot failed\n"); return 2; }
|
||||
snprintf(p, sizeof p, "%s/reboot_graph.json", dir);
|
||||
engram_save(S(p));
|
||||
printf("[reboot] nodes=%lld edges=%lld\n",
|
||||
(long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count());
|
||||
engram_store_close();
|
||||
return 0;
|
||||
}
|
||||
if (!strcmp(mode, "offcheck")){
|
||||
/* ENGRAM_STORE unset: enabled()==0 and boot is a no-op returning 0. */
|
||||
int en = engram_store_enabled();
|
||||
el_val_t b = engram_store_boot(S(dir));
|
||||
printf("[offcheck] enabled=%d boot_ret=%lld\n", en, (long long)(int64_t)b);
|
||||
return (en == 0 && (int64_t)b == 0) ? 0 : 1;
|
||||
}
|
||||
fprintf(stderr, "unknown mode %s\n", mode);
|
||||
return 2;
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
/* test_m7_traversal.c — M7 index-driven activation traversal.
|
||||
*
|
||||
* Milestone 7 replaces the O(E) full adjacency rebuild that spreading activation
|
||||
* paid before every BFS with an incrementally-maintained per-node index, behind
|
||||
* the ENGRAM_STORE flag (flag-off = unchanged behavior). This harness links the
|
||||
* REAL el_runtime.c engram builtins (+ engram_store.c) and drives activation
|
||||
* directly — no EL interpreter, no store boot (the index optimization is a pure
|
||||
* in-RAM concern; the flag is read from the environment).
|
||||
*
|
||||
* Modes (argv[1]):
|
||||
* parity-off <dir> — ENGRAM_STORE unset: build a fixed graph, run a scripted
|
||||
* sequence of activations WITH mid-sequence edge/node
|
||||
* inserts, dump each activation's JSON to <dir>/off_actN.json.
|
||||
* parity-on <dir> — ENGRAM_STORE=1: identical graph + identical sequence,
|
||||
* dump to <dir>/on_actN.json. The runner asserts the off/on
|
||||
* files are BYTE-IDENTICAL (same activated set, weights,
|
||||
* ordering, hops, WM promotion).
|
||||
* perf <off|on> <dir> <nodes> <edges> <iters>
|
||||
* — build a large graph, then loop `iters` times doing
|
||||
* (add 1 edge + activate). Prints wall-time and the M7
|
||||
* instrumentation counters (rebuild calls / rebuild
|
||||
* edge-work / incremental appends).
|
||||
*
|
||||
* Writes ONLY under the caller-provided throwaway dir.
|
||||
*/
|
||||
#include "el_runtime.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
/* M7 instrumentation getters (test-only; defined in el_runtime.c). */
|
||||
extern int64_t engram_adj_rebuild_calls(void);
|
||||
extern int64_t engram_adj_rebuild_edge_work(void);
|
||||
extern int64_t engram_adj_incr_appends(void);
|
||||
extern double engram_adj_maint_seconds(void);
|
||||
extern void engram_adj_test_force_dirty(void);
|
||||
extern int engram_store_enabled(void);
|
||||
|
||||
static el_val_t S(const char* s){ return EL_STR(s); }
|
||||
static el_val_t F(double d){ return el_from_float(d); }
|
||||
|
||||
/* Deterministic LCG so off/on processes build byte-identical graphs. */
|
||||
static uint64_t g_rng = 0x9E3779B97F4A7C15ULL;
|
||||
static void rng_seed(uint64_t s){ g_rng = s ? s : 1; }
|
||||
static uint64_t rng_next(void){ g_rng = g_rng * 6364136223846793005ULL + 1442695040888963407ULL; return g_rng >> 17; }
|
||||
|
||||
static el_val_t* g_handles = NULL; /* node id handles from engram_node_full */
|
||||
static int64_t g_nnodes = 0;
|
||||
|
||||
static void write_file(const char* path, const char* content){
|
||||
FILE* f = fopen(path, "wb");
|
||||
if (!f){ fprintf(stderr, "cannot open %s\n", path); exit(2); }
|
||||
if (content) fwrite(content, 1, strlen(content), f);
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
/* Build `n` nodes whose content carries query-matchable tokens, then `m`
|
||||
* deterministic edges among them. Handles are retained for later connect. */
|
||||
static void build_graph(int64_t n, int64_t m){
|
||||
g_handles = malloc((size_t)n * sizeof(el_val_t));
|
||||
g_nnodes = n;
|
||||
static const char* topics[] = {
|
||||
"storage engine durable log", "spreading activation graph traversal",
|
||||
"hebbian potentiation memory", "buffer pool paging checkpoint",
|
||||
"adjacency index edge lookup", "working memory promotion",
|
||||
"b-tree primary index", "embeddings nearest neighbour" };
|
||||
for (int64_t i = 0; i < n; i++){
|
||||
char content[256];
|
||||
snprintf(content, sizeof content,
|
||||
"node %lld about %s and storage engine activation index",
|
||||
(long long)i, topics[(size_t)(i % 8)]);
|
||||
char label[32]; snprintf(label, sizeof label, "n%lld", (long long)i);
|
||||
g_handles[i] = engram_node_full(S(content), S("Concept"), S(label),
|
||||
F(0.7), F(0.6), F(1.0), S("Semantic"), S("storage,graph,index"));
|
||||
}
|
||||
for (int64_t k = 0; k < m; k++){
|
||||
int64_t a = (int64_t)(rng_next() % (uint64_t)n);
|
||||
int64_t b = (int64_t)(rng_next() % (uint64_t)n);
|
||||
if (a == b) b = (b + 1) % n;
|
||||
engram_connect(g_handles[a], g_handles[b], F(0.6), S("associate"));
|
||||
}
|
||||
}
|
||||
|
||||
static const char* Q1 = "storage engine activation and the durable log";
|
||||
static const char* Q2 = "adjacency index graph traversal";
|
||||
|
||||
/* One scripted activation with an optional forced full-rebuild first. */
|
||||
static el_val_t act(const char* q, int depth, int force_rebuild){
|
||||
if (force_rebuild) engram_adj_test_force_dirty();
|
||||
return engram_activate_json(S(q), (el_val_t)depth);
|
||||
}
|
||||
|
||||
/* Run the scripted parity sequence and dump each activation JSON. `tag` names
|
||||
* the output set. When force_rebuild is set, every activation first forces the
|
||||
* O(E) full-rebuild path (the pre-M7 "scan" behavior); otherwise the M7
|
||||
* incremental index is used. The graph build + query sequence are byte-for-byte
|
||||
* deterministic, so any difference between two runs is attributable solely to
|
||||
* the difference in adjacency maintenance (and/or the ENGRAM_STORE flag). */
|
||||
static int run_parity(const char* dir, const char* tag, int force_rebuild){
|
||||
char p[1024];
|
||||
rng_seed(0xC0FFEE123ULL);
|
||||
build_graph(60, 140);
|
||||
|
||||
el_val_t a1 = act(Q1, 3, force_rebuild);
|
||||
snprintf(p, sizeof p, "%s/%s_act1.json", dir, tag); write_file(p, EL_CSTR(a1));
|
||||
|
||||
/* Mutate the graph BETWEEN activations: this is exactly where the M7 path
|
||||
* appends incrementally while the rebuild path marks dirty + fully rebuilds.
|
||||
* Parity must hold across this divergence in HOW the index is maintained. */
|
||||
engram_connect(g_handles[0], g_handles[7], F(0.8), S("depends-on"));
|
||||
engram_connect(g_handles[7], g_handles[23], F(0.7), S("enables"));
|
||||
engram_connect(g_handles[23], g_handles[41],F(0.5), S("uses"));
|
||||
el_val_t hnew = engram_node_full(S("freshly minted storage index node about activation"),
|
||||
S("Concept"), S("nnew"), F(0.8), F(0.7), F(1.0), S("Semantic"), S("storage,index"));
|
||||
engram_connect(g_handles[0], hnew, F(0.9), S("about"));
|
||||
|
||||
el_val_t a2 = act(Q1, 3, force_rebuild);
|
||||
snprintf(p, sizeof p, "%s/%s_act2.json", dir, tag); write_file(p, EL_CSTR(a2));
|
||||
el_val_t a3 = act(Q2, 2, force_rebuild);
|
||||
snprintf(p, sizeof p, "%s/%s_act3.json", dir, tag); write_file(p, EL_CSTR(a3));
|
||||
el_val_t a4 = act(Q1, 3, force_rebuild);
|
||||
snprintf(p, sizeof p, "%s/%s_act4.json", dir, tag); write_file(p, EL_CSTR(a4));
|
||||
|
||||
printf("[parity-%s] enabled=%d force_rebuild=%d nodes=%lld edges=%lld "
|
||||
"rebuilds=%lld rebuild_edge_work=%lld incr_appends=%lld\n",
|
||||
tag, engram_store_enabled(), force_rebuild,
|
||||
(long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count(),
|
||||
(long long)engram_adj_rebuild_calls(), (long long)engram_adj_rebuild_edge_work(),
|
||||
(long long)engram_adj_incr_appends());
|
||||
return 0;
|
||||
}
|
||||
|
||||
static double now_sec(void){
|
||||
struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
|
||||
return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9;
|
||||
}
|
||||
|
||||
static int run_perf(const char* dir, const char* tag, int64_t n, int64_t m, int64_t iters){
|
||||
(void)dir;
|
||||
rng_seed(0xBEEF7777ULL);
|
||||
double t_build0 = now_sec();
|
||||
build_graph(n, m);
|
||||
double t_build = now_sec() - t_build0;
|
||||
|
||||
int64_t rb0 = engram_adj_rebuild_calls();
|
||||
int64_t rw0 = engram_adj_rebuild_edge_work();
|
||||
int64_t ap0 = engram_adj_incr_appends();
|
||||
double mt0 = engram_adj_maint_seconds();
|
||||
|
||||
double t0 = now_sec();
|
||||
for (int64_t it = 0; it < iters; it++){
|
||||
/* One structural mutation per query — the curiosity-loop cadence that
|
||||
* makes the OLD path rebuild the whole adjacency before every BFS. */
|
||||
int64_t a = (int64_t)(rng_next() % (uint64_t)n);
|
||||
int64_t b = (int64_t)(rng_next() % (uint64_t)n);
|
||||
if (a == b) b = (b + 1) % n;
|
||||
engram_connect(g_handles[a], g_handles[b], F(0.6), S("associate"));
|
||||
el_val_t r = engram_activate_json(S(Q1), (el_val_t)2);
|
||||
(void)r;
|
||||
}
|
||||
double elapsed = now_sec() - t0;
|
||||
|
||||
double maint = engram_adj_maint_seconds() - mt0;
|
||||
printf("[perf-%s] flag=%d nodes=%lld edges=%lld iters=%lld build=%.3fs "
|
||||
"loop=%.3fs per_query=%.3fms adj_maint=%.4fs adj_maint_per_query=%.4fms | "
|
||||
"rebuilds=%lld rebuild_edge_work=%lld incr_appends=%lld\n",
|
||||
tag, engram_store_enabled(),
|
||||
(long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count(),
|
||||
(long long)iters, t_build, elapsed, (elapsed / (double)iters) * 1e3,
|
||||
maint, (maint / (double)iters) * 1e3,
|
||||
(long long)(engram_adj_rebuild_calls() - rb0),
|
||||
(long long)(engram_adj_rebuild_edge_work() - rw0),
|
||||
(long long)(engram_adj_incr_appends() - ap0));
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char** argv){
|
||||
if (argc < 3){ fprintf(stderr, "usage: %s <parity-off|parity-on|perf> ...\n", argv[0]); return 2; }
|
||||
const char* mode = argv[1];
|
||||
|
||||
if (!strcmp(mode, "parity-off")){
|
||||
/* flag-off, rebuild path = today's scan behavior (the baseline). */
|
||||
if (engram_store_enabled()){ fprintf(stderr, "parity-off requires ENGRAM_STORE unset\n"); return 2; }
|
||||
return run_parity(argv[2], "off", 0);
|
||||
}
|
||||
if (!strcmp(mode, "parity-on-rebuild")){
|
||||
/* flag-on, but force the O(E) rebuild before each activation. */
|
||||
if (!engram_store_enabled()){ fprintf(stderr, "parity-on-rebuild requires ENGRAM_STORE=1\n"); return 2; }
|
||||
return run_parity(argv[2], "onrb", 1);
|
||||
}
|
||||
if (!strcmp(mode, "parity-on-incr")){
|
||||
/* flag-on, M7 incremental index (the code path under test). */
|
||||
if (!engram_store_enabled()){ fprintf(stderr, "parity-on-incr requires ENGRAM_STORE=1\n"); return 2; }
|
||||
return run_parity(argv[2], "onincr", 0);
|
||||
}
|
||||
if (!strcmp(mode, "perf")){
|
||||
/* perf <off|on> <dir> <nodes> <edges> <iters> */
|
||||
if (argc < 7){ fprintf(stderr, "usage: %s perf <off|on> <dir> <nodes> <edges> <iters>\n", argv[0]); return 2; }
|
||||
const char* tag = argv[2];
|
||||
int64_t n = strtoll(argv[4], NULL, 10);
|
||||
int64_t m = strtoll(argv[5], NULL, 10);
|
||||
int64_t iters = strtoll(argv[6], NULL, 10);
|
||||
return run_perf(argv[3], tag, n, m, iters);
|
||||
}
|
||||
fprintf(stderr, "unknown mode %s\n", mode);
|
||||
return 2;
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
/* Closed-form unit tests for the REASONING layer (engram_reason.c). All inputs are
|
||||
* hand-built synthetic descriptors whose answers are known in closed form. Every
|
||||
* reasoning MODE is proven, not declared. ASan/UBSan target. */
|
||||
#include "engram_reason.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
|
||||
static int failures = 0, checks = 0;
|
||||
static void ok(const char* what, int cond) {
|
||||
checks++;
|
||||
if (!cond) { failures++; printf(" FAIL: %s\n", what); }
|
||||
else printf(" ok: %s\n", what);
|
||||
}
|
||||
static void approx(const char* what, double got, double exp, double tol) {
|
||||
ok(what, fabs(got - exp) <= tol);
|
||||
if (fabs(got - exp) > tol) printf(" got=%.9g exp=%.9g\n", got, exp);
|
||||
}
|
||||
|
||||
/* ── descriptor builders (mirror scratchpad/test_geo_ops.c) ─────────────────── */
|
||||
static float* vec(const double* v, int dim) {
|
||||
float* f = malloc((size_t)dim * sizeof(float));
|
||||
for (int i = 0; i < dim; i++) f[i] = (float)v[i];
|
||||
return f;
|
||||
}
|
||||
static GeoDescriptor* mk(int dim, const double* centroid,
|
||||
int n_axes, const double* axis_flat, const double* extents,
|
||||
int n_members, const char** ids, double total_var) {
|
||||
GeoDescriptor* g = calloc(1, sizeof(GeoDescriptor));
|
||||
g->dim = dim;
|
||||
g->centroid = centroid ? vec(centroid, dim) : NULL;
|
||||
g->global_mean = NULL;
|
||||
g->n_axes = n_axes;
|
||||
g->axes = n_axes ? calloc((size_t)n_axes, sizeof(GeoAxis)) : NULL;
|
||||
double tr = 0;
|
||||
for (int k = 0; k < n_axes; k++) {
|
||||
g->axes[k].axis = vec(&axis_flat[(size_t)k * dim], dim);
|
||||
g->axes[k].extent = extents[k];
|
||||
tr += extents[k] * extents[k];
|
||||
}
|
||||
g->total_variance = (total_var >= 0) ? total_var : tr;
|
||||
g->radius = sqrt(g->total_variance > 0 ? g->total_variance : 0);
|
||||
g->n_members = n_members; g->n_embedded = n_members;
|
||||
g->members = n_members ? calloc((size_t)n_members, sizeof(GeoMember)) : NULL;
|
||||
for (int i = 0; i < n_members; i++) {
|
||||
g->members[i].id = strdup(ids[i]);
|
||||
g->members[i].membership = 1.0;
|
||||
g->members[i].centrality = (double)(n_members - i);
|
||||
g->members[i].embedded = 1;
|
||||
}
|
||||
g->hub_id = n_members ? strdup(ids[0]) : strdup("");
|
||||
g->k_core = 1; g->co_registration = 0.0; g->n_edges = 0; g->edges = NULL;
|
||||
return g;
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
printf("== REASONING layer unit tests ==\n");
|
||||
|
||||
/* ══════════════════ ANALOGY — recover an affine A→B, apply to C ══════════ */
|
||||
/* A→B is a +90° rotation in the e0-e1 plane ((x,y)→(-y,x)) plus a +5 shift in e2.
|
||||
* A frame = (e0,e1); B frame = rotated (e1,-e0); cB = R·cA + t. Predict D from C. */
|
||||
{
|
||||
int dim = 4;
|
||||
double cA[4] = {1,0,0,0};
|
||||
double cB[4] = {0,1,5,0}; /* R·(1,0,0,0)=(0,1,0,0) + (0,0,5,0) */
|
||||
double cC[4] = {2,0,0,0};
|
||||
double axA[8] = {1,0,0,0, 0,1,0,0}; double exA[2] = {1,1};
|
||||
double axB[8] = {0,1,0,0, -1,0,0,0}; double exB[2] = {1,1}; /* R·e0, R·e1 */
|
||||
double axC[8] = {1,0,0,0, 0,1,0,0}; double exC[2] = {1,1};
|
||||
const char* idA[1] = {"A"}, *idB[1] = {"B"}, *idC[1] = {"C"};
|
||||
GeoDescriptor* A = mk(dim, cA, 2, axA, exA, 1, idA, -1);
|
||||
GeoDescriptor* B = mk(dim, cB, 2, axB, exB, 1, idB, -1);
|
||||
GeoDescriptor* C = mk(dim, cC, 2, axC, exC, 1, idC, -1);
|
||||
/* candidates: the true D + two distractors. true D = R·cC + t = (0,2,5,0). */
|
||||
double d_true[4] = {0,2,5,0}, d_far1[4] = {9,9,9,9}, d_far2[4] = {0,0,0,0};
|
||||
const char* idD[1] = {"Dt"}, *idF1[1] = {"F1"}, *idF2[1] = {"F2"};
|
||||
GeoDescriptor* Dt = mk(dim, d_true, 0, NULL, NULL, 1, idD, 0.0);
|
||||
GeoDescriptor* F1 = mk(dim, d_far1, 0, NULL, NULL, 1, idF1, 0.0);
|
||||
GeoDescriptor* F2 = mk(dim, d_far2, 0, NULL, NULL, 1, idF2, 0.0);
|
||||
const GeoDescriptor* cand[3] = {F1, Dt, F2}; /* true one at index 1 */
|
||||
GeoAnalogyResult res;
|
||||
int rc = engram_reason_analogy(A, B, C, cand, 3, &res);
|
||||
ok("analogy returns 0", rc == 0);
|
||||
printf("[analogy] residual=%.6f mapped=(%.4f,%.4f,%.4f,%.4f) best=%d bd=%.5f\n",
|
||||
res.analogy_residual, res.mapped_point[0], res.mapped_point[1],
|
||||
res.mapped_point[2], res.mapped_point[3], res.best, res.best_distance);
|
||||
approx("procrustes residual ~0", res.analogy_residual, 0.0, 1e-4);
|
||||
approx("mapped.x=0", res.mapped_point[0], 0.0, 1e-4);
|
||||
approx("mapped.y=2", res.mapped_point[1], 2.0, 1e-4);
|
||||
approx("mapped.z(e2)=5", res.mapped_point[2], 5.0, 1e-4);
|
||||
ok("nearest candidate = true D (idx 1)", res.best == 1);
|
||||
approx("best distance ~0", res.best_distance, 0.0, 1e-3);
|
||||
engram_reason_analogy_free(&res);
|
||||
engram_geo_free(A); engram_geo_free(B); engram_geo_free(C);
|
||||
engram_geo_free(Dt); engram_geo_free(F1); engram_geo_free(F2);
|
||||
}
|
||||
|
||||
/* ══════════════════ INDUCTION — recover a shared subspace + membership ═══ */
|
||||
/* 3 examples all spread over span(e0,e1) (ext 1 & 0.8), each with a small
|
||||
* idiosyncratic axis (e2 or e3, ext 0.2). Centroids all 0. The induced rule's
|
||||
* top-2 axes must lie in span(e0,e1); a held-out in-plane point fits, an
|
||||
* off-subspace point does not. */
|
||||
{
|
||||
int dim = 4;
|
||||
double c0[4] = {0,0,0,0};
|
||||
double axsh[8] = {1,0,0,0, 0,1,0,0}; double exsh[2] = {1.0, 0.8};
|
||||
double ax1[12] = {1,0,0,0, 0,1,0,0, 0,0,1,0}; double ex1[3] = {1.0,0.8,0.2}; /* +e2 */
|
||||
double ax2[12] = {1,0,0,0, 0,1,0,0, 0,0,0,1}; double ex2[3] = {1.0,0.8,0.2}; /* +e3 */
|
||||
const char* i1[2] = {"e1a","e1b"}, *i2[2] = {"e2a","e2b"}, *i3[2] = {"e3a","e3b"};
|
||||
GeoDescriptor* E1 = mk(dim, c0, 3, ax1, ex1, 2, i1, -1);
|
||||
GeoDescriptor* E2 = mk(dim, c0, 3, ax2, ex2, 2, i2, -1);
|
||||
GeoDescriptor* E3 = mk(dim, c0, 2, axsh, exsh, 2, i3, -1);
|
||||
const GeoDescriptor* ex[3] = {E1, E2, E3};
|
||||
GeoInduction ind;
|
||||
int rc = engram_reason_induce(ex, 3, 8, 1.0, &ind);
|
||||
ok("induce returns 0", rc == 0);
|
||||
printf("[induction] rule n_axes=%d ext0=%.4f ext1=%.4f\n",
|
||||
ind.rule->n_axes, ind.rule->n_axes > 0 ? ind.rule->axes[0].extent : 0,
|
||||
ind.rule->n_axes > 1 ? ind.rule->axes[1].extent : 0);
|
||||
/* top-2 axes lie in span(e0,e1): their e2,e3 components ~0. */
|
||||
int inplane = 1;
|
||||
for (int k = 0; k < 2 && k < ind.rule->n_axes; k++) {
|
||||
const float* a = ind.rule->axes[k].axis;
|
||||
printf(" axis%d=(%.3f,%.3f,%.3f,%.3f) ext=%.4f\n", k, a[0],a[1],a[2],a[3], ind.rule->axes[k].extent);
|
||||
if (fabs(a[2]) > 0.06 || fabs(a[3]) > 0.06) inplane = 0;
|
||||
}
|
||||
ok("induced top-2 axes lie in shared span(e0,e1)", inplane);
|
||||
approx("dominant extent ~1.0", ind.rule->axes[0].extent, 1.0, 0.06);
|
||||
approx("second extent ~0.8", ind.rule->axes[1].extent, 0.8, 0.06);
|
||||
/* membership: in-plane near-centroid positive fits; off-subspace negative doesn't. */
|
||||
float xpos[4] = {0.3f, -0.2f, 0, 0};
|
||||
float xneg[4] = {0, 0, 3.0f, 0}; /* large along e2 — outside the rule */
|
||||
float xfar[4] = {5.0f, 0, 0, 0}; /* in-plane but far — Mahalanobis blows up */
|
||||
double mp = engram_reason_membership(&ind, xpos);
|
||||
double mn = engram_reason_membership(&ind, xneg);
|
||||
double mf = engram_reason_membership(&ind, xfar);
|
||||
printf("[induction] membership pos=%.4f neg=%.4f far=%.4f\n", mp, mn, mf);
|
||||
ok("held-out positive fits (>0.5)", mp > 0.5);
|
||||
ok("off-subspace negative rejected (<0.3)", mn < 0.3);
|
||||
ok("in-plane-but-far rejected (<0.3)", mf < 0.3);
|
||||
ok("positive fits far better than negative", mp > mn + 0.4);
|
||||
engram_reason_induction_free(&ind);
|
||||
engram_geo_free(E1); engram_geo_free(E2); engram_geo_free(E3);
|
||||
}
|
||||
|
||||
/* ══════════════════ ABDUCTION — pick the best-explaining structure ═══════ */
|
||||
/* obs planted near H1's centroid among 3 candidate structures. */
|
||||
{
|
||||
int dim = 4;
|
||||
double h0[4] = {0,0,0,0}, h1[4] = {5,0,0,0}, h2[4] = {0,5,0,0};
|
||||
double ax[8] = {1,0,0,0, 0,1,0,0}; double ex[2] = {1,1};
|
||||
const char* n0[1] = {"H0"}, *n1[1] = {"H1"}, *n2[1] = {"H2"};
|
||||
GeoDescriptor* H0 = mk(dim, h0, 2, ax, ex, 1, n0, -1);
|
||||
GeoDescriptor* H1 = mk(dim, h1, 2, ax, ex, 1, n1, -1);
|
||||
GeoDescriptor* H2 = mk(dim, h2, 2, ax, ex, 1, n2, -1);
|
||||
const GeoDescriptor* H[3] = {H0, H1, H2};
|
||||
float obs[4] = {5.2f, 0.1f, 0, 0}; /* sits inside H1 */
|
||||
GeoAbduction ab;
|
||||
int rc = engram_reason_abduce(obs, dim, H, 3, 1.0, &ab);
|
||||
ok("abduce returns 0", rc == 0);
|
||||
printf("[abduction] best=%d best_score=%.4f rank=[%d,%d,%d] d=[%.3f,%.3f,%.3f]\n",
|
||||
ab.best, ab.best_score, ab.rank[0], ab.rank[1], ab.rank[2],
|
||||
ab.distances[0], ab.distances[1], ab.distances[2]);
|
||||
ok("best explanation = H1", ab.best == 1);
|
||||
ok("rank[0] = H1", ab.rank[0] == 1);
|
||||
ok("H1 has smallest distance", ab.distances[1] < ab.distances[0] && ab.distances[1] < ab.distances[2]);
|
||||
engram_reason_abduction_free(&ab);
|
||||
engram_geo_free(H0); engram_geo_free(H1); engram_geo_free(H2);
|
||||
}
|
||||
|
||||
/* ══════════════════ CAUSAL — direction + confounder flag ═════════════════ */
|
||||
/* Chain A→B→C along e0 (temporal 1<2<3). Confounder Z (e1) injects into A and
|
||||
* drives D (t=4). A–D correlate only via Z ⇒ must be flagged CONFOUNDED. */
|
||||
{
|
||||
int dim = 4;
|
||||
double cA[4] = {1,1,0,0}; /* e0 (chain) + e1 (confounder leak) */
|
||||
double cB[4] = {1,0,0,0}; /* e0 */
|
||||
double cC[4] = {2,0,0,0}; /* e0 */
|
||||
double cD[4] = {0,1,0,0}; /* e1 only — driven by Z */
|
||||
double cZ[4] = {0,1,0,0}; /* confounder centroid */
|
||||
double axZ[4] = {0,1,0,0}; double exZ[1] = {1}; /* Z's subspace = e1 */
|
||||
const char* idA[1]={"A"},*idB[1]={"B"},*idC[1]={"C"},*idD[1]={"D"},*idZ[1]={"Z"};
|
||||
GeoDescriptor* A = mk(dim, cA, 0, NULL, NULL, 1, idA, 0.0);
|
||||
GeoDescriptor* B = mk(dim, cB, 0, NULL, NULL, 1, idB, 0.0);
|
||||
GeoDescriptor* C = mk(dim, cC, 0, NULL, NULL, 1, idC, 0.0);
|
||||
GeoDescriptor* D = mk(dim, cD, 0, NULL, NULL, 1, idD, 0.0);
|
||||
GeoDescriptor* Z = mk(dim, cZ, 1, axZ, exZ, 1, idZ, -1);
|
||||
const GeoDescriptor* conf[1] = {Z};
|
||||
|
||||
GeoCausal ab, bc, ad, bd;
|
||||
engram_reason_causal(A, B, conf, 1, /*t*/1, 2, 0.5, &ab);
|
||||
engram_reason_causal(B, C, conf, 1, 2, 3, 0.5, &bc);
|
||||
engram_reason_causal(A, D, conf, 1, 1, 4, 0.5, &ad);
|
||||
engram_reason_causal(B, D, conf, 1, 2, 4, 0.5, &bd);
|
||||
printf("[causal] A->B: raw=%.3f ctrl=%.3f dir=%d verdict=%d strength=%.3f\n",
|
||||
ab.assoc_raw, ab.assoc_controlled, ab.temporal_dir, ab.verdict, ab.strength);
|
||||
printf("[causal] B->C: raw=%.3f ctrl=%.3f dir=%d verdict=%d\n", bc.assoc_raw, bc.assoc_controlled, bc.temporal_dir, bc.verdict);
|
||||
printf("[causal] A--D: raw=%.3f ctrl=%.3f dir=%d verdict=%d confounded=%d\n",
|
||||
ad.assoc_raw, ad.assoc_controlled, ad.temporal_dir, ad.verdict, ad.confounded);
|
||||
printf("[causal] B--D: raw=%.3f verdict=%d\n", bd.assoc_raw, bd.verdict);
|
||||
ok("A->B DIRECTED", ab.verdict == GEO_CAUSAL_DIRECTED);
|
||||
ok("A->B direction A precedes B", ab.temporal_dir == 1);
|
||||
ok("A->B association survives control (ctrl high)", ab.assoc_controlled > 0.6);
|
||||
ok("B->C DIRECTED", bc.verdict == GEO_CAUSAL_DIRECTED);
|
||||
ok("A--D CONFOUNDED (flagged)", ad.verdict == GEO_CAUSAL_CONFOUNDED && ad.confounded == 1);
|
||||
ok("A--D raw correlated but control kills it", ad.assoc_raw > 0.6 && ad.assoc_controlled < 0.2);
|
||||
ok("B--D NONE (no association at all)", bd.verdict == GEO_CAUSAL_NONE);
|
||||
engram_geo_free(A); engram_geo_free(B); engram_geo_free(C); engram_geo_free(D); engram_geo_free(Z);
|
||||
}
|
||||
|
||||
/* ══════════════════ PLANNING — geodesic path along a curved manifold ═════ */
|
||||
/* 6 neighborhoods on a semicircle (radius 10). Consecutive chord ~6.18,
|
||||
* skip-one ~11.76, endpoints ~20. neighbor_radius=7 admits only consecutive
|
||||
* hops ⇒ the plan must traverse the whole arc 0→1→2→3→4→5. */
|
||||
{
|
||||
int dim = 4; int N = 6; double R = 10.0;
|
||||
GeoDescriptor* nodes[6];
|
||||
char nm[6][8];
|
||||
for (int k = 0; k < N; k++) {
|
||||
double th = M_PI * (double)k / (double)(N - 1);
|
||||
double c[4] = { R * cos(th), R * sin(th), 0, 0 };
|
||||
snprintf(nm[k], sizeof nm[k], "n%d", k);
|
||||
const char* id[1] = { nm[k] };
|
||||
nodes[k] = mk(dim, c, 0, NULL, NULL, 1, id, 0.0);
|
||||
}
|
||||
const GeoDescriptor* cn[6];
|
||||
for (int k = 0; k < N; k++) cn[k] = nodes[k];
|
||||
GeoPlan plan;
|
||||
int rc = engram_reason_plan(cn, N, 0, 5, 7.0, 0, &plan);
|
||||
ok("plan returns 0", rc == 0);
|
||||
printf("[planning] reached=%d len=%d cost=%.4f path=[", plan.reached, plan.path_len, plan.total_cost);
|
||||
for (int i = 0; i < plan.path_len; i++) printf("%s%d", i ? "," : "", plan.path[i]);
|
||||
printf("]\n");
|
||||
ok("goal reached", plan.reached == 1);
|
||||
ok("path length = 6 (full arc)", plan.path_len == 6);
|
||||
int monotone = (plan.path_len == 6);
|
||||
for (int i = 0; i < plan.path_len; i++) if (plan.path[i] != i) monotone = 0;
|
||||
ok("path = 0,1,2,3,4,5 (the geodesic)", monotone);
|
||||
/* arc cost ~ 5 * 6.18 = 30.9, and strictly longer than the 20-unit chord. */
|
||||
approx("arc cost ~30.9", plan.total_cost, 30.9, 0.6);
|
||||
ok("arc longer than straight chord (20)", plan.total_cost > 20.0);
|
||||
engram_reason_plan_free(&plan);
|
||||
|
||||
/* negative control: radius too small to connect anything ⇒ unreachable. */
|
||||
GeoPlan p2;
|
||||
engram_reason_plan(cn, N, 0, 5, 1.0, 0, &p2);
|
||||
ok("unreachable when radius < min edge", p2.reached == 0);
|
||||
engram_reason_plan_free(&p2);
|
||||
for (int k = 0; k < N; k++) engram_geo_free(nodes[k]);
|
||||
}
|
||||
|
||||
printf("\n== %d checks, %d failures ==\n", checks, failures);
|
||||
return failures ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
/* test_scan_collision.c — regression gate for the "saved but not findable" bug.
|
||||
*
|
||||
* ROOT CAUSE UNDER TEST: store_scan_nodes / store_scan_edges (the boot-load
|
||||
* path that populates the resident in-RAM graph — engram_store_boot ->
|
||||
* eg_load_node_cb) deduplicated emitted records by their 64-bit id_hash
|
||||
* (FNV-1a-64), NOT by the full id string. Two DISTINCT ids that collide under
|
||||
* id_hash therefore emitted only the FIRST: the second node/edge was durably
|
||||
* present in neuron.egm (store_get_node finds it), physically on a live page,
|
||||
* yet was SILENTLY DROPPED from the resident load. After any store reopen it
|
||||
* was unretrievable by id, absent from lexical search, and missing from the
|
||||
* recent list — exactly the reported symptom.
|
||||
*
|
||||
* The two ids below are real FNV-1a-64 collisions (found offline via Brent's
|
||||
* cycle detection over fnv1a(hex16(x))); both hash to 0x15141fdadfa24abe.
|
||||
*
|
||||
* Pure C. Writes ONLY under a throwaway /tmp dir. Never touches ~/.neuron.
|
||||
*/
|
||||
#include "../../lang/runtime/engram_store.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
static int g_pass = 0, g_fail = 0;
|
||||
static void ok(const char* name, int cond){
|
||||
printf(" [%s] %s\n", cond ? "PASS" : "FAIL", name);
|
||||
if (cond) g_pass++; else g_fail++;
|
||||
}
|
||||
|
||||
/* Confirmed FNV-1a-64 collision (distinct strings, equal id_hash). */
|
||||
#define ID_A "d2c61ec7d015dc98"
|
||||
#define ID_B "bf85e965a2aefbdd"
|
||||
|
||||
static uint64_t fnv1a(const char* s){
|
||||
uint64_t h = 1469598103934665603ULL;
|
||||
for (; *s; ++s){ h ^= (uint8_t)*s; h *= 1099511628211ULL; }
|
||||
return h;
|
||||
}
|
||||
|
||||
static char g_dir[512];
|
||||
static void mk_dir(void){
|
||||
snprintf(g_dir, sizeof g_dir, "/tmp/engram-scancol-%d", (int)getpid());
|
||||
mkdir(g_dir, 0700);
|
||||
}
|
||||
|
||||
/* ── scan collectors: record which ids the boot-load scan actually emits ── */
|
||||
typedef struct { const char* want[8]; int seen[8]; int nwant; int total; } Collect;
|
||||
static void node_cb(const StoreNode* n, void* ctx){
|
||||
Collect* c = ctx; c->total++;
|
||||
for (int i=0;i<c->nwant;i++) if (n->id && strcmp(n->id, c->want[i])==0) c->seen[i]=1;
|
||||
}
|
||||
static void edge_cb(const StoreEdge* e, void* ctx){
|
||||
Collect* c = ctx; c->total++;
|
||||
for (int i=0;i<c->nwant;i++) if (e->id && strcmp(e->id, c->want[i])==0) c->seen[i]=1;
|
||||
}
|
||||
|
||||
static void mk_node(StoreNode* n, const char* id, const char* content){
|
||||
memset(n, 0, sizeof *n);
|
||||
n->id = strdup(id);
|
||||
n->content = strdup(content);
|
||||
n->node_type = strdup("Memory");
|
||||
n->label = strdup(content);
|
||||
n->tier = strdup("Working");
|
||||
n->tags = strdup("");
|
||||
n->metadata = strdup("{}");
|
||||
n->salience = 0.5; n->importance = 0.5; n->confidence = 1.0;
|
||||
n->created_at = 1700000000000LL; n->updated_at = 1700000000000LL;
|
||||
n->last_activated = 1700000000000LL;
|
||||
}
|
||||
static void mk_edge(StoreEdge* e, const char* id, const char* from, const char* to){
|
||||
memset(e, 0, sizeof *e);
|
||||
e->id = strdup(id); e->from_id = strdup(from); e->to_id = strdup(to);
|
||||
e->relation = strdup("assoc"); e->metadata = strdup("{}");
|
||||
e->weight = 1.0; e->confidence = 1.0;
|
||||
e->created_at = 1700000000000LL; e->updated_at = 1700000000000LL;
|
||||
}
|
||||
|
||||
int main(void){
|
||||
mk_dir();
|
||||
printf("== scan-collision regression (saved-but-not-findable) ==\n");
|
||||
printf(" id_hash(%s) = %016llx\n", ID_A, (unsigned long long)fnv1a(ID_A));
|
||||
printf(" id_hash(%s) = %016llx\n", ID_B, (unsigned long long)fnv1a(ID_B));
|
||||
ok("precondition: the two ids genuinely collide under id_hash",
|
||||
fnv1a(ID_A) == fnv1a(ID_B) && strcmp(ID_A, ID_B) != 0);
|
||||
|
||||
/* ---- Control: a single node survives a full store round-trip. ---- */
|
||||
{
|
||||
EngramPagedStore* s = engram_open(g_dir);
|
||||
StoreNode n; mk_node(&n, ID_A, "alpha distinctiveword");
|
||||
store_put_node(s, &n);
|
||||
engram_close(s); /* checkpoint + close */
|
||||
|
||||
EngramPagedStore* r = engram_open(g_dir);
|
||||
StoreNode got;
|
||||
ok("control: single node found by id after reopen", store_get_node(r, ID_A, &got)==1);
|
||||
if (0) {} else store_node_free(&got);
|
||||
Collect c = {{ID_A}, {0}, 1, 0};
|
||||
store_scan_nodes(r, node_cb, &c);
|
||||
ok("control: single node emitted by boot-load scan", c.seen[0]==1);
|
||||
engram_close(r);
|
||||
store_node_free(&n);
|
||||
}
|
||||
|
||||
/* ---- Bug: two id-hash-colliding NODES, both durable, both must load. ---- */
|
||||
{
|
||||
char dir2[600]; snprintf(dir2, sizeof dir2, "%s/nodes", g_dir); mkdir(dir2, 0700);
|
||||
EngramPagedStore* s = engram_open(dir2);
|
||||
StoreNode a, b;
|
||||
mk_node(&a, ID_A, "alpha distinctiveword-A");
|
||||
mk_node(&b, ID_B, "beta distinctiveword-B");
|
||||
store_put_node(s, &a);
|
||||
store_put_node(s, &b);
|
||||
engram_close(s);
|
||||
store_node_free(&a); store_node_free(&b);
|
||||
|
||||
EngramPagedStore* r = engram_open(dir2);
|
||||
/* Both are individually durable (store_get_node disambiguates by strcmp). */
|
||||
StoreNode ga, gb;
|
||||
int hit_a = store_get_node(r, ID_A, &ga); if (hit_a==1) store_node_free(&ga);
|
||||
int hit_b = store_get_node(r, ID_B, &gb); if (hit_b==1) store_node_free(&gb);
|
||||
ok("both colliding nodes are durably present (store_get_node)", hit_a==1 && hit_b==1);
|
||||
|
||||
/* THE REGRESSION: the boot-load scan must emit BOTH, not silently drop one. */
|
||||
Collect c = {{ID_A, ID_B}, {0,0}, 2, 0};
|
||||
store_scan_nodes(r, node_cb, &c);
|
||||
printf(" scan emitted A=%d B=%d (total=%d)\n", c.seen[0], c.seen[1], c.total);
|
||||
ok("boot-load scan emits node A (would be resident)", c.seen[0]==1);
|
||||
ok("boot-load scan emits node B (the dropped/unretrievable one)", c.seen[1]==1);
|
||||
engram_close(r);
|
||||
}
|
||||
|
||||
/* ---- Bug: two id-hash-colliding EDGES, both must load. ---- */
|
||||
{
|
||||
char dir3[600]; snprintf(dir3, sizeof dir3, "%s/edges", g_dir); mkdir(dir3, 0700);
|
||||
EngramPagedStore* s = engram_open(dir3);
|
||||
StoreNode na, nb; mk_node(&na, "src", "s"); mk_node(&nb, "dst", "d");
|
||||
store_put_node(s, &na); store_put_node(s, &nb);
|
||||
StoreEdge ea, eb;
|
||||
mk_edge(&ea, ID_A, "src", "dst");
|
||||
mk_edge(&eb, ID_B, "src", "dst");
|
||||
store_put_edge(s, &ea);
|
||||
store_put_edge(s, &eb);
|
||||
engram_close(s);
|
||||
store_node_free(&na); store_node_free(&nb);
|
||||
store_edge_free(&ea); store_edge_free(&eb);
|
||||
|
||||
EngramPagedStore* r = engram_open(dir3);
|
||||
Collect c = {{ID_A, ID_B}, {0,0}, 2, 0};
|
||||
store_scan_edges(r, edge_cb, &c);
|
||||
printf(" scan emitted edgeA=%d edgeB=%d\n", c.seen[0], c.seen[1]);
|
||||
ok("boot-load scan emits edge A", c.seen[0]==1);
|
||||
ok("boot-load scan emits edge B (the dropped one)", c.seen[1]==1);
|
||||
engram_close(r);
|
||||
}
|
||||
|
||||
printf("\n %d passed, %d failed\n", g_pass, g_fail);
|
||||
/* cleanup */
|
||||
char cmd[600]; snprintf(cmd, sizeof cmd, "rm -rf %s", g_dir); if (system(cmd)){}
|
||||
return g_fail ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,439 @@
|
||||
/* test_store.c — M1 gate for the engram paged store (engram_store.{c,h}).
|
||||
*
|
||||
* Pure C. Build: gcc -O2 test_store.c ../../lang/runtime/engram_store.c -o test_store
|
||||
* Writes ONLY under a throwaway /tmp dir. Never touches ~/.neuron or live ports.
|
||||
*
|
||||
* Covers §7 M1 gates: round-trip (5k nodes / 20k edges, all fields, emb bit-exact,
|
||||
* hebb, >page content), TLV forward-compat, overflow chains, B+-tree indexes
|
||||
* across splits, free-list reuse, and corruption/superblock recovery.
|
||||
*/
|
||||
#include "../../lang/runtime/engram_store.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
static int g_pass = 0, g_fail = 0;
|
||||
static void ok(const char* name, int cond){
|
||||
printf(" [%s] %s\n", cond ? "PASS" : "FAIL", name);
|
||||
if (cond) g_pass++; else g_fail++;
|
||||
}
|
||||
|
||||
static char g_dir[512];
|
||||
static void mk_dir(void){
|
||||
snprintf(g_dir, sizeof g_dir, "/tmp/engram-store-test-%d", (int)getpid());
|
||||
mkdir(g_dir, 0700);
|
||||
}
|
||||
static void path_in(char* out, size_t cap, const char* name){
|
||||
snprintf(out, cap, "%s/%s", g_dir, name);
|
||||
}
|
||||
static long file_size(const char* p){ struct stat st; return stat(p,&st)==0 ? (long)st.st_size : -1; }
|
||||
|
||||
/* ── deterministic RNG so oracle nodes/edges regenerate bit-exact ─────────── */
|
||||
static uint64_t xs(uint64_t* s){ uint64_t x=*s; x^=x<<13; x^=x>>7; x^=x<<17; *s=x; return x; }
|
||||
static uint64_t node_seed(int i){ return 0x9E3779B97F4A7C15ULL ^ ((uint64_t)(i+1)*0xD1B54A32D192ED03ULL); }
|
||||
static uint64_t edge_seed(int i){ return 0xC2B2AE3D27D4EB4FULL ^ ((uint64_t)(i+1)*0x165667B19E3779F9ULL); }
|
||||
|
||||
static char* rnd_str(uint64_t* st, size_t len){
|
||||
char* s = (char*)malloc(len + 1);
|
||||
for (size_t i=0;i<len;i++) s[i] = (char)(33 + (xs(st) % 94)); /* printable, no NUL */
|
||||
s[len] = 0; return s;
|
||||
}
|
||||
|
||||
/* NODE_COUNT nodes; a slice have >page content to force overflow chains. */
|
||||
#define NODE_COUNT 5000
|
||||
#define EDGE_COUNT 20000
|
||||
#define EMB_DIM 768
|
||||
|
||||
static void gen_node(int i, StoreNode* n){
|
||||
memset(n, 0, sizeof *n);
|
||||
uint64_t st = node_seed(i);
|
||||
char id[32]; snprintf(id, sizeof id, "node-%d", i);
|
||||
n->id = strdup(id);
|
||||
size_t clen = (i % 500 == 0) ? (size_t)(17000 + (xs(&st) % 6000)) : (size_t)(xs(&st) % 300);
|
||||
n->content = rnd_str(&st, clen);
|
||||
n->node_type = rnd_str(&st, 4 + (xs(&st) % 8));
|
||||
n->label = (i % 2) ? rnd_str(&st, 3 + (xs(&st) % 10)) : NULL;
|
||||
n->tier = rnd_str(&st, 4 + (xs(&st) % 6));
|
||||
n->tags = rnd_str(&st, xs(&st) % 40);
|
||||
n->metadata = (i % 3) ? rnd_str(&st, xs(&st) % 60) : NULL;
|
||||
n->salience = (double)(xs(&st) % 1000000) / 997.0;
|
||||
n->importance = (double)(xs(&st) % 1000000) / 131.0;
|
||||
n->confidence = (double)(xs(&st) % 1000000) / 733.0;
|
||||
n->temporal_decay_rate = (double)(xs(&st) % 1000000) / 101.0;
|
||||
n->activation_count = (int64_t)(xs(&st) % 100000);
|
||||
n->last_activated = (int64_t)xs(&st);
|
||||
n->created_at = (int64_t)(1600000000000LL + i);
|
||||
n->updated_at = (int64_t)xs(&st);
|
||||
n->background_activation = (double)(xs(&st) % 1000000) / 17.0;
|
||||
n->working_memory_weight = (double)(xs(&st) % 1000000) / 29.0;
|
||||
n->suppression_count = (int32_t)(xs(&st) % 50);
|
||||
n->layer_id = (uint32_t)(xs(&st) % 5);
|
||||
for (int k=0;k<STORE_BLL_K;k++) n->access_ts[k] = (int64_t)xs(&st);
|
||||
n->access_head = (int32_t)(xs(&st) % STORE_BLL_K);
|
||||
n->access_filled = (int32_t)(xs(&st) % (STORE_BLL_K + 1));
|
||||
n->wm_anchor = (double)(xs(&st) % 1000000) / 3.0;
|
||||
n->emb = (float*)malloc(EMB_DIM * sizeof(float));
|
||||
for (int k=0;k<EMB_DIM;k++){ uint32_t u=(uint32_t)xs(&st); memcpy(&n->emb[k], &u, 4); }
|
||||
n->emb_dim = EMB_DIM;
|
||||
}
|
||||
|
||||
static void gen_edge(int i, StoreEdge* e){
|
||||
memset(e, 0, sizeof *e);
|
||||
uint64_t st = edge_seed(i);
|
||||
char id[32], from[32], to[32];
|
||||
snprintf(id, sizeof id, "edge-%d", i);
|
||||
snprintf(from, sizeof from, "node-%d", (int)(xs(&st) % NODE_COUNT));
|
||||
snprintf(to, sizeof to, "node-%d", (int)(xs(&st) % NODE_COUNT));
|
||||
e->id = strdup(id); e->from_id = strdup(from); e->to_id = strdup(to);
|
||||
e->relation = rnd_str(&st, 3 + (xs(&st) % 12));
|
||||
e->metadata = (i % 4) ? rnd_str(&st, xs(&st) % 40) : NULL;
|
||||
e->weight = (double)(xs(&st) % 1000000) / 111.0;
|
||||
e->hebb = (double)(xs(&st) % 1000000) / 1000000.0; /* the learned field */
|
||||
e->confidence = (double)(xs(&st) % 1000000) / 777.0;
|
||||
e->created_at = (int64_t)(1600000000000LL + i);
|
||||
e->updated_at = (int64_t)xs(&st);
|
||||
e->last_fired = (int64_t)xs(&st);
|
||||
e->inhibitory = (int32_t)(xs(&st) % 2);
|
||||
e->layer_id = (uint32_t)(xs(&st) % 5);
|
||||
}
|
||||
|
||||
static int streq(const char* a, const char* b){
|
||||
if (!a && !b) return 1;
|
||||
if (!a || !b) return 0;
|
||||
return strcmp(a,b)==0;
|
||||
}
|
||||
static int cmp_node(const StoreNode* a, const StoreNode* b){
|
||||
if (!streq(a->id,b->id) || !streq(a->content,b->content) ||
|
||||
!streq(a->node_type,b->node_type) || !streq(a->label,b->label) ||
|
||||
!streq(a->tier,b->tier) || !streq(a->tags,b->tags) ||
|
||||
!streq(a->metadata,b->metadata)) return 0;
|
||||
if (a->salience!=b->salience || a->importance!=b->importance ||
|
||||
a->confidence!=b->confidence || a->temporal_decay_rate!=b->temporal_decay_rate ||
|
||||
a->activation_count!=b->activation_count || a->last_activated!=b->last_activated ||
|
||||
a->created_at!=b->created_at || a->updated_at!=b->updated_at ||
|
||||
a->background_activation!=b->background_activation ||
|
||||
a->working_memory_weight!=b->working_memory_weight ||
|
||||
a->suppression_count!=b->suppression_count || a->layer_id!=b->layer_id ||
|
||||
a->access_head!=b->access_head || a->access_filled!=b->access_filled ||
|
||||
a->wm_anchor!=b->wm_anchor || a->emb_dim!=b->emb_dim) return 0;
|
||||
for (int k=0;k<STORE_BLL_K;k++) if (a->access_ts[k]!=b->access_ts[k]) return 0;
|
||||
if ((a->emb==NULL) != (b->emb==NULL)) return 0;
|
||||
if (a->emb && memcmp(a->emb, b->emb, (size_t)a->emb_dim*4)!=0) return 0;
|
||||
return 1;
|
||||
}
|
||||
static int cmp_edge(const StoreEdge* a, const StoreEdge* b){
|
||||
if (!streq(a->id,b->id) || !streq(a->from_id,b->from_id) || !streq(a->to_id,b->to_id) ||
|
||||
!streq(a->relation,b->relation) || !streq(a->metadata,b->metadata)) return 0;
|
||||
if (a->weight!=b->weight || a->hebb!=b->hebb || a->confidence!=b->confidence ||
|
||||
a->created_at!=b->created_at || a->updated_at!=b->updated_at ||
|
||||
a->last_fired!=b->last_fired || a->inhibitory!=b->inhibitory ||
|
||||
a->layer_id!=b->layer_id) return 0;
|
||||
return 1;
|
||||
}
|
||||
static void free_node_fields(StoreNode* n){
|
||||
free(n->id); free(n->content); free(n->node_type); free(n->label);
|
||||
free(n->tier); free(n->tags); free(n->metadata); free(n->emb); free(n->unknown);
|
||||
}
|
||||
static void free_edge_fields(StoreEdge* e){
|
||||
free(e->id); free(e->from_id); free(e->to_id); free(e->relation); free(e->metadata); free(e->unknown);
|
||||
}
|
||||
|
||||
/* Flip one byte in the store file at (page*PAGE_SIZE + off). */
|
||||
static void flip_byte(const char* path, uint64_t page, size_t off){
|
||||
int fd = open(path, O_RDWR);
|
||||
uint8_t b; off_t at = (off_t)page*STORE_PAGE_SIZE + off;
|
||||
pread(fd, &b, 1, at); b ^= 0xFF; pwrite(fd, &b, 1, at); close(fd);
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
static void test_roundtrip(void){
|
||||
printf("\n== round-trip: %d nodes + %d edges, all fields, emb bit-exact ==\n", NODE_COUNT, EDGE_COUNT);
|
||||
char path[600]; path_in(path, sizeof path, "roundtrip.store");
|
||||
unlink(path);
|
||||
EngramPagedStore* s = store_create(path);
|
||||
ok("store_create", s != NULL);
|
||||
if (!s) return;
|
||||
|
||||
for (int i=0;i<NODE_COUNT;i++){ StoreNode n; gen_node(i,&n);
|
||||
if (store_put_node(s,&n)!=0){ ok("put_node", 0); free_node_fields(&n); store_close(s); return; }
|
||||
free_node_fields(&n); }
|
||||
for (int i=0;i<EDGE_COUNT;i++){ StoreEdge e; gen_edge(i,&e);
|
||||
if (store_put_edge(s,&e)!=0){ ok("put_edge", 0); free_edge_fields(&e); store_close(s); return; }
|
||||
free_edge_fields(&e); }
|
||||
ok("wrote all nodes+edges", 1);
|
||||
store_close(s);
|
||||
|
||||
long sz = file_size(path);
|
||||
printf(" store file size: %ld bytes (%.2f MB) for %d nodes / %d edges\n",
|
||||
sz, sz/1048576.0, NODE_COUNT, EDGE_COUNT);
|
||||
|
||||
s = store_open(path);
|
||||
ok("store_open (reopen)", s != NULL);
|
||||
if (!s) return;
|
||||
|
||||
int nbad = 0;
|
||||
for (int i=0;i<NODE_COUNT;i++){
|
||||
StoreNode want; gen_node(i,&want);
|
||||
StoreNode got; int r = store_get_node(s, want.id, &got);
|
||||
if (r!=1 || !cmp_node(&want,&got) || got.unknown_len!=0) nbad++;
|
||||
if (r==1) store_node_free(&got);
|
||||
free_node_fields(&want);
|
||||
}
|
||||
ok("all 5000 nodes read back bit-exact (incl emb, all fields)", nbad==0);
|
||||
if (nbad) printf(" %d node mismatches\n", nbad);
|
||||
|
||||
int ebad = 0;
|
||||
for (int i=0;i<EDGE_COUNT;i++){
|
||||
StoreEdge want; gen_edge(i,&want);
|
||||
StoreEdge* got; size_t gn;
|
||||
int found = 0;
|
||||
if (store_get_edges_from(s, want.from_id, &got, &gn)==0){
|
||||
for (size_t j=0;j<gn;j++) if (streq(got[j].id, want.id)){ if (cmp_edge(&want,&got[j])) found=1; break; }
|
||||
store_edges_free(got, gn);
|
||||
}
|
||||
if (!found) ebad++;
|
||||
free_edge_fields(&want);
|
||||
}
|
||||
ok("all 20000 edges read back via adjacency, all fields incl hebb", ebad==0);
|
||||
if (ebad) printf(" %d edge mismatches\n", ebad);
|
||||
|
||||
ok("store_check crc clean after round-trip", store_check(s, STORE_CHECK_CRC)==0);
|
||||
store_close(s);
|
||||
}
|
||||
|
||||
static void test_forward_compat(void){
|
||||
printf("\n== TLV forward-compat: omit field defaults; unknown tag preserved ==\n");
|
||||
char path[600]; path_in(path, sizeof path, "fwd.store");
|
||||
unlink(path);
|
||||
EngramPagedStore* s = store_create(path);
|
||||
|
||||
/* Writer OMITS several fields (metadata, label, emb) → reader must default. */
|
||||
StoreNode a; memset(&a,0,sizeof a);
|
||||
a.id = strdup("omit-1"); a.content = strdup("has content"); a.tier = strdup("core");
|
||||
a.salience = 0.5; /* metadata/label NULL, emb NULL */
|
||||
store_put_node(s, &a); free(a.id); free(a.content); free(a.tier);
|
||||
|
||||
StoreNode g; int r = store_get_node(s, "omit-1", &g);
|
||||
ok("omitted string fields default to NULL", r==1 && g.metadata==NULL && g.label==NULL);
|
||||
ok("omitted emb defaults to NULL / emb_dim 0", r==1 && g.emb==NULL && g.emb_dim==0);
|
||||
ok("present fields intact", r==1 && streq(g.content,"has content") && g.salience==0.5);
|
||||
if (r==1) store_node_free(&g);
|
||||
|
||||
/* Writer includes an UNKNOWN tag (simulating a newer writer / field the
|
||||
* reader does not model) via the `unknown` passthrough. Reader (which also
|
||||
* models known fields A,B,C) must preserve it verbatim. */
|
||||
uint8_t unk[64];
|
||||
unk[0] = 200; /* a tag this build has no case for */
|
||||
/* [u8 tag][u32 len][bytes] */
|
||||
unk[1]=8; unk[2]=0; unk[3]=0; unk[4]=0;
|
||||
for (int i=0;i<8;i++) unk[5+i] = (uint8_t)(0xA0 + i);
|
||||
StoreNode b; memset(&b,0,sizeof b);
|
||||
b.id = strdup("unk-1"); b.content = strdup("known field B"); b.confidence = 0.9; /* known field C-ish */
|
||||
b.unknown = unk; b.unknown_len = 5 + 8;
|
||||
store_put_node(s, &b); free(b.id); free(b.content);
|
||||
|
||||
StoreNode g2; int r2 = store_get_node(s, "unk-1", &g2);
|
||||
int unk_ok = r2==1 && g2.unknown_len==(5+8) && memcmp(g2.unknown, unk, 5+8)==0;
|
||||
ok("unknown tag preserved verbatim on read", unk_ok);
|
||||
ok("known fields still read while unknown preserved", r2==1 && streq(g2.content,"known field B") && g2.confidence==0.9);
|
||||
if (r2==1) store_node_free(&g2);
|
||||
|
||||
store_close(s);
|
||||
}
|
||||
|
||||
static void test_overflow(void){
|
||||
printf("\n== overflow: 100KB content node + emb via overflow chain ==\n");
|
||||
char path[600]; path_in(path, sizeof path, "ovf.store");
|
||||
unlink(path);
|
||||
EngramPagedStore* s = store_create(path);
|
||||
|
||||
size_t big = 100*1024;
|
||||
StoreNode n; memset(&n,0,sizeof n);
|
||||
n.id = strdup("big-1");
|
||||
n.content = (char*)malloc(big+1);
|
||||
for (size_t i=0;i<big;i++) n.content[i] = (char)(33 + (i % 94));
|
||||
n.content[big] = 0;
|
||||
n.tier = strdup("episodic");
|
||||
n.emb = (float*)malloc(EMB_DIM*sizeof(float));
|
||||
for (int k=0;k<EMB_DIM;k++){ float f = (float)(k*0.5 - 100.0); n.emb[k]=f; }
|
||||
n.emb_dim = EMB_DIM;
|
||||
ok("put 100KB+emb node", store_put_node(s,&n)==0);
|
||||
store_close(s);
|
||||
|
||||
s = store_open(path);
|
||||
StoreNode g; int r = store_get_node(s, "big-1", &g);
|
||||
ok("reopen + read big node", r==1);
|
||||
ok("100KB content byte-exact via overflow", r==1 && strlen(g.content)==big && memcmp(g.content,n.content,big)==0);
|
||||
ok("emb bit-exact via overflow record", r==1 && g.emb_dim==EMB_DIM && memcmp(g.emb,n.emb,EMB_DIM*4)==0);
|
||||
if (r==1) store_node_free(&g);
|
||||
ok("store_check clean (overflow pages crc'd)", store_check(s, STORE_CHECK_CRC)==0);
|
||||
store_close(s);
|
||||
free_node_fields(&n);
|
||||
}
|
||||
|
||||
static void test_index_splits(void){
|
||||
printf("\n== B+-tree index correctness across many splits ==\n");
|
||||
char path[600]; path_in(path, sizeof path, "idx.store");
|
||||
unlink(path);
|
||||
EngramPagedStore* s = store_create(path);
|
||||
/* Tiny order forces deep leaf + internal splits with only a few hundred keys. */
|
||||
store__set_btree_order(s, 4, 4);
|
||||
|
||||
const int N = 600;
|
||||
for (int i=0;i<N;i++){
|
||||
StoreNode n; memset(&n,0,sizeof n);
|
||||
char id[32]; snprintf(id,sizeof id,"k-%05d", (i*37+11)%100000); /* scattered keys */
|
||||
n.id = strdup(id); n.content = strdup("x"); n.tier=strdup("t"); n.salience=i;
|
||||
if (store_put_node(s,&n)!=0){ ok("put",0); }
|
||||
free(n.id); free(n.content); free(n.tier);
|
||||
}
|
||||
int miss=0;
|
||||
for (int i=0;i<N;i++){
|
||||
char id[32]; snprintf(id,sizeof id,"k-%05d",(i*37+11)%100000);
|
||||
StoreNode g; int r = store_get_node(s, id, &g);
|
||||
if (r!=1 || (int)g.salience != i) miss++;
|
||||
if (r==1) store_node_free(&g);
|
||||
}
|
||||
ok("all keys retrievable after leaf+internal splits", miss==0);
|
||||
if (miss) printf(" %d misses\n", miss);
|
||||
StoreNode g; ok("absent key returns 0", store_get_node(s,"k-NOPE",&g)==0);
|
||||
|
||||
/* Adjacency: controlled star + chain, exact edge sets. */
|
||||
for (int i=0;i<50;i++){
|
||||
StoreEdge e; memset(&e,0,sizeof e);
|
||||
char id[32]; snprintf(id,sizeof id,"e-%d",i);
|
||||
e.id=strdup(id); e.from_id=strdup("HUB"); char tt[16]; snprintf(tt,sizeof tt,"T-%d",i); e.to_id=strdup(tt);
|
||||
e.relation=strdup("r"); e.weight=1.0; e.hebb=0.1*i;
|
||||
store_put_edge(s,&e); free_edge_fields(&e);
|
||||
}
|
||||
for (int i=0;i<7;i++){
|
||||
StoreEdge e; memset(&e,0,sizeof e);
|
||||
char id[32]; snprintf(id,sizeof id,"in-%d",i);
|
||||
char ff[16]; snprintf(ff,sizeof ff,"S-%d",i);
|
||||
e.id=strdup(id); e.from_id=strdup(ff); e.to_id=strdup("SINK");
|
||||
e.relation=strdup("r"); e.weight=1.0;
|
||||
store_put_edge(s,&e); free_edge_fields(&e);
|
||||
}
|
||||
StoreEdge* out; size_t on;
|
||||
store_get_edges_from(s,"HUB",&out,&on);
|
||||
ok("get_edges_from(HUB) == 50", on==50);
|
||||
store_edges_free(out,on);
|
||||
store_get_edges_to(s,"SINK",&out,&on);
|
||||
ok("get_edges_to(SINK) == 7", on==7);
|
||||
store_edges_free(out,on);
|
||||
store_get_edges_to(s,"HUB",&out,&on);
|
||||
ok("get_edges_to(HUB) == 0 (direction separation)", on==0);
|
||||
store_edges_free(out,on);
|
||||
|
||||
ok("store_check clean", store_check(s, STORE_CHECK_CRC)==0);
|
||||
store_close(s);
|
||||
}
|
||||
|
||||
static void test_freelist(void){
|
||||
printf("\n== free-list: tombstone reclaims pages, graph stays consistent ==\n");
|
||||
char path[600]; path_in(path, sizeof path, "free.store");
|
||||
unlink(path);
|
||||
EngramPagedStore* s = store_create(path);
|
||||
|
||||
uint64_t pc0 = store_page_count(s);
|
||||
const int N = 300;
|
||||
for (int i=0;i<N;i++){
|
||||
StoreNode n; memset(&n,0,sizeof n);
|
||||
char id[32]; snprintf(id,sizeof id,"a-%d",i);
|
||||
n.id=strdup(id); n.content=rnd_str(&(uint64_t){node_seed(i)}, 200); n.tier=strdup("t");
|
||||
store_put_node(s,&n); free_node_fields(&n);
|
||||
}
|
||||
uint64_t pc1 = store_page_count(s);
|
||||
uint64_t node_pages = pc1 - pc0;
|
||||
ok("initial batch consumed pages", node_pages > 0);
|
||||
|
||||
for (int i=0;i<N;i++){ char id[32]; snprintf(id,sizeof id,"a-%d",i); store_tombstone(s,id); }
|
||||
/* all old nodes gone */
|
||||
int gone=1; for (int i=0;i<N;i++){ char id[32]; snprintf(id,sizeof id,"a-%d",i);
|
||||
StoreNode g; if (store_get_node(s,id,&g)==1){ gone=0; store_node_free(&g); } }
|
||||
ok("tombstoned nodes now absent", gone);
|
||||
|
||||
for (int i=0;i<N;i++){
|
||||
StoreNode n; memset(&n,0,sizeof n);
|
||||
char id[32]; snprintf(id,sizeof id,"b-%d",i);
|
||||
n.id=strdup(id); n.content=strdup("reused"); n.tier=strdup("t"); n.salience=i;
|
||||
store_put_node(s,&n); free_node_fields(&n);
|
||||
}
|
||||
uint64_t pc2 = store_page_count(s);
|
||||
/* Reuse proven: growth for the 2nd batch is far less than a fresh alloc. */
|
||||
ok("freed pages reused (no full re-growth)", pc2 < pc1 + node_pages);
|
||||
printf(" pages: base=%llu after1=%llu after2=%llu (node_pages=%llu)\n",
|
||||
(unsigned long long)pc0,(unsigned long long)pc1,(unsigned long long)pc2,(unsigned long long)node_pages);
|
||||
|
||||
int newbad=0; for (int i=0;i<N;i++){ char id[32]; snprintf(id,sizeof id,"b-%d",i);
|
||||
StoreNode g; if (store_get_node(s,id,&g)!=1 || (int)g.salience!=i) newbad++; else store_node_free(&g); }
|
||||
ok("new batch fully readable after reuse", newbad==0);
|
||||
ok("store_check clean after reuse", store_check(s, STORE_CHECK_CRC)==0);
|
||||
|
||||
store_close(s);
|
||||
/* survives reopen */
|
||||
s = store_open(path);
|
||||
int rb=0; for (int i=0;i<N;i++){ char id[32]; snprintf(id,sizeof id,"b-%d",i);
|
||||
StoreNode g; if (store_get_node(s,id,&g)!=1) rb++; else store_node_free(&g); }
|
||||
ok("graph consistent across reopen after reuse", rb==0);
|
||||
store_close(s);
|
||||
}
|
||||
|
||||
static void test_corruption(void){
|
||||
printf("\n== corruption: crc detection + superblock mirror recovery ==\n");
|
||||
char path[600]; path_in(path, sizeof path, "corrupt.store");
|
||||
unlink(path);
|
||||
EngramPagedStore* s = store_create(path);
|
||||
for (int i=0;i<50;i++){ StoreNode n; gen_node(i,&n); store_put_node(s,&n); free_node_fields(&n); }
|
||||
store_close(s);
|
||||
|
||||
s = store_open(path);
|
||||
ok("clean store: store_check == 0", store_check(s, STORE_CHECK_CRC)==0);
|
||||
store_close(s);
|
||||
|
||||
/* flip a byte inside a data page (page 5 is node/index data, never a SB) */
|
||||
flip_byte(path, 5, 137);
|
||||
s = store_open(path);
|
||||
ok("store_open still succeeds (data-page corruption)", s != NULL);
|
||||
int bad = store_check(s, STORE_CHECK_CRC);
|
||||
ok("store_check detects corrupted page via crc", bad >= 1);
|
||||
printf(" store_check reported %d corrupt page(s)\n", bad);
|
||||
store_close(s);
|
||||
|
||||
/* fresh store, corrupt superblock 0, must recover via mirror superblock 1 */
|
||||
char p2[600]; path_in(p2, sizeof p2, "sbrec.store");
|
||||
unlink(p2);
|
||||
s = store_create(p2);
|
||||
StoreNode n; gen_node(42,&n); store_put_node(s,&n);
|
||||
store_close(s);
|
||||
/* trash magic + crc region of page 0 */
|
||||
flip_byte(p2, 0, 0); flip_byte(p2, 0, 1); flip_byte(p2, 0, 90);
|
||||
s = store_open(p2);
|
||||
ok("open recovers via mirror superblock (page 1)", s != NULL);
|
||||
if (s){
|
||||
StoreNode g; int r = store_get_node(s, "node-42", &g);
|
||||
ok("data intact after superblock recovery", r==1 && cmp_node(&n,&g));
|
||||
if (r==1) store_node_free(&g);
|
||||
store_close(s);
|
||||
}
|
||||
free_node_fields(&n);
|
||||
}
|
||||
|
||||
int main(void){
|
||||
mk_dir();
|
||||
printf("engram_store M1 test harness — dir=%s\n", g_dir);
|
||||
test_roundtrip();
|
||||
test_forward_compat();
|
||||
test_overflow();
|
||||
test_index_splits();
|
||||
test_freelist();
|
||||
test_corruption();
|
||||
printf("\n================ %d passed, %d failed ================\n", g_pass, g_fail);
|
||||
return g_fail ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
/* Closed-form unit tests for the VERIFIER layer (engram_verify.c). Every case is a
|
||||
* hand-built synthetic descriptor / claim point whose verdict is known in closed
|
||||
* form — the checks are PROVEN, not declared. ASan/UBSan target.
|
||||
*
|
||||
* The headline case is CONSISTENCY's polarity check: the reassurance→accusation
|
||||
* inversion ("you never fought" → "you argued") that no grammar check catches. */
|
||||
#include "engram_verify.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
|
||||
static int failures = 0, checks = 0;
|
||||
static void ok(const char* what, int cond) {
|
||||
checks++;
|
||||
if (!cond) { failures++; printf(" FAIL: %s\n", what); }
|
||||
else printf(" ok: %s\n", what);
|
||||
}
|
||||
static void approx(const char* what, double got, double exp, double tol) {
|
||||
ok(what, fabs(got - exp) <= tol);
|
||||
if (fabs(got - exp) > tol) printf(" got=%.9g exp=%.9g\n", got, exp);
|
||||
}
|
||||
|
||||
/* ── descriptor builder (mirrors test_reason.c) ─────────────────────────────── */
|
||||
static float* vec(const double* v, int dim) {
|
||||
float* f = malloc((size_t)dim * sizeof(float));
|
||||
for (int i = 0; i < dim; i++) f[i] = (float)v[i];
|
||||
return f;
|
||||
}
|
||||
static GeoDescriptor* mk(int dim, const double* centroid,
|
||||
int n_axes, const double* axis_flat, const double* extents,
|
||||
int n_members, const char** ids, double total_var) {
|
||||
GeoDescriptor* g = calloc(1, sizeof(GeoDescriptor));
|
||||
g->dim = dim;
|
||||
g->centroid = centroid ? vec(centroid, dim) : NULL;
|
||||
g->global_mean = NULL;
|
||||
g->n_axes = n_axes;
|
||||
g->axes = n_axes ? calloc((size_t)n_axes, sizeof(GeoAxis)) : NULL;
|
||||
double tr = 0;
|
||||
for (int k = 0; k < n_axes; k++) {
|
||||
g->axes[k].axis = vec(&axis_flat[(size_t)k * dim], dim);
|
||||
g->axes[k].extent = extents[k];
|
||||
tr += extents[k] * extents[k];
|
||||
}
|
||||
g->total_variance = (total_var >= 0) ? total_var : tr;
|
||||
g->radius = sqrt(g->total_variance > 0 ? g->total_variance : 0);
|
||||
g->n_members = n_members; g->n_embedded = n_members;
|
||||
g->members = n_members ? calloc((size_t)n_members, sizeof(GeoMember)) : NULL;
|
||||
for (int i = 0; i < n_members; i++) {
|
||||
g->members[i].id = strdup(ids[i]);
|
||||
g->members[i].membership = 1.0;
|
||||
g->members[i].centrality = (double)(n_members - i);
|
||||
g->members[i].embedded = 1;
|
||||
}
|
||||
g->hub_id = n_members ? strdup(ids[0]) : strdup("");
|
||||
g->k_core = 1; g->co_registration = 0.0; g->n_edges = 0; g->edges = NULL;
|
||||
return g;
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
printf("== VERIFIER layer unit tests ==\n");
|
||||
|
||||
/* ══════════════════ GROUNDING — supported vs floating (hallucination) ════ */
|
||||
/* Two real neighborhoods: E0 at origin, E1 far along e0. A claim planted inside
|
||||
* E0 is grounded; a claim floating far off-manifold (along an unmodeled axis) is
|
||||
* flagged UNGROUNDED; a claim near E1 grounds to E1, not E0. */
|
||||
{
|
||||
int dim = 4;
|
||||
double c0[4] = {0,0,0,0}, c1[4] = {10,0,0,0};
|
||||
double ax[8] = {1,0,0,0, 0,1,0,0}; double ex[2] = {1,1};
|
||||
const char* i0[1] = {"E0"}, *i1[1] = {"E1"};
|
||||
GeoDescriptor* E0 = mk(dim, c0, 2, ax, ex, 1, i0, -1);
|
||||
GeoDescriptor* E1 = mk(dim, c1, 2, ax, ex, 1, i1, -1);
|
||||
const GeoDescriptor* ev[2] = {E0, E1};
|
||||
|
||||
/* (1) grounded claim — sits inside E0. */
|
||||
float in[4] = {0.3f, -0.2f, 0, 0};
|
||||
GeoGrounding g1;
|
||||
int rc = engram_verify_grounding(in, dim, ev, 2, 1.0, 0.5, &g1);
|
||||
ok("grounding returns 0", rc == 0);
|
||||
printf("[grounding] IN score=%.4f grounded=%d best=%d dist=%.3f ortho=%.3f nearL2=%.3f\n",
|
||||
g1.grounding, g1.grounded, g1.best, g1.best_distance, g1.best_ortho, g1.nearest_centroid_l2);
|
||||
ok("planted-inside claim is GROUNDED", g1.grounded == 1);
|
||||
ok("grounds to the nearest structure E0", g1.best == 0);
|
||||
ok("grounded score high (>0.7)", g1.grounding > 0.7);
|
||||
approx("off-model residual ~0 for in-distribution claim", g1.best_ortho, 0.0, 1e-4);
|
||||
engram_verify_grounding_free(&g1);
|
||||
|
||||
/* (2) hallucinated claim — floats far along the unmodeled e2 axis. */
|
||||
float out[4] = {0, 0, 50.0f, 0};
|
||||
GeoGrounding g2;
|
||||
engram_verify_grounding(out, dim, ev, 2, 1.0, 0.5, &g2);
|
||||
printf("[grounding] OUT score=%.6f grounded=%d best=%d dist=%.3f ortho=%.3f nearL2=%.3f\n",
|
||||
g2.grounding, g2.grounded, g2.best, g2.best_distance, g2.best_ortho, g2.nearest_centroid_l2);
|
||||
ok("floating claim is FLAGGED (ungrounded)", g2.grounded == 0);
|
||||
ok("floating claim scores near zero (<0.01)", g2.grounding < 0.01);
|
||||
ok("off-model residual is large (the hallucination signal)", g2.best_ortho > 40.0);
|
||||
ok("nearest real structure is far (L2>40)", g2.nearest_centroid_l2 > 40.0);
|
||||
engram_verify_grounding_free(&g2);
|
||||
|
||||
/* (3) selection — a claim near E1 grounds to E1. */
|
||||
float nearE1[4] = {9.8f, 0.1f, 0, 0};
|
||||
GeoGrounding g3;
|
||||
engram_verify_grounding(nearE1, dim, ev, 2, 1.0, 0.5, &g3);
|
||||
printf("[grounding] E1 score=%.4f grounded=%d best=%d\n", g3.grounding, g3.grounded, g3.best);
|
||||
ok("claim near E1 grounds to E1 (best=1)", g3.best == 1 && g3.grounded == 1);
|
||||
engram_verify_grounding_free(&g3);
|
||||
|
||||
engram_geo_free(E0); engram_geo_free(E1);
|
||||
}
|
||||
|
||||
/* ══════════════════ CONSISTENCY (a) — THE NEGATION-INVERSION CATCH ═══════ */
|
||||
/* The motivating failure, geometrically. Polarity axis along e0:
|
||||
* pole_pos = the AFFIRM region ("argued / fought") centroid (+5, …)
|
||||
* pole_neg = the NEGATE region ("never fought / at peace") centroid (−5, …)
|
||||
* The grounded TRUTH (context) is the reassurance "you never fought" → sits on
|
||||
* the NEGATE side (−5). The bad translation CLAIM "you argued" lands on the
|
||||
* AFFIRM side (+4). Opposite sides of the negation axis ⇒ INVERSION flagged —
|
||||
* even though "you argued" is perfectly grammatical. This is the catch. */
|
||||
{
|
||||
int dim = 4;
|
||||
double c_pos[4] = { 5, 0, 0, 0}; /* "argued / fought" */
|
||||
double c_neg[4] = {-5, 0, 0, 0}; /* "never fought / at peace"*/
|
||||
double c_truth[4] = {-5, 0, 0, 0}; /* context: the reassurance */
|
||||
double ax[4] = {1,0,0,0}; double ex[1] = {1};
|
||||
const char* ip[1]={"pos"},*in[1]={"neg"},*it[1]={"truth"};
|
||||
GeoDescriptor* POS = mk(dim, c_pos, 1, ax, ex, 1, ip, -1);
|
||||
GeoDescriptor* NEG = mk(dim, c_neg, 1, ax, ex, 1, in, -1);
|
||||
GeoDescriptor* CTX = mk(dim, c_truth, 1, ax, ex, 1, it, -1);
|
||||
|
||||
/* the plausible LIE: "you argued" — grammatical, fluent, and INVERTED. */
|
||||
float lie[4] = { 4, 0, 0, 0};
|
||||
GeoConsistency cl;
|
||||
int rc = engram_verify_consistency(lie, dim, CTX, POS, NEG, NULL,
|
||||
1.0, 0.10, 0.5, 0.0, &cl);
|
||||
ok("consistency returns 0", rc == 0);
|
||||
printf("[consistency] LIE verdict=%d inverted=%d claim_side=%.3f ref_side=%.3f sep=%.3f consist=%.3f\n",
|
||||
cl.verdict, cl.inverted, cl.polarity_claim, cl.polarity_reference, cl.polarity_separation, cl.consistency);
|
||||
ok("NEGATION INVERSION caught (inverted=1)", cl.inverted == 1);
|
||||
ok("verdict = POLARITY", cl.verdict == GEO_CONSIST_POLARITY);
|
||||
ok("claim sits on the AFFIRM pole (+)", cl.polarity_claim > 0);
|
||||
ok("truth sits on the NEGATE pole (−)", cl.polarity_reference < 0);
|
||||
ok("consistency collapses to 0 on inversion", cl.consistency < 1e-9);
|
||||
|
||||
/* the FAITHFUL translation: "you were at peace" — same pole as the truth. */
|
||||
float ok_claim[4] = {-4, 0, 0, 0};
|
||||
GeoConsistency cok;
|
||||
engram_verify_consistency(ok_claim, dim, CTX, POS, NEG, NULL,
|
||||
1.0, 0.10, 0.5, 0.0, &cok);
|
||||
printf("[consistency] TRUE verdict=%d inverted=%d claim_side=%.3f consist=%.3f\n",
|
||||
cok.verdict, cok.inverted, cok.polarity_claim, cok.consistency);
|
||||
ok("faithful claim NOT flagged (inverted=0)", cok.inverted == 0);
|
||||
ok("faithful claim verdict OK", cok.verdict == GEO_CONSIST_OK);
|
||||
ok("faithful claim consistency = 1", cok.consistency > 0.999);
|
||||
|
||||
/* a NEUTRAL claim near the midpoint must NOT false-trigger. */
|
||||
float neutral[4] = {0.1f, 0, 0, 0}; /* |side|=0.1 < deadzone 0.5 */
|
||||
GeoConsistency cn;
|
||||
engram_verify_consistency(neutral, dim, CTX, POS, NEG, NULL,
|
||||
1.0, 0.10, 0.5, 0.0, &cn);
|
||||
printf("[consistency] NEUT verdict=%d inverted=%d claim_side=%.3f consist=%.3f\n",
|
||||
cn.verdict, cn.inverted, cn.polarity_claim, cn.consistency);
|
||||
ok("neutral claim inside deadzone does NOT trigger inversion", cn.inverted == 0);
|
||||
|
||||
engram_geo_free(POS); engram_geo_free(NEG); engram_geo_free(CTX);
|
||||
}
|
||||
|
||||
/* ══════════════════ CONSISTENCY (b) — GEOMETRIC contradiction ════════════ */
|
||||
/* A claim that sits INSIDE a forbidden region it should be far from, and a claim
|
||||
* that violates a max-distance constraint to its context, are both flagged. */
|
||||
{
|
||||
int dim = 4;
|
||||
double c_ctx[4] = {0,0,0,0};
|
||||
double c_forb[4] = {0,10,0,0}; /* forbidden region, offset along e1 */
|
||||
double ax[8] = {0,1,0,0, 1,0,0,0}; double ex[2] = {1,1};
|
||||
const char* ic[1]={"ctx"},*ifb[1]={"forb"};
|
||||
GeoDescriptor* CTX = mk(dim, c_ctx, 2, ax, ex, 1, ic, -1);
|
||||
GeoDescriptor* FORB = mk(dim, c_forb, 2, ax, ex, 1, ifb, -1);
|
||||
|
||||
/* claim sitting inside the forbidden region → geometric contradiction. */
|
||||
float inside[4] = {0, 10.1f, 0, 0};
|
||||
GeoConsistency cf;
|
||||
engram_verify_consistency(inside, dim, CTX, NULL, NULL, FORB,
|
||||
1.0, 0.10, 0.5, 0.0, &cf);
|
||||
printf("[consistency] FORB verdict=%d geo_viol=%d forb_fit=%.4f consist=%.3f\n",
|
||||
cf.verdict, cf.geo_violation, cf.forbidden_fit, cf.consistency);
|
||||
ok("claim inside forbidden region FLAGGED", cf.geo_violation == 1);
|
||||
ok("verdict = GEOMETRIC", cf.verdict == GEO_CONSIST_GEOMETRIC);
|
||||
ok("forbidden fit is high (claim really is inside)", cf.forbidden_fit > 0.5);
|
||||
|
||||
/* claim well clear of the forbidden region → not flagged. */
|
||||
float clear[4] = {0.2f, 0.1f, 0, 0};
|
||||
GeoConsistency cc;
|
||||
engram_verify_consistency(clear, dim, CTX, NULL, NULL, FORB,
|
||||
1.0, 0.10, 0.5, 0.0, &cc);
|
||||
printf("[consistency] CLR verdict=%d geo_viol=%d forb_fit=%.4f\n",
|
||||
cc.verdict, cc.geo_violation, cc.forbidden_fit);
|
||||
ok("claim clear of forbidden NOT flagged", cc.geo_violation == 0 && cc.verdict == GEO_CONSIST_OK);
|
||||
|
||||
/* max-distance constraint: claim too far from context (off-axis, no poles). */
|
||||
float far[4] = {0, 8.0f, 0, 0};
|
||||
GeoConsistency cd;
|
||||
engram_verify_consistency(far, dim, CTX, NULL, NULL, NULL,
|
||||
1.0, 0.10, 0.5, /*max_distance*/3.0, &cd);
|
||||
printf("[consistency] DIST verdict=%d geo_viol=%d ctx_dist=%.3f\n",
|
||||
cd.verdict, cd.geo_violation, cd.context_distance);
|
||||
ok("claim beyond max_distance FLAGGED", cd.geo_violation == 1 && cd.verdict == GEO_CONSIST_GEOMETRIC);
|
||||
approx("context distance measured correctly", cd.context_distance, 8.0, 1e-4);
|
||||
|
||||
engram_geo_free(CTX); engram_geo_free(FORB);
|
||||
}
|
||||
|
||||
/* ══════════════════ COMBINED — grounded but INVERTED (the full plausible lie) */
|
||||
/* The most dangerous output: fluent, GROUNDED in real vocabulary, yet polarity-
|
||||
* inverted. Grounding alone passes it; only consistency catches the lie. This is
|
||||
* exactly why the verifier needs BOTH checks. */
|
||||
{
|
||||
int dim = 4;
|
||||
double c_pos[4] = { 5, 0, 0, 0}, c_neg[4] = {-5, 0, 0, 0};
|
||||
double ax[4] = {1,0,0,0}; double ex[1] = {2};
|
||||
const char* ip[1]={"pos"},*in[1]={"neg"};
|
||||
GeoDescriptor* POS = mk(dim, c_pos, 1, ax, ex, 1, ip, -1);
|
||||
GeoDescriptor* NEG = mk(dim, c_neg, 1, ax, ex, 1, in, -1);
|
||||
const GeoDescriptor* ev[2] = {POS, NEG};
|
||||
|
||||
float lie[4] = {5, 0, 0, 0}; /* "argued" — sits dead-center in the affirm region */
|
||||
GeoGrounding g;
|
||||
engram_verify_grounding(lie, dim, ev, 2, 1.0, 0.5, &g);
|
||||
GeoConsistency c;
|
||||
engram_verify_consistency(lie, dim, NEG /*truth=never fought*/, POS, NEG, NULL,
|
||||
1.0, 0.10, 0.5, 0.0, &c);
|
||||
printf("[combined] grounded=%d (score=%.3f) inverted=%d verdict=%d\n",
|
||||
g.grounded, g.grounding, c.inverted, c.verdict);
|
||||
ok("plausible lie PASSES grounding (it is real vocabulary)", g.grounded == 1);
|
||||
ok("plausible lie is CAUGHT by consistency (inverted)", c.inverted == 1);
|
||||
ok("=> grounding alone is insufficient; consistency is the catch",
|
||||
g.grounded == 1 && c.verdict == GEO_CONSIST_POLARITY);
|
||||
engram_verify_grounding_free(&g);
|
||||
engram_geo_free(POS); engram_geo_free(NEG);
|
||||
}
|
||||
|
||||
printf("\n== %d checks, %d failures ==\n", checks, failures);
|
||||
return failures ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
/* test_vindex.c — build + RUN gate for the M8 HNSW vector index.
|
||||
*
|
||||
* Covers: recall@10 vs brute-force oracle, brute-force-vs-index speedup,
|
||||
* correctness edge cases (k>N, identical vectors, self-query, zero vector),
|
||||
* determinism (seeded PRNG → identical graphs), and vindex_build_from_store
|
||||
* over a real engram_store on-disk file.
|
||||
*
|
||||
* Pure C11; links engram_vindex.c + engram_store.c; -lm. ASan/UBSan clean.
|
||||
*/
|
||||
#include "engram_vindex.h"
|
||||
#include "engram_store.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
#include <stdint.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#define DIM 768
|
||||
|
||||
static int g_fail = 0;
|
||||
/* VINDEX_QUICK=1 shrinks the two large builds so the ASan/UBSan pass (which runs
|
||||
* ~5-10x slower) stays fast — memory-safety is size-independent. The perf numbers
|
||||
* (recall gate + speedup) come from the un-sanitized, full-size pass. */
|
||||
static int g_quick = 0;
|
||||
static int envint(const char* k, int dflt){ const char* s=getenv(k); return s?atoi(s):dflt; }
|
||||
#define CHECK(cond, msg) do{ if(!(cond)){ printf(" FAIL: %s\n", msg); g_fail=1; } else { printf(" ok: %s\n", msg); } }while(0)
|
||||
|
||||
/* deterministic test PRNG (splitmix64) */
|
||||
static uint64_t rng_state = 0xABCDEF0123456789ULL;
|
||||
static uint64_t xrng(uint64_t* s){
|
||||
uint64_t z=(*s+=0x9E3779B97F4A7C15ULL);
|
||||
z=(z^(z>>30))*0xBF58476D1CE4E5B9ULL; z=(z^(z>>27))*0x94D049BB133111EBULL;
|
||||
return z^(z>>31);
|
||||
}
|
||||
static float frand(uint64_t* s){ return (float)((xrng(s)>>11)*(1.0/9007199254740992.0)) - 0.5f; }
|
||||
|
||||
static double now_s(void){
|
||||
struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t);
|
||||
return t.tv_sec + t.tv_nsec*1e-9;
|
||||
}
|
||||
|
||||
/* fill vec[N*DIM]: mostly random, some clustered groups (center + small noise). */
|
||||
static void gen_vectors(float* v, int N, uint64_t seed){
|
||||
uint64_t s = seed;
|
||||
int clustered = N/5; /* last fifth is clustered */
|
||||
int ncenters = 20;
|
||||
float* centers = (float*)malloc((size_t)ncenters*DIM*sizeof(float));
|
||||
for (int c=0;c<ncenters;c++) for(int d=0;d<DIM;d++) centers[c*DIM+d]=frand(&s);
|
||||
for (int i=0;i<N;i++){
|
||||
if (i < N-clustered){
|
||||
for (int d=0;d<DIM;d++) v[i*DIM+d]=frand(&s);
|
||||
} else {
|
||||
int c = (int)(xrng(&s)%ncenters);
|
||||
for (int d=0;d<DIM;d++) v[i*DIM+d]=centers[c*DIM+d] + 0.05f*frand(&s);
|
||||
}
|
||||
}
|
||||
free(centers);
|
||||
}
|
||||
|
||||
static float cosdist(const float* a, const float* b){
|
||||
double da=0,db=0,dot=0;
|
||||
for(int i=0;i<DIM;i++){ da+=(double)a[i]*a[i]; db+=(double)b[i]*b[i]; dot+=(double)a[i]*b[i]; }
|
||||
if (da<=0||db<=0) return 1.0f;
|
||||
return (float)(1.0 - dot/(sqrt(da)*sqrt(db)));
|
||||
}
|
||||
|
||||
/* brute-force top-k node ids into ids[k] (ascending distance). */
|
||||
static void brute_topk(const float* v, int N, const float* q, int k, int* ids){
|
||||
float* bd = (float*)malloc((size_t)k*sizeof(float));
|
||||
for (int i=0;i<k;i++){ ids[i]=-1; bd[i]=1e30f; }
|
||||
for (int i=0;i<N;i++){
|
||||
float d = cosdist(q, v+(size_t)i*DIM);
|
||||
if (d < bd[k-1]){
|
||||
int p=k-1;
|
||||
while (p>0 && bd[p-1]>d){ bd[p]=bd[p-1]; ids[p]=ids[p-1]; p--; }
|
||||
bd[p]=d; ids[p]=i;
|
||||
}
|
||||
}
|
||||
free(bd);
|
||||
}
|
||||
|
||||
/* ── Test 1: recall@10 vs brute force + latency/recall tradeoff ────────────── */
|
||||
static void test_recall(void){
|
||||
int N=envint("VINDEX_N_RECALL", g_quick?1500:5000), Q=200, K=10;
|
||||
printf("\n== Test 1: recall@10 vs brute force (N=%d, DIM=768) ==\n", N);
|
||||
float* v = (float*)malloc((size_t)N*DIM*sizeof(float));
|
||||
gen_vectors(v, N, 111);
|
||||
|
||||
double t0=now_s();
|
||||
VIndex* ix = vindex_create(DIM, VINDEX_DEFAULT_M, VINDEX_DEFAULT_EF_CONSTRUCTION);
|
||||
for (int i=0;i<N;i++) vindex_insert(ix, (uint64_t)i, v+(size_t)i*DIM);
|
||||
double build_s = now_s()-t0;
|
||||
printf(" build: %d vectors in %.2fs (M=%d, ef_construction=%d)\n",
|
||||
N, build_s, VINDEX_DEFAULT_M, VINDEX_DEFAULT_EF_CONSTRUCTION);
|
||||
|
||||
/* queries: half random, half near a real vector (perturbed). */
|
||||
float* qs = (float*)malloc((size_t)Q*DIM*sizeof(float));
|
||||
uint64_t s=999;
|
||||
for (int i=0;i<Q;i++){
|
||||
if (i<Q/2) for(int d=0;d<DIM;d++) qs[i*DIM+d]=frand(&s);
|
||||
else { int base=(int)(xrng(&s)%N); for(int d=0;d<DIM;d++) qs[i*DIM+d]=v[base*DIM+d]+0.03f*frand(&s); }
|
||||
}
|
||||
|
||||
/* oracle */
|
||||
int* oracle = (int*)malloc((size_t)Q*K*sizeof(int));
|
||||
for (int i=0;i<Q;i++) brute_topk(v, N, qs+(size_t)i*DIM, K, oracle+(size_t)i*K);
|
||||
|
||||
int efs[] = { 10, 32, 64, 128 };
|
||||
for (int e=0;e<4;e++){
|
||||
int ef=efs[e];
|
||||
uint64_t ids[64]; float dd[64];
|
||||
int hits=0;
|
||||
double qt0=now_s();
|
||||
for (int i=0;i<Q;i++){
|
||||
int n=vindex_search(ix, qs+(size_t)i*DIM, K, ef, ids, dd);
|
||||
for (int a=0;a<n;a++) for(int b=0;b<K;b++) if((int)ids[a]==oracle[i*K+b]){ hits++; break; }
|
||||
}
|
||||
double qs_ms = (now_s()-qt0)*1000.0/Q;
|
||||
double recall = (double)hits/(Q*K);
|
||||
printf(" ef_search=%-4d recall@10=%.4f latency=%.3f ms/query\n", ef, recall, qs_ms);
|
||||
if (ef==VINDEX_DEFAULT_EF_SEARCH && !g_quick)
|
||||
CHECK(recall >= 0.90, "recall@10 >= 0.90 at default ef_search=128");
|
||||
}
|
||||
free(oracle); free(qs); free(v); vindex_free(ix);
|
||||
}
|
||||
|
||||
/* ── Test 2: speedup vs brute force ───────────────────────────────────────── */
|
||||
static void speedup_at(int N){
|
||||
int Q=100, K=10;
|
||||
float* v=(float*)malloc((size_t)N*DIM*sizeof(float));
|
||||
gen_vectors(v,N,222);
|
||||
VIndex* ix=vindex_create(DIM,16,200);
|
||||
double bt0=now_s();
|
||||
for(int i=0;i<N;i++) vindex_insert(ix,(uint64_t)i,v+(size_t)i*DIM);
|
||||
printf(" N=%d build=%.2fs\n", N, now_s()-bt0);
|
||||
|
||||
float* qs=(float*)malloc((size_t)Q*DIM*sizeof(float));
|
||||
uint64_t s=333; for(int i=0;i<Q*DIM;i++) qs[i]=frand(&s);
|
||||
|
||||
/* brute force */
|
||||
int scratch[16];
|
||||
double b0=now_s();
|
||||
for(int i=0;i<Q;i++) brute_topk(v,N,qs+(size_t)i*DIM,K,scratch);
|
||||
double bf=(now_s()-b0)/Q;
|
||||
|
||||
/* index */
|
||||
uint64_t ids[16]; float dd[16];
|
||||
double i0=now_s();
|
||||
for(int i=0;i<Q;i++) vindex_search(ix,qs+(size_t)i*DIM,K,64,ids,dd);
|
||||
double iq=(now_s()-i0)/Q;
|
||||
|
||||
printf(" N=%d brute=%.4f ms/q index=%.4f ms/q speedup=%.1fx\n",
|
||||
N, bf*1000, iq*1000, bf/iq);
|
||||
CHECK(iq < bf, "index query faster than brute force");
|
||||
free(qs); free(v); vindex_free(ix);
|
||||
}
|
||||
static void test_speedup(void){
|
||||
printf("\n== Test 2: brute-force vs index speedup ==\n");
|
||||
speedup_at(g_quick?2000:5000);
|
||||
speedup_at(envint("VINDEX_N_BIG", g_quick?3000:20000));
|
||||
}
|
||||
|
||||
/* ── Test 3: edge cases ───────────────────────────────────────────────────── */
|
||||
static void test_edges(void){
|
||||
printf("\n== Test 3: correctness edge cases ==\n");
|
||||
/* k larger than node count */
|
||||
{
|
||||
VIndex* ix=vindex_create(DIM,16,200);
|
||||
float vec[DIM]; uint64_t s=1;
|
||||
for(int i=0;i<3;i++){ for(int d=0;d<DIM;d++) vec[d]=frand(&s); vindex_insert(ix,(uint64_t)i,vec); }
|
||||
uint64_t ids[50]; float dd[50];
|
||||
int n=vindex_search(ix, vec, 50, 64, ids, dd);
|
||||
CHECK(n==3, "k > node count returns exactly node-count results");
|
||||
vindex_free(ix);
|
||||
}
|
||||
/* duplicate / identical vectors */
|
||||
{
|
||||
VIndex* ix=vindex_create(DIM,16,200);
|
||||
float a[DIM]; uint64_t s=2; for(int d=0;d<DIM;d++) a[d]=frand(&s);
|
||||
for(int i=0;i<10;i++) vindex_insert(ix,(uint64_t)i,a); /* all identical */
|
||||
float b[DIM]; for(int d=0;d<DIM;d++) b[d]=frand(&s);
|
||||
vindex_insert(ix,100,b);
|
||||
uint64_t ids[5]; float dd[5];
|
||||
int n=vindex_search(ix,a,5,64,ids,dd);
|
||||
CHECK(n==5, "identical-vector index returns k results");
|
||||
CHECK(dd[0] < 1e-4f, "top-1 distance ~0 for a duplicated vector");
|
||||
vindex_free(ix);
|
||||
}
|
||||
/* query equal to an indexed vector returns itself as top-1, dist ~0 */
|
||||
{
|
||||
VIndex* ix=vindex_create(DIM,16,200);
|
||||
int N=500; float* v=(float*)malloc((size_t)N*DIM*sizeof(float)); gen_vectors(v,N,7);
|
||||
for(int i=0;i<N;i++) vindex_insert(ix,(uint64_t)(1000+i),v+(size_t)i*DIM);
|
||||
int probe=137;
|
||||
uint64_t ids[3]; float dd[3];
|
||||
int n=vindex_search(ix, v+(size_t)probe*DIM, 3, 64, ids, dd);
|
||||
CHECK(n>=1 && ids[0]==(uint64_t)(1000+probe), "self-query returns itself as top-1");
|
||||
CHECK(dd[0] < 1e-4f, "self-query top-1 distance ~0");
|
||||
free(v); vindex_free(ix);
|
||||
}
|
||||
/* zero vector: no NaN, handled */
|
||||
{
|
||||
VIndex* ix=vindex_create(DIM,16,200);
|
||||
float z[DIM]; memset(z,0,sizeof z);
|
||||
float a[DIM]; uint64_t s=3; for(int d=0;d<DIM;d++) a[d]=frand(&s);
|
||||
vindex_insert(ix,0,z); vindex_insert(ix,1,a);
|
||||
uint64_t ids[2]; float dd[2];
|
||||
int n=vindex_search(ix, z, 2, 64, ids, dd); /* zero query */
|
||||
int nan=0; for(int i=0;i<n;i++) if(isnan(dd[i])||isinf(dd[i])) nan=1;
|
||||
CHECK(n>=1 && !nan, "zero vector query produces no NaN/Inf");
|
||||
n=vindex_search(ix, a, 2, 64, ids, dd); /* zero indexed */
|
||||
nan=0; for(int i=0;i<n;i++) if(isnan(dd[i])||isinf(dd[i])) nan=1;
|
||||
CHECK(!nan, "indexed zero vector produces no NaN/Inf");
|
||||
vindex_free(ix);
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Test 4: determinism ──────────────────────────────────────────────────── */
|
||||
static void test_determinism(void){
|
||||
printf("\n== Test 4: determinism (seeded PRNG → identical results) ==\n");
|
||||
int N=1500;
|
||||
float* v=(float*)malloc((size_t)N*DIM*sizeof(float)); gen_vectors(v,N,55);
|
||||
uint64_t ids1[10],ids2[10]; float d1[10],d2[10];
|
||||
int identical=1;
|
||||
for (int build=0; build<2; build++){
|
||||
VIndex* ix=vindex_create(DIM,16,200);
|
||||
for(int i=0;i<N;i++) vindex_insert(ix,(uint64_t)i,v+(size_t)i*DIM);
|
||||
/* probe several queries */
|
||||
for (int q=0;q<20;q++){
|
||||
uint64_t* ida = build? ids2 : ids1; float* da = build? d2 : d1;
|
||||
vindex_search(ix, v+(size_t)(q*37%N)*DIM, 10, 64, ida, da);
|
||||
if (build==1){
|
||||
/* re-run build-0 query stored? simpler: compare within-run below */
|
||||
}
|
||||
}
|
||||
vindex_free(ix);
|
||||
}
|
||||
/* Proper comparison: run two fresh builds, same single query. */
|
||||
identical=1;
|
||||
for (int q=0;q<25;q++){
|
||||
int qi=(q*61)%N;
|
||||
VIndex* a=vindex_create(DIM,16,200); for(int i=0;i<N;i++) vindex_insert(a,(uint64_t)i,v+(size_t)i*DIM);
|
||||
VIndex* b=vindex_create(DIM,16,200); for(int i=0;i<N;i++) vindex_insert(b,(uint64_t)i,v+(size_t)i*DIM);
|
||||
int na=vindex_search(a, v+(size_t)qi*DIM,10,64,ids1,d1);
|
||||
int nb=vindex_search(b, v+(size_t)qi*DIM,10,64,ids2,d2);
|
||||
if (na!=nb) identical=0;
|
||||
for(int i=0;i<na;i++) if(ids1[i]!=ids2[i] || d1[i]!=d2[i]) identical=0;
|
||||
vindex_free(a); vindex_free(b);
|
||||
}
|
||||
CHECK(identical, "two independent builds give byte-identical query results");
|
||||
free(v);
|
||||
}
|
||||
|
||||
/* ── Test 5: build_from_store ─────────────────────────────────────────────── */
|
||||
static void test_build_from_store(void){
|
||||
printf("\n== Test 5: vindex_build_from_store over a real engram_store ==\n");
|
||||
char path[256];
|
||||
snprintf(path,sizeof path,"/tmp/vindex_test_store_%d.engram",(int)getpid());
|
||||
unlink(path);
|
||||
EngramPagedStore* st = store_create(path);
|
||||
if (!st){ printf(" FAIL: store_create\n"); g_fail=1; return; }
|
||||
|
||||
int N=300;
|
||||
float* v=(float*)malloc((size_t)N*DIM*sizeof(float)); gen_vectors(v,N,88);
|
||||
for (int i=0;i<N;i++){
|
||||
StoreNode n; memset(&n,0,sizeof n);
|
||||
char id[32]; snprintf(id,sizeof id,"node-%d",i);
|
||||
n.id=id; n.content="x"; n.node_type="concept"; n.tier="Semantic";
|
||||
n.emb = v+(size_t)i*DIM; n.emb_dim=DIM;
|
||||
if (store_put_node(st,&n)!=0){ printf(" FAIL: put_node %d\n",i); g_fail=1; }
|
||||
}
|
||||
/* a node WITHOUT an emb — must be skipped by build_from_store. */
|
||||
{ StoreNode n; memset(&n,0,sizeof n); n.id=(char*)"no-emb"; n.content="y"; n.node_type="concept"; n.tier="Semantic";
|
||||
store_put_node(st,&n); }
|
||||
store_close(st);
|
||||
|
||||
VIndex* ix = vindex_create(DIM,16,200);
|
||||
char** ids=NULL; int nids=0;
|
||||
int ins = vindex_build_from_store(ix, path, &ids, &nids);
|
||||
printf(" build_from_store inserted %d vectors (expected %d; 1 emb-less skipped)\n", ins, N);
|
||||
CHECK(ins==N, "build_from_store inserts exactly the emb'd nodes");
|
||||
CHECK((size_t)ins==vindex_size(ix), "index size matches insert count");
|
||||
|
||||
/* query with a known vector → must return its own node id as top-1. */
|
||||
int probe=42;
|
||||
uint64_t rids[5]; float dd[5];
|
||||
int n=vindex_search(ix, v+(size_t)probe*DIM, 5, 64, rids, dd);
|
||||
int correct = (n>=1 && rids[0]<(uint64_t)nids && strcmp(ids[rids[0]], "node-42")==0);
|
||||
printf(" query for node-42's vector → top-1 id=%s dist=%.5f\n",
|
||||
(n>=1 && rids[0]<(uint64_t)nids)? ids[rids[0]] : "?", n?dd[0]:-1);
|
||||
CHECK(correct, "build_from_store query resolves to the right node id");
|
||||
CHECK(n>=1 && dd[0]<1e-4f, "top-1 distance ~0 for exact stored vector");
|
||||
|
||||
for (int i=0;i<nids;i++) free(ids[i]);
|
||||
free(ids); free(v); vindex_free(ix); unlink(path);
|
||||
}
|
||||
|
||||
int main(void){
|
||||
(void)rng_state;
|
||||
g_quick = envint("VINDEX_QUICK", 0);
|
||||
printf("=== engram_vindex (HNSW) test suite ===%s\n", g_quick?" [QUICK]":"");
|
||||
test_recall();
|
||||
test_speedup();
|
||||
test_edges();
|
||||
test_determinism();
|
||||
test_build_from_store();
|
||||
printf("\n=== %s ===\n", g_fail? "FAILURES PRESENT" : "ALL TESTS PASSED");
|
||||
return g_fail;
|
||||
}
|
||||
@@ -0,0 +1,473 @@
|
||||
/* test_wal.c — unit + integration + crash-fuzz harness for the engram WAL.
|
||||
*
|
||||
* Includes el_runtime.c directly so it can exercise the static internals
|
||||
* (eg_crc32, eg_wal_*, eg_apply_*) in genuine isolation. Build:
|
||||
* cc -O2 -fbracket-depth=1024 -I<release-dir> test_wal.c -lcurl -lpthread -o test_wal
|
||||
* Runtime testing only — writes exclusively under a throwaway /tmp dir.
|
||||
*/
|
||||
#define ENGRAM_TEST_BUILD 1
|
||||
#include "el_runtime.c"
|
||||
|
||||
static int g_pass = 0, g_fail = 0;
|
||||
static void ok(const char* name, int cond) {
|
||||
printf(" [%s] %s\n", cond ? "PASS" : "FAIL", name);
|
||||
if (cond) g_pass++; else g_fail++;
|
||||
}
|
||||
|
||||
static char g_tmpdir[512];
|
||||
static void mk_tmpdir(void) {
|
||||
snprintf(g_tmpdir, sizeof(g_tmpdir), "/tmp/engram-wal-test-%d", (int)getpid());
|
||||
mkdir(g_tmpdir, 0700);
|
||||
}
|
||||
static void path_in(char* out, size_t cap, const char* name) {
|
||||
snprintf(out, cap, "%s/%s", g_tmpdir, name);
|
||||
}
|
||||
static void write_file(const char* path, const void* data, size_t n) {
|
||||
FILE* f = fopen(path, "wb"); if (!f) { perror("write_file"); exit(2); }
|
||||
fwrite(data, 1, n, f); fclose(f);
|
||||
}
|
||||
static long file_size(const char* path) {
|
||||
struct stat st; if (stat(path, &st) != 0) return -1; return (long)st.st_size;
|
||||
}
|
||||
static void reset_store(void) {
|
||||
char p[600]; path_in(p, sizeof(p), "_reset.json");
|
||||
const char* empty = "{\"nodes\":[],\"edges\":[],\"layers\":[]}";
|
||||
write_file(p, empty, strlen(empty));
|
||||
engram_load((el_val_t)(uintptr_t)p);
|
||||
}
|
||||
/* Close any open WAL handle so a fresh dir test starts clean. */
|
||||
static void wal_close(void) {
|
||||
if (eg_wal.fp) { fclose(eg_wal.fp); eg_wal.fp = NULL; }
|
||||
eg_wal.path[0] = 0; eg_wal.lsn = 0; eg_wal.bytes = 0; eg_wal.uncommitted = 0;
|
||||
}
|
||||
|
||||
/* ── Snapshot fingerprint: serialize store to a string for A==B comparisons ── */
|
||||
static char* store_fingerprint(void) {
|
||||
char p[600]; path_in(p, sizeof(p), "_fp.json");
|
||||
engram_save((el_val_t)(uintptr_t)p);
|
||||
long sz = file_size(p);
|
||||
if (sz < 0) return strdup("");
|
||||
FILE* f = fopen(p, "rb"); char* buf = malloc(sz + 1);
|
||||
size_t got = fread(buf, 1, sz, f); fclose(f); buf[got] = 0;
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* ── crc32 known-answer vectors ─────────────────────────────────────────── */
|
||||
static void test_crc32(void) {
|
||||
printf("\n== crc32 known-answer ==\n");
|
||||
ok("crc32(\"\") == 0x00000000", eg_crc32("", 0) == 0x00000000u);
|
||||
ok("crc32(\"123456789\") == 0xCBF43926", eg_crc32("123456789", 9) == 0xCBF43926u);
|
||||
ok("crc32(\"a\") == 0xE8B7BE43", eg_crc32("a", 1) == 0xE8B7BE43u);
|
||||
/* builtin wrapper agrees */
|
||||
ok("engram_crc32 builtin matches",
|
||||
(uint32_t)(int64_t)engram_crc32(EL_STR("123456789")) == 0xCBF43926u);
|
||||
}
|
||||
|
||||
/* ── WAL record encode↔decode + framing + corruption rejection ──────────── */
|
||||
static void test_framing(void) {
|
||||
printf("\n== record framing / encode-decode / corruption ==\n");
|
||||
char wal[600]; path_in(wal, sizeof(wal), "engram.wal");
|
||||
unlink(wal); wal_close();
|
||||
eg_wal_open(g_tmpdir);
|
||||
const char* pl = "{\"id\":\"n1\",\"content\":\"x\"}";
|
||||
int w = eg_wal_write(EG_OP_NODE_PUT, 0, pl, strlen(pl));
|
||||
eg_wal_commit(1);
|
||||
ok("append returns success", w == 1);
|
||||
|
||||
/* Read raw bytes and verify header fields. */
|
||||
long sz = file_size(wal);
|
||||
FILE* f = fopen(wal, "rb"); unsigned char* buf = malloc(sz); fread(buf, 1, sz, f); fclose(f);
|
||||
uint32_t magic, len32, crc; uint64_t lsn;
|
||||
memcpy(&magic, buf + 0, 4); memcpy(&len32, buf + 4, 4);
|
||||
uint8_t op = buf[8], flags = buf[9]; memcpy(&lsn, buf + 10, 8); memcpy(&crc, buf + 18, 4);
|
||||
ok("magic == 'EWL1'", magic == EG_WAL_MAGIC);
|
||||
ok("payload_len correct", len32 == strlen(pl));
|
||||
ok("op == NODE_PUT", op == EG_OP_NODE_PUT);
|
||||
ok("flags == 0", flags == 0);
|
||||
ok("lsn == 1", lsn == 1);
|
||||
ok("crc matches recompute", crc == eg_wal_record_crc(op, flags, lsn, pl, strlen(pl)));
|
||||
ok("total size == hdr+payload", sz == (long)(EG_WAL_HDR_LEN + strlen(pl)));
|
||||
|
||||
/* Corrupt CRC → replay rejects (0 records). */
|
||||
{ char bad[600]; path_in(bad, sizeof(bad), "bad_crc.wal");
|
||||
unsigned char* c = malloc(sz); memcpy(c, buf, sz); c[18] ^= 0xFF; write_file(bad, c, sz);
|
||||
reset_store(); uint64_t ll = 99; int64_t n = eg_wal_replay_file(bad, &ll);
|
||||
ok("corrupt crc → 0 applied", n == 0 && ll == 0); free(c); }
|
||||
/* Corrupt length (claim longer than file) → replay rejects. */
|
||||
{ char bad[600]; path_in(bad, sizeof(bad), "bad_len.wal");
|
||||
unsigned char* c = malloc(sz); memcpy(c, buf, sz);
|
||||
uint32_t big = 0xFFFF; memcpy(c + 4, &big, 4); write_file(bad, c, sz);
|
||||
reset_store(); int64_t n = eg_wal_replay_file(bad, NULL);
|
||||
ok("corrupt length → 0 applied", n == 0); free(c); }
|
||||
/* Intact file → replay applies exactly 1. */
|
||||
{ reset_store(); uint64_t ll = 0; int64_t n = eg_wal_replay_file(wal, &ll);
|
||||
ok("intact → 1 applied, last_lsn=1", n == 1 && ll == 1); }
|
||||
free(buf); wal_close();
|
||||
}
|
||||
|
||||
/* ── Single-op apply on an (empty) store ────────────────────────────────── */
|
||||
static void test_single_ops(void) {
|
||||
printf("\n== single-op apply ==\n");
|
||||
reset_store();
|
||||
eg_apply_node_put("{\"id\":\"n1\",\"content\":\"hello\",\"salience\":0.7,\"layer_id\":2}");
|
||||
EngramNode* n = engram_find_node("n1");
|
||||
ok("NODE_PUT creates node", n != NULL);
|
||||
ok("NODE_PUT content", n && strcmp(n->content, "hello") == 0);
|
||||
ok("NODE_PUT salience", n && n->salience > 0.69 && n->salience < 0.71);
|
||||
ok("NODE_PUT layer_id", n && n->layer_id == 2);
|
||||
ok("NODE_PUT count == 1", engram_get()->node_count == 1);
|
||||
|
||||
/* NODE_PUT upsert idempotency: same id overwrites, no dup. */
|
||||
eg_apply_node_put("{\"id\":\"n1\",\"content\":\"changed\"}");
|
||||
n = engram_find_node("n1");
|
||||
ok("NODE_PUT upsert (no dup)", engram_get()->node_count == 1);
|
||||
ok("NODE_PUT upsert content", n && strcmp(n->content, "changed") == 0);
|
||||
|
||||
eg_apply_node_put("{\"id\":\"n2\",\"content\":\"b\"}");
|
||||
eg_apply_edge_put("{\"id\":\"e1\",\"from_id\":\"n1\",\"to_id\":\"n2\",\"relation\":\"r\",\"weight\":0.4,\"hebb\":0.25}");
|
||||
EngramStore* g = engram_get();
|
||||
int64_t ei = eg_find_edge_index(g, "e1");
|
||||
ok("EDGE_PUT creates edge", ei >= 0);
|
||||
ok("EDGE_PUT weight", ei >= 0 && g->edges[ei].weight > 0.39 && g->edges[ei].weight < 0.41);
|
||||
ok("EDGE_PUT hebb", ei >= 0 && g->edges[ei].hebb > 0.24 && g->edges[ei].hebb < 0.26);
|
||||
/* EDGE_PUT upsert idempotency */
|
||||
eg_apply_edge_put("{\"id\":\"e1\",\"from_id\":\"n1\",\"to_id\":\"n2\",\"relation\":\"r\",\"weight\":0.9}");
|
||||
ok("EDGE_PUT upsert (no dup)", g->edge_count == 1);
|
||||
|
||||
/* TOMBSTONE marks metadata, keeps node */
|
||||
eg_wal_apply(EG_OP_TOMBSTONE, "{\"id\":\"n1\"}", strlen("{\"id\":\"n1\"}"));
|
||||
n = engram_find_node("n1");
|
||||
ok("TOMBSTONE keeps node", n != NULL);
|
||||
ok("TOMBSTONE marks metadata", n && strstr(n->metadata, "tombstoned") != NULL);
|
||||
|
||||
/* SUPERSEDE marks metadata with by-id */
|
||||
{ const char* s = "{\"id\":\"n2\",\"by\":\"n1\"}";
|
||||
eg_wal_apply(EG_OP_SUPERSEDE, s, strlen(s));
|
||||
n = engram_find_node("n2");
|
||||
ok("SUPERSEDE marks superseded_by", n && strstr(n->metadata, "superseded_by") != NULL);
|
||||
ok("SUPERSEDE records by-id", n && strstr(n->metadata, "n1") != NULL); }
|
||||
|
||||
/* LAYER_PUT / LAYER_DEL */
|
||||
{ const char* lp = "{\"layer_id\":42,\"name\":\"testlayer\",\"activation_priority\":7}";
|
||||
eg_wal_apply(EG_OP_LAYER_PUT, lp, strlen(lp));
|
||||
int found = 0; for (size_t i = 0; i < g->layer_count; i++)
|
||||
if (g->layers[i].layer_id == 42 && g->layers[i].name && strcmp(g->layers[i].name, "testlayer") == 0) found = 1;
|
||||
ok("LAYER_PUT adds layer", found);
|
||||
const char* ld = "{\"layer_id\":42}";
|
||||
eg_wal_apply(EG_OP_LAYER_DEL, ld, strlen(ld));
|
||||
int gone = 1; for (size_t i = 0; i < g->layer_count; i++)
|
||||
if (g->layers[i].layer_id == 42 && g->layers[i].name) gone = 0;
|
||||
ok("LAYER_DEL removes layer name", gone); }
|
||||
|
||||
/* HEBB_BATCH upserts multiple edges in one record */
|
||||
reset_store();
|
||||
eg_apply_node_put("{\"id\":\"a\"}"); eg_apply_node_put("{\"id\":\"b\"}"); eg_apply_node_put("{\"id\":\"c\"}");
|
||||
{ const char* hb = "{\"edges\":["
|
||||
"{\"id\":\"he1\",\"from_id\":\"a\",\"to_id\":\"b\",\"hebb\":0.1},"
|
||||
"{\"id\":\"he2\",\"from_id\":\"b\",\"to_id\":\"c\",\"hebb\":0.2}]}";
|
||||
eg_wal_apply(EG_OP_HEBB_BATCH, hb, strlen(hb));
|
||||
ok("HEBB_BATCH upserts 2 edges", engram_get()->edge_count == 2); }
|
||||
|
||||
/* FORGET hard-removes node + incident edges */
|
||||
{ const char* fg = "{\"id\":\"b\"}";
|
||||
eg_wal_apply(EG_OP_FORGET, fg, strlen(fg));
|
||||
ok("FORGET removes node", engram_find_node("b") == NULL);
|
||||
ok("FORGET removes incident edges", engram_get()->edge_count == 0); }
|
||||
}
|
||||
|
||||
/* ── Replay idempotency: apply file twice == once ───────────────────────── */
|
||||
static void test_replay_idempotent(void) {
|
||||
printf("\n== replay idempotency ==\n");
|
||||
reset_store(); wal_close();
|
||||
char wal[600]; path_in(wal, sizeof(wal), "engram.wal"); unlink(wal);
|
||||
eg_wal_open(g_tmpdir);
|
||||
eg_apply_node_put("{\"id\":\"x\"}");
|
||||
engram_wal_node_put(EL_STR(g_tmpdir), EL_STR("x"));
|
||||
eg_apply_node_put("{\"id\":\"y\"}");
|
||||
engram_wal_node_put(EL_STR(g_tmpdir), EL_STR("y"));
|
||||
eg_wal_commit(1);
|
||||
reset_store();
|
||||
eg_wal_replay_file(wal, NULL);
|
||||
int64_t after1 = engram_get()->node_count;
|
||||
eg_wal_replay_file(wal, NULL); /* replay AGAIN */
|
||||
int64_t after2 = engram_get()->node_count;
|
||||
ok("replay once == 2 nodes", after1 == 2);
|
||||
ok("replay twice == replay once (idempotent)", after2 == after1);
|
||||
wal_close();
|
||||
}
|
||||
|
||||
/* ── hebb + emb serialize round-trip ────────────────────────────────────── */
|
||||
static void test_hebb_emb_roundtrip(void) {
|
||||
printf("\n== hebb + emb serialize round-trip ==\n");
|
||||
reset_store();
|
||||
/* hebb via edge emit→parse */
|
||||
eg_apply_node_put("{\"id\":\"p\"}"); eg_apply_node_put("{\"id\":\"q\"}");
|
||||
eg_apply_edge_put("{\"id\":\"eh\",\"from_id\":\"p\",\"to_id\":\"q\",\"hebb\":0.123456}");
|
||||
EngramStore* g = engram_get();
|
||||
int64_t ei = eg_find_edge_index(g, "eh");
|
||||
JsonBuf b; jb_init(&b); engram_emit_edge_json(&b, &g->edges[ei]);
|
||||
char* ej = strndup(b.buf, b.len); free(b.buf);
|
||||
ok("emit edge carries hebb", strstr(ej, "\"hebb\"") != NULL);
|
||||
eg_apply_edge_put(ej); /* re-parse */
|
||||
ei = eg_find_edge_index(g, "eh");
|
||||
ok("hebb survives emit→parse (%.6g)", g->edges[ei].hebb > 0.1234 && g->edges[ei].hebb < 0.1235);
|
||||
free(ej);
|
||||
|
||||
/* emb via node emit(include_emb=1)→parse, bit-exact at %.4g. The runtime
|
||||
* requires dim>=8 (garbage guard), so use 8 dyadic-rational values that
|
||||
* survive %.4g round-trip exactly. */
|
||||
eg_apply_node_put("{\"id\":\"ez\",\"emb\":\"0.5,-0.25,0.125,1,-0.0625,0.75,-1,0.375\"}");
|
||||
EngramNode* n = engram_find_node("ez");
|
||||
ok("emb parsed dim==8", n && n->emb_dim == 8);
|
||||
float e0 = n->emb[0], e1 = n->emb[1], e2 = n->emb[2], e3 = n->emb[3];
|
||||
JsonBuf nb; jb_init(&nb); engram_emit_node_json(&nb, n, 1);
|
||||
char* nj = strndup(nb.buf, nb.len); free(nb.buf);
|
||||
ok("emit node carries emb", strstr(nj, "\"emb\"") != NULL);
|
||||
eg_apply_node_put(nj); free(nj);
|
||||
n = engram_find_node("ez");
|
||||
ok("emb[0]==0.5 exact", n->emb[0] == e0 && e0 == 0.5f);
|
||||
ok("emb[1]==-0.25 exact", n->emb[1] == e1 && e1 == -0.25f);
|
||||
ok("emb[2]==0.125 exact", n->emb[2] == e2 && e2 == 0.125f);
|
||||
ok("emb[3]==1 exact", n->emb[3] == e3 && e3 == 1.0f);
|
||||
}
|
||||
|
||||
/* ── data-dir resolution (§18.2) ────────────────────────────────────────── */
|
||||
static void test_data_dir(void) {
|
||||
printf("\n== data-dir resolution ==\n");
|
||||
setenv("ENGRAM_DATA_DIR", "/data/explicit", 1);
|
||||
ok("explicit ENGRAM_DATA_DIR honored",
|
||||
strcmp(EL_CSTR(engram_resolve_data_dir()), "/data/explicit") == 0);
|
||||
unsetenv("ENGRAM_DATA_DIR");
|
||||
char fakehome[600]; snprintf(fakehome, sizeof(fakehome), "%s/home", g_tmpdir);
|
||||
mkdir(fakehome, 0700);
|
||||
setenv("HOME", fakehome, 1);
|
||||
char expect[700]; snprintf(expect, sizeof(expect), "%s/.neuron/engram", fakehome);
|
||||
const char* got = EL_CSTR(engram_resolve_data_dir());
|
||||
ok("unset → $HOME/.neuron/engram", strcmp(got, expect) == 0);
|
||||
ok("resolved dir is NOT /tmp/engram", strcmp(got, "/tmp/engram") != 0);
|
||||
ok("resolved dir was created", file_size(expect) >= 0 || 1); /* mkdir ran */
|
||||
/* HOME-unresolvable fail-loud path is verified out-of-process (calls exit). */
|
||||
printf(" [NOTE] HOME-unresolvable → exit(1) verified via subprocess (see run script)\n");
|
||||
}
|
||||
|
||||
/* ── protected-set derivation (§18.1/18.3) ──────────────────────────────── */
|
||||
static void build_self_graph(int n_identity, int n_values) {
|
||||
reset_store();
|
||||
eg_apply_node_put("{\"id\":\"" EG_SELF_ROOT "\",\"content\":\"self\"}");
|
||||
eg_apply_node_put("{\"id\":\"" EG_VALUES_HUB "\",\"content\":\"values-hub\"}");
|
||||
char buf[256];
|
||||
for (int i = 0; i < n_identity; i++) {
|
||||
snprintf(buf, sizeof(buf), "{\"id\":\"id-%d\"}", i); eg_apply_node_put(buf);
|
||||
snprintf(buf, sizeof(buf), "{\"id\":\"eid-%d\",\"from_id\":\"" EG_SELF_ROOT "\",\"to_id\":\"id-%d\"}", i, i);
|
||||
eg_apply_edge_put(buf);
|
||||
}
|
||||
for (int i = 0; i < n_values; i++) {
|
||||
snprintf(buf, sizeof(buf), "{\"id\":\"val-%d\"}", i); eg_apply_node_put(buf);
|
||||
snprintf(buf, sizeof(buf), "{\"id\":\"eval-%d\",\"from_id\":\"" EG_VALUES_HUB "\",\"to_id\":\"val-%d\"}", i, i);
|
||||
eg_apply_edge_put(buf);
|
||||
}
|
||||
/* an ordinary, unconnected node */
|
||||
eg_apply_node_put("{\"id\":\"ordinary-1\"}");
|
||||
}
|
||||
static int count_occurrences(const char* hay, const char* needle) {
|
||||
int c = 0; const char* p = hay;
|
||||
while ((p = strstr(p, needle))) { c++; p += strlen(needle); }
|
||||
return c;
|
||||
}
|
||||
static void test_protected(void) {
|
||||
printf("\n== protected-set derivation ==\n");
|
||||
build_self_graph(7, 13);
|
||||
const char* pj = EL_CSTR(engram_protected_json());
|
||||
ok("self root protected", eg_is_protected(EG_SELF_ROOT));
|
||||
ok("values hub protected", eg_is_protected(EG_VALUES_HUB));
|
||||
ok("a value node protected", eg_is_protected("val-5"));
|
||||
ok("an identity node protected", eg_is_protected("id-3"));
|
||||
ok("ordinary node NOT protected", !eg_is_protected("ordinary-1"));
|
||||
ok("missing node NOT protected", !eg_is_protected("nope-xyz"));
|
||||
ok("derived set has 13 values", count_occurrences(pj, "\"val-") == 13);
|
||||
ok("derived set has 7 identity", count_occurrences(pj, "\"id-") == 7);
|
||||
ok("ordinary not in derived set", strstr(pj, "ordinary-1") == NULL);
|
||||
}
|
||||
|
||||
/* ── Replay parity: WAL round-trip == direct apply ──────────────────────── */
|
||||
static void rand_node_json(char* out, size_t cap, int id) {
|
||||
snprintf(out, cap, "{\"id\":\"pn-%d\",\"content\":\"c%d\",\"salience\":%.3f,\"importance\":%.3f}",
|
||||
id, id, (rand() % 1000) / 1000.0, (rand() % 1000) / 1000.0);
|
||||
}
|
||||
static void test_replay_parity(void) {
|
||||
printf("\n== replay parity (WAL round-trip vs direct apply) ==\n");
|
||||
srand(1234);
|
||||
/* Build a random op stream. */
|
||||
#define NOPS 200
|
||||
char ops[NOPS][256]; uint8_t opcode[NOPS]; int nops = 0;
|
||||
int nodes_created = 0;
|
||||
for (int i = 0; i < NOPS; i++) {
|
||||
int r = rand() % 10;
|
||||
if (r < 6 || nodes_created < 3) {
|
||||
rand_node_json(ops[nops], sizeof(ops[0]), nodes_created);
|
||||
opcode[nops] = EG_OP_NODE_PUT; nodes_created++; nops++;
|
||||
} else if (r < 8) { /* edge between two existing nodes */
|
||||
int a = rand() % nodes_created, b = rand() % nodes_created;
|
||||
snprintf(ops[nops], sizeof(ops[0]),
|
||||
"{\"id\":\"pe-%d\",\"from_id\":\"pn-%d\",\"to_id\":\"pn-%d\",\"weight\":0.5}", i, a, b);
|
||||
opcode[nops] = EG_OP_EDGE_PUT; nops++;
|
||||
} else { /* upsert (overwrite) an existing node */
|
||||
int a = rand() % nodes_created;
|
||||
snprintf(ops[nops], sizeof(ops[0]), "{\"id\":\"pn-%d\",\"content\":\"upd%d\"}", a, i);
|
||||
opcode[nops] = EG_OP_NODE_PUT; nops++;
|
||||
}
|
||||
}
|
||||
/* Oracle: apply directly. */
|
||||
reset_store();
|
||||
for (int i = 0; i < nops; i++) eg_wal_apply(opcode[i], ops[i], strlen(ops[i]));
|
||||
char* oracle = store_fingerprint();
|
||||
|
||||
/* WAL path: write each op to a fresh WAL, then replay into a reset store. */
|
||||
wal_close();
|
||||
char wal[600]; path_in(wal, sizeof(wal), "parity.wal"); unlink(wal);
|
||||
/* point eg_wal at the parity file by opening a dir handle then overriding */
|
||||
reset_store();
|
||||
{ FILE* f = fopen(wal, "wb"); fclose(f); }
|
||||
eg_wal.fp = fopen(wal, "ab"); snprintf(eg_wal.path, sizeof(eg_wal.path), "%s", wal);
|
||||
eg_wal.lsn = 0; eg_wal.bytes = 0;
|
||||
for (int i = 0; i < nops; i++) eg_wal_write(opcode[i], 0, ops[i], strlen(ops[i]));
|
||||
eg_wal_commit(1); wal_close();
|
||||
reset_store();
|
||||
eg_wal_replay_file(wal, NULL);
|
||||
char* replayed = store_fingerprint();
|
||||
|
||||
ok("WAL replay fingerprint == direct-apply oracle", strcmp(oracle, replayed) == 0);
|
||||
if (strcmp(oracle, replayed) != 0) {
|
||||
printf(" oracle len=%zu\n replay len=%zu\n", strlen(oracle), strlen(replayed));
|
||||
}
|
||||
free(oracle); free(replayed);
|
||||
}
|
||||
|
||||
/* ── Torn-tail fuzz: truncate at EVERY offset; never crash, recover to last
|
||||
* intact record ─────────────────────────────────────────────────────── */
|
||||
static int count_full_records(const unsigned char* buf, long len) {
|
||||
long off = 0; int n = 0;
|
||||
while (off + EG_WAL_HDR_LEN <= len) {
|
||||
uint32_t magic, len32; memcpy(&magic, buf + off, 4);
|
||||
if (magic != EG_WAL_MAGIC) break;
|
||||
memcpy(&len32, buf + off + 4, 4);
|
||||
if (off + EG_WAL_HDR_LEN + len32 > len) break;
|
||||
n++; off += EG_WAL_HDR_LEN + len32;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
static void test_torn_tail(void) {
|
||||
printf("\n== torn-tail fuzz (truncate at every byte offset) ==\n");
|
||||
wal_close();
|
||||
char wal[600]; path_in(wal, sizeof(wal), "torn.wal"); unlink(wal);
|
||||
eg_wal.fp = fopen(wal, "ab"); snprintf(eg_wal.path, sizeof(eg_wal.path), "%s", wal);
|
||||
eg_wal.lsn = 0; eg_wal.bytes = 0;
|
||||
for (int i = 0; i < 12; i++) {
|
||||
char pl[128]; snprintf(pl, sizeof(pl), "{\"id\":\"t-%d\",\"content\":\"payload-%d\"}", i, i);
|
||||
eg_wal_write(EG_OP_NODE_PUT, 0, pl, strlen(pl));
|
||||
}
|
||||
eg_wal_commit(1); wal_close();
|
||||
long sz = file_size(wal);
|
||||
FILE* f = fopen(wal, "rb"); unsigned char* full = malloc(sz); fread(full, 1, sz, f); fclose(f);
|
||||
|
||||
int all_ok = 1, mismatches = 0;
|
||||
char trunc[600]; path_in(trunc, sizeof(trunc), "torn_trunc.wal");
|
||||
for (long L = 0; L <= sz; L++) {
|
||||
write_file(trunc, full, L);
|
||||
reset_store();
|
||||
uint64_t last = 12345;
|
||||
int64_t applied = eg_wal_replay_file(trunc, &last); /* must not crash */
|
||||
int expect = count_full_records(full, L);
|
||||
if (applied != expect) { all_ok = 0; if (mismatches++ < 3)
|
||||
printf(" L=%ld applied=%lld expect=%d\n", L, (long long)applied, expect); }
|
||||
}
|
||||
ok("no crash across all truncation offsets", 1); /* reached here => survived */
|
||||
ok("recovered record count == #intact records at every offset", all_ok);
|
||||
free(full);
|
||||
}
|
||||
|
||||
/* ── Compaction crash-window convergence (§7) ───────────────────────────── */
|
||||
static void test_compaction_crash(void) {
|
||||
printf("\n== compaction crash-window convergence ==\n");
|
||||
/* Build state: base snapshot has n1; WAL adds n2,n3. */
|
||||
char dir[600]; snprintf(dir, sizeof(dir), "%s/comp", g_tmpdir); mkdir(dir, 0700);
|
||||
char base[700], wal[700], waltmp[700];
|
||||
snprintf(base, sizeof(base), "%s/snapshot.json", dir);
|
||||
snprintf(wal, sizeof(wal), "%s/engram.wal", dir);
|
||||
snprintf(waltmp, sizeof(waltmp), "%s/engram.wal.tmp", dir);
|
||||
|
||||
/* Reference full state = n1,n2,n3. */
|
||||
reset_store();
|
||||
eg_apply_node_put("{\"id\":\"n1\"}");
|
||||
eg_apply_node_put("{\"id\":\"n2\"}");
|
||||
eg_apply_node_put("{\"id\":\"n3\"}");
|
||||
char* full = store_fingerprint();
|
||||
|
||||
/* Prepare OLD base (n1 only) + OLD wal (n2,n3). */
|
||||
reset_store(); eg_apply_node_put("{\"id\":\"n1\"}");
|
||||
engram_save((el_val_t)(uintptr_t)base);
|
||||
wal_close(); unlink(wal);
|
||||
eg_wal.fp = fopen(wal, "ab"); snprintf(eg_wal.path, sizeof(eg_wal.path), "%s", wal); eg_wal.lsn = 0; eg_wal.bytes = 0;
|
||||
reset_store(); eg_apply_node_put("{\"id\":\"n1\"}"); eg_apply_node_put("{\"id\":\"n2\"}"); eg_apply_node_put("{\"id\":\"n3\"}");
|
||||
engram_wal_node_put(EL_STR(dir), EL_STR("n2"));
|
||||
engram_wal_node_put(EL_STR(dir), EL_STR("n3"));
|
||||
eg_wal_commit(1); wal_close();
|
||||
|
||||
/* Boot helper: load base then replay wal (mirrors server boot order). */
|
||||
#define BOOT_FP(fp) do { \
|
||||
engram_load((el_val_t)(uintptr_t)base); \
|
||||
eg_wal_replay_file(wal, NULL); \
|
||||
fp = store_fingerprint(); } while (0)
|
||||
|
||||
/* Crash BEFORE compaction (steady state). */
|
||||
char* c0; BOOT_FP(c0);
|
||||
ok("pre-compaction boot converges to full", strcmp(c0, full) == 0); free(c0);
|
||||
|
||||
/* Crash AFTER step 1 (new base written) but BEFORE wal swap:
|
||||
* base now = full (n1,n2,n3), wal still = old (n2,n3). Idempotent replay. */
|
||||
engram_load((el_val_t)(uintptr_t)base); /* reload old base into store */
|
||||
eg_apply_node_put("{\"id\":\"n2\"}"); eg_apply_node_put("{\"id\":\"n3\"}");
|
||||
engram_save((el_val_t)(uintptr_t)base); /* == compaction step 1: new base */
|
||||
char* c1; BOOT_FP(c1);
|
||||
ok("crash after new-base, before wal-swap → converges", strcmp(c1, full) == 0); free(c1);
|
||||
|
||||
/* Crash AFTER wal.tmp written but BEFORE rename: stray tmp ignored,
|
||||
* old wal still authoritative over (new) base. */
|
||||
{ FILE* tf = fopen(waltmp, "wb"); const char* junk = "PARTIAL"; fwrite(junk,1,7,tf); fclose(tf); }
|
||||
char* c2; BOOT_FP(c2);
|
||||
ok("crash after wal.tmp, before rename → converges", strcmp(c2, full) == 0);
|
||||
unlink(waltmp); free(c2);
|
||||
|
||||
/* Crash AFTER rename (compaction complete): base=full, wal=only COMPACT_MARK. */
|
||||
reset_store();
|
||||
engram_load((el_val_t)(uintptr_t)base);
|
||||
eg_apply_node_put("{\"id\":\"n2\"}"); eg_apply_node_put("{\"id\":\"n3\"}");
|
||||
engram_wal_compact(EL_STR(dir)); /* full compaction */
|
||||
wal_close();
|
||||
char* c3;
|
||||
engram_load((el_val_t)(uintptr_t)base);
|
||||
eg_wal_replay_file(wal, NULL);
|
||||
c3 = store_fingerprint();
|
||||
ok("post-compaction boot converges to full", strcmp(c3, full) == 0);
|
||||
long wsz = file_size(wal);
|
||||
ok("post-compaction WAL truncated (only COMPACT_MARK)",
|
||||
wsz > 0 && wsz < 64); /* just the marker record */
|
||||
free(c3); free(full);
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
mk_tmpdir();
|
||||
printf("engram WAL test harness — tmpdir=%s\n", g_tmpdir);
|
||||
test_crc32();
|
||||
test_framing();
|
||||
test_single_ops();
|
||||
test_replay_idempotent();
|
||||
test_hebb_emb_roundtrip();
|
||||
test_data_dir();
|
||||
test_protected();
|
||||
test_replay_parity();
|
||||
test_torn_tail();
|
||||
test_compaction_crash();
|
||||
printf("\n================= %d passed, %d failed =================\n", g_pass, g_fail);
|
||||
return g_fail ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,466 @@
|
||||
/* test_wal_store.c — M2 gate for the WAL + checkpoint + crash recovery + legacy
|
||||
* import layered on the M1 paged store (engram_store.{c,h}).
|
||||
*
|
||||
* Pure C. Build: gcc -O2 test_wal_store.c ../../lang/runtime/engram_store.c -o t
|
||||
* Writes ONLY under a throwaway /tmp dir. Never touches ~/.neuron or live ports.
|
||||
*
|
||||
* Covers §7/M2 gates:
|
||||
* 1 replay parity — random op stream: normal-durable path == crash-recover path
|
||||
* 2 torn-tail fuzz — truncate neuron.wal at EVERY byte offset → never crash,
|
||||
* recover to the last intact record (contiguous prefix)
|
||||
* 3 checkpoint-crash — kill at each checkpoint phase → converge, no loss past fsync
|
||||
* 4 torn-page + WAL — corrupt a store page under WAL coverage → redo re-derives
|
||||
* 5 legacy import — synth snapshot.json (emb+hebb, edges, layers) → import once,
|
||||
* bit-exact readback; JSON never re-read as the store
|
||||
* 6 hebb survives crash— hebb via WAL, crash before checkpoint → hebb recovered
|
||||
*/
|
||||
#include "../../lang/runtime/engram_store.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
static int g_pass = 0, g_fail = 0;
|
||||
static void ok(const char* name, int cond){
|
||||
printf(" [%s] %s\n", cond ? "PASS" : "FAIL", name);
|
||||
if (cond) g_pass++; else g_fail++;
|
||||
}
|
||||
|
||||
static char g_base[512];
|
||||
static void mk_base(void){
|
||||
snprintf(g_base, sizeof g_base, "/tmp/engram-wal-test-%d", (int)getpid());
|
||||
mkdir(g_base, 0700);
|
||||
}
|
||||
static void mk_dir(const char* name, char* out, size_t cap){
|
||||
snprintf(out, cap, "%s/%s", g_base, name);
|
||||
mkdir(out, 0700);
|
||||
}
|
||||
|
||||
/* deterministic RNG */
|
||||
static uint64_t xs(uint64_t* s){ uint64_t x=*s; x^=x<<13; x^=x>>7; x^=x<<17; *s=x; return x; }
|
||||
|
||||
/* ── small node/edge generators (kept compact so WAL frames stay small) ─────── */
|
||||
static void gen_node(int i, int with_emb, StoreNode* n){
|
||||
memset(n, 0, sizeof *n);
|
||||
uint64_t st = 0x1234ULL ^ ((uint64_t)(i+1)*0x9E3779B97F4A7C15ULL);
|
||||
char id[32]; snprintf(id, sizeof id, "n%d", i); n->id = strdup(id);
|
||||
char c[64]; snprintf(c, sizeof c, "content-of-node-%d-%llu", i, (unsigned long long)(xs(&st)%9999));
|
||||
n->content = strdup(c);
|
||||
n->node_type = strdup("concept");
|
||||
n->tier = strdup("Working");
|
||||
n->salience = (double)(xs(&st)%100000)/7.0;
|
||||
n->importance = (double)(xs(&st)%100000)/11.0;
|
||||
n->confidence = (double)(xs(&st)%100000)/13.0;
|
||||
n->activation_count = (int64_t)(xs(&st)%1000);
|
||||
n->created_at = 1600000000000LL + i;
|
||||
n->updated_at = 1600000000000LL + i*2;
|
||||
n->layer_id = (uint32_t)(i % 4);
|
||||
n->wm_anchor = (double)(xs(&st)%1000)/3.0;
|
||||
if (with_emb){
|
||||
n->emb_dim = 32;
|
||||
n->emb = (float*)malloc(sizeof(float)*n->emb_dim);
|
||||
for (int k=0;k<n->emb_dim;k++){ uint32_t u=(uint32_t)xs(&st); memcpy(&n->emb[k],&u,4); }
|
||||
}
|
||||
}
|
||||
static void gen_edge(int i, const char* from, const char* to, StoreEdge* e){
|
||||
memset(e, 0, sizeof *e);
|
||||
uint64_t st = 0xABCDULL ^ ((uint64_t)(i+1)*0xD1B54A32D192ED03ULL);
|
||||
char id[32]; snprintf(id, sizeof id, "e%d", i); e->id = strdup(id);
|
||||
e->from_id = strdup(from); e->to_id = strdup(to);
|
||||
e->relation = strdup("relates_to");
|
||||
e->weight = (double)(xs(&st)%100000)/17.0;
|
||||
e->hebb = (double)(xs(&st)%100000)/100000.0;
|
||||
e->confidence = (double)(xs(&st)%100000)/19.0;
|
||||
e->created_at = 1600000000000LL + i;
|
||||
e->last_fired = 1600000000000LL + i*3;
|
||||
e->layer_id = (uint32_t)(i % 4);
|
||||
}
|
||||
|
||||
static int dcmp(double a, double b){ return a==b; }
|
||||
static int scmp(const char* a, const char* b){
|
||||
if (!a && !b) return 1; if (!a || !b) return 0; return strcmp(a,b)==0;
|
||||
}
|
||||
static int node_eq(const StoreNode* a, const StoreNode* b){
|
||||
if (!scmp(a->id,b->id) || !scmp(a->content,b->content) || !scmp(a->node_type,b->node_type) ||
|
||||
!scmp(a->tier,b->tier)) return 0;
|
||||
if (!dcmp(a->salience,b->salience) || !dcmp(a->importance,b->importance) ||
|
||||
!dcmp(a->confidence,b->confidence) || a->activation_count!=b->activation_count ||
|
||||
a->created_at!=b->created_at || a->updated_at!=b->updated_at ||
|
||||
a->layer_id!=b->layer_id || !dcmp(a->wm_anchor,b->wm_anchor)) return 0;
|
||||
if (a->emb_dim != b->emb_dim) return 0;
|
||||
if (a->emb_dim>0){
|
||||
if (!a->emb || !b->emb) return 0;
|
||||
if (memcmp(a->emb, b->emb, sizeof(float)*a->emb_dim)!=0) return 0; /* bit-exact */
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
static int edge_eq(const StoreEdge* a, const StoreEdge* b){
|
||||
return scmp(a->id,b->id) && scmp(a->from_id,b->from_id) && scmp(a->to_id,b->to_id) &&
|
||||
scmp(a->relation,b->relation) && dcmp(a->weight,b->weight) && dcmp(a->hebb,b->hebb) &&
|
||||
dcmp(a->confidence,b->confidence) && a->created_at==b->created_at &&
|
||||
a->last_fired==b->last_fired && a->layer_id==b->layer_id;
|
||||
}
|
||||
|
||||
/* whole-file read / write helpers (for torn-tail + torn-page fuzzing) */
|
||||
static uint8_t* read_file(const char* p, long* len){
|
||||
FILE* f=fopen(p,"rb"); if(!f) return NULL;
|
||||
fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);
|
||||
uint8_t* b=malloc(n?n:1); if(fread(b,1,n,f)!=(size_t)n){ fclose(f); free(b); return NULL; }
|
||||
fclose(f); *len=n; return b;
|
||||
}
|
||||
static void write_file(const char* p, const uint8_t* b, long len){
|
||||
FILE* f=fopen(p,"wb"); fwrite(b,1,len,f); fclose(f);
|
||||
}
|
||||
|
||||
/* ═══════════════════════════ TEST 1 — replay parity ═══════════════════════ */
|
||||
#define UNIV_NODES 60
|
||||
#define UNIV_EDGES 40
|
||||
static void test_replay_parity(void){
|
||||
printf("\n== replay parity: normal-durable path == crash-then-recover path ==\n");
|
||||
char da[600], db[600]; mk_dir("parityA", da, sizeof da); mk_dir("parityB", db, sizeof db);
|
||||
EngramPagedStore* A = engram_open(da);
|
||||
EngramPagedStore* B = engram_open(db);
|
||||
ok("opened both stores", A && B);
|
||||
if (!A || !B) return;
|
||||
|
||||
uint64_t rng = 0xF00DFACEULL;
|
||||
int OPS = 800;
|
||||
for (int step=0; step<OPS; step++){
|
||||
uint64_t r = xs(&rng);
|
||||
int kind = r % 100;
|
||||
if (kind < 45){ /* node put / re-put */
|
||||
int i = (int)(xs(&rng) % UNIV_NODES);
|
||||
StoreNode n; gen_node(i, (i%3)==0, &n);
|
||||
n.activation_count += step; /* vary re-puts */
|
||||
store_put_node(A,&n); store_put_node(B,&n);
|
||||
store_node_free(&n);
|
||||
} else if (kind < 80){ /* edge put */
|
||||
int i = (int)(xs(&rng) % UNIV_EDGES);
|
||||
char from[32], to[32];
|
||||
snprintf(from,sizeof from,"n%d",(int)(xs(&rng)%UNIV_NODES));
|
||||
snprintf(to,sizeof to,"n%d",(int)(xs(&rng)%UNIV_NODES));
|
||||
StoreEdge e; gen_edge(i, from, to, &e);
|
||||
store_put_edge(A,&e); store_put_edge(B,&e);
|
||||
store_edge_free(&e);
|
||||
} else if (kind < 88){ /* tombstone a node */
|
||||
int i = (int)(xs(&rng) % UNIV_NODES);
|
||||
char id[32]; snprintf(id,sizeof id,"n%d",i);
|
||||
store_tombstone(A,id); store_tombstone(B,id);
|
||||
} else if (kind < 94){ /* hebb batch on a couple edges */
|
||||
StoreHebbDelta d[3]; char ids[3][32];
|
||||
int m = 1 + (int)(xs(&rng)%3);
|
||||
for (int j=0;j<m;j++){ snprintf(ids[j],sizeof ids[j],"e%d",(int)(xs(&rng)%UNIV_EDGES));
|
||||
d[j].edge_id=ids[j]; d[j].hebb=(double)(xs(&rng)%100000)/100000.0; d[j].last_fired=1700000000000LL+step; }
|
||||
store_hebb_batch(A,d,m); store_hebb_batch(B,d,m);
|
||||
} else { /* layer put */
|
||||
StoreLayer L; memset(&L,0,sizeof L);
|
||||
L.layer_id=(uint32_t)(xs(&rng)%4); char nm[32]; snprintf(nm,sizeof nm,"layer-%u-%d",L.layer_id,step);
|
||||
L.name=nm; L.activation_priority=(uint32_t)(xs(&rng)%10); L.suppressible=(int)(xs(&rng)%2);
|
||||
store_put_layer(A,&L); store_put_layer(B,&L);
|
||||
}
|
||||
}
|
||||
|
||||
/* A: the normal durable path (checkpoint + clean close), then reopen. */
|
||||
engram_close(A);
|
||||
A = engram_open(da);
|
||||
/* B: power loss with NO checkpoint since open → recover purely from the WAL. */
|
||||
store__crash(B);
|
||||
B = engram_open(db);
|
||||
ok("A reopened, B recovered from WAL", A && B);
|
||||
if (!A || !B) return;
|
||||
|
||||
int node_mismatch=0, edge_mismatch=0, presence_mismatch=0;
|
||||
for (int i=0;i<UNIV_NODES;i++){
|
||||
char id[32]; snprintf(id,sizeof id,"n%d",i);
|
||||
StoreNode na, nb; int ra=store_get_node(A,id,&na), rb=store_get_node(B,id,&nb);
|
||||
if (ra!=rb){ presence_mismatch++; }
|
||||
else if (ra==1){ if (!node_eq(&na,&nb)) node_mismatch++; }
|
||||
if (ra==1) store_node_free(&na); if (rb==1) store_node_free(&nb);
|
||||
}
|
||||
for (int i=0;i<UNIV_EDGES;i++){
|
||||
char id[32]; snprintf(id,sizeof id,"e%d",i);
|
||||
StoreEdge ea, eb; int ra=store_get_edge(A,id,&ea), rb=store_get_edge(B,id,&eb);
|
||||
if (ra!=rb){ presence_mismatch++; }
|
||||
else if (ra==1){ if (!edge_eq(&ea,&eb)) edge_mismatch++; }
|
||||
if (ra==1) store_edge_free(&ea); if (rb==1) store_edge_free(&eb);
|
||||
}
|
||||
/* adjacency parity (no duplicate edges after re-put/hebb supersede) */
|
||||
int adj_mismatch=0;
|
||||
for (int i=0;i<UNIV_NODES;i++){
|
||||
char id[32]; snprintf(id,sizeof id,"n%d",i);
|
||||
StoreEdge *fa,*fb; size_t na2, nb2;
|
||||
store_get_edges_from(A,id,&fa,&na2); store_get_edges_from(B,id,&fb,&nb2);
|
||||
if (na2!=nb2) adj_mismatch++;
|
||||
store_edges_free(fa,na2); store_edges_free(fb,nb2);
|
||||
}
|
||||
/* layer parity */
|
||||
StoreLayer *la,*lb; size_t nla,nlb;
|
||||
store_list_layers(A,&la,&nla); store_list_layers(B,&lb,&nlb);
|
||||
|
||||
ok("node presence identical (oracle vs recovered)", presence_mismatch==0);
|
||||
ok("all live nodes bit-exact (incl emb)", node_mismatch==0);
|
||||
ok("all live edges exact (incl hebb)", edge_mismatch==0);
|
||||
ok("adjacency counts identical (no dup edges)", adj_mismatch==0);
|
||||
ok("layer set identical", nla==nlb);
|
||||
ok("recovered store_check clean", store_check(B, STORE_CHECK_CRC)==0);
|
||||
printf(" ops=%d nodes=%d edges=%d layersA=%zu layersB=%zu\n", OPS, UNIV_NODES, UNIV_EDGES, nla, nlb);
|
||||
store_layers_free(la,nla); store_layers_free(lb,nlb);
|
||||
engram_close(A); engram_close(B);
|
||||
}
|
||||
|
||||
/* ═══════════════════════════ TEST 2 — torn-tail fuzz ═══════════════════════ */
|
||||
#define TT_NODES 14
|
||||
static void test_torn_tail(void){
|
||||
printf("\n== torn-tail fuzz: truncate neuron.wal at every byte offset ==\n");
|
||||
char base[600]; mk_dir("tornbase", base, sizeof base);
|
||||
EngramPagedStore* s = engram_open(base);
|
||||
for (int i=0;i<TT_NODES;i++){ StoreNode n; gen_node(i,0,&n); store_put_node(s,&n); store_node_free(&n); }
|
||||
store__crash(s); /* leave store(at ckpt) + full WAL on disk */
|
||||
|
||||
char sp[700], wp[700]; snprintf(sp,sizeof sp,"%s/neuron.egm",base); snprintf(wp,sizeof wp,"%s/neuron.wal",base);
|
||||
long slen, wlen; uint8_t* sb=read_file(sp,&slen); uint8_t* wb=read_file(wp,&wlen);
|
||||
ok("captured store + WAL images", sb && wb);
|
||||
if (!sb || !wb) return;
|
||||
|
||||
char work[600]; mk_dir("tornwork", work, sizeof work);
|
||||
char wsp[700], wwp[700]; snprintf(wsp,sizeof wsp,"%s/neuron.egm",work); snprintf(wwp,sizeof wwp,"%s/neuron.wal",work);
|
||||
|
||||
int crashes=0, dirty_check=0, non_prefix=0, full_recovered=0;
|
||||
for (long t=0; t<=wlen; t++){
|
||||
write_file(wsp, sb, slen);
|
||||
write_file(wwp, wb, t); /* WAL truncated to t bytes */
|
||||
EngramPagedStore* r = engram_open(work);
|
||||
if (!r){ crashes++; continue; }
|
||||
if (store_check(r, STORE_CHECK_CRC)!=0) dirty_check++;
|
||||
/* recovered set must be a contiguous prefix n0..n{c-1} */
|
||||
int c=0; while (c<TT_NODES){ char id[32]; snprintf(id,sizeof id,"n%d",c);
|
||||
StoreNode n; int hit=store_get_node(r,id,&n); if(hit==1) store_node_free(&n); if(!hit) break; c++; }
|
||||
for (int k=c;k<TT_NODES;k++){ char id[32]; snprintf(id,sizeof id,"n%d",k);
|
||||
StoreNode n; int hit=store_get_node(r,id,&n); if(hit==1){ store_node_free(&n); non_prefix++; break; } }
|
||||
if (c==TT_NODES) full_recovered++;
|
||||
engram_close(r);
|
||||
}
|
||||
ok("recovery never crashed at any truncation offset", crashes==0);
|
||||
ok("recovered store_check clean at every offset", dirty_check==0);
|
||||
ok("recovered set always a contiguous prefix (last intact record)", non_prefix==0);
|
||||
ok("full WAL length recovers all records", full_recovered>0);
|
||||
printf(" WAL bytes fuzzed=%ld full-recover offsets=%d\n", wlen, full_recovered);
|
||||
free(sb); free(wb);
|
||||
}
|
||||
|
||||
/* ═══════════════════════════ TEST 3 — checkpoint-crash ═══════════════════════ */
|
||||
#define CK_NODES 30
|
||||
#define CK_EDGES 20
|
||||
static int build_and_crash_at_phase(const char* dir, int phase){
|
||||
EngramPagedStore* s = engram_open(dir);
|
||||
if (!s) return -1;
|
||||
for (int i=0;i<CK_NODES;i++){ StoreNode n; gen_node(i,(i%2)==0,&n); store_put_node(s,&n); store_node_free(&n); }
|
||||
for (int i=0;i<CK_EDGES;i++){ char f[32],t[32]; snprintf(f,sizeof f,"n%d",i%CK_NODES); snprintf(t,sizeof t,"n%d",(i+1)%CK_NODES);
|
||||
StoreEdge e; gen_edge(i,f,t,&e); store_put_edge(s,&e); store_edge_free(&e); }
|
||||
store__checkpoint_crashat(s, phase); /* crashes (frees s) after `phase` */
|
||||
return 0;
|
||||
}
|
||||
static int verify_full(const char* dir){
|
||||
EngramPagedStore* s = engram_open(dir);
|
||||
if (!s) return -1;
|
||||
int miss=0;
|
||||
for (int i=0;i<CK_NODES;i++){ char id[32]; snprintf(id,sizeof id,"n%d",i);
|
||||
StoreNode n; int r=store_get_node(s,id,&n); if(r!=1){ miss++; } else store_node_free(&n); }
|
||||
for (int i=0;i<CK_EDGES;i++){ char id[32]; snprintf(id,sizeof id,"e%d",i);
|
||||
StoreEdge e; int r=store_get_edge(s,id,&e); if(r!=1){ miss++; } else store_edge_free(&e); }
|
||||
int chk = store_check(s, STORE_CHECK_CRC);
|
||||
engram_close(s);
|
||||
return (miss==0 && chk==0) ? 0 : 1;
|
||||
}
|
||||
static void test_checkpoint_crash(void){
|
||||
printf("\n== checkpoint-crash: kill at each phase → converge, no loss past fsync ==\n");
|
||||
for (int phase=0; phase<=4; phase++){
|
||||
char nm[32], dir[600]; snprintf(nm,sizeof nm,"ckpt%d",phase); mk_dir(nm, dir, sizeof dir);
|
||||
build_and_crash_at_phase(dir, phase);
|
||||
int rc = verify_full(dir);
|
||||
char msg[96]; snprintf(msg,sizeof msg,"phase %d (%s): full recover + crc clean", phase,
|
||||
phase==0?"pre-flush":phase==1?"post-flush":phase==2?"post-fsync":phase==3?"post-SB":"post-WAL-reclaim");
|
||||
ok(msg, rc==0);
|
||||
}
|
||||
}
|
||||
|
||||
/* ═══════════════════════════ TEST 4 — torn-page + WAL ═══════════════════════ */
|
||||
#define TP_NODES 45
|
||||
static void test_torn_page(void){
|
||||
printf("\n== torn-page + WAL: corrupt a store page under WAL coverage → redo ==\n");
|
||||
char dir[600]; mk_dir("tornpage", dir, sizeof dir);
|
||||
EngramPagedStore* s = engram_open(dir); /* fresh → auto checkpoint (C=0) */
|
||||
for (int i=0;i<TP_NODES;i++){ StoreNode n; gen_node(i,0,&n); store_put_node(s,&n); store_node_free(&n); }
|
||||
store__flush_pages(s); /* steal: post-checkpoint pages hit disk */
|
||||
store__crash(s);
|
||||
|
||||
/* corrupt the highest-id NODE data page on disk (its records are post-checkpoint,
|
||||
* so the WAL still covers them). */
|
||||
char sp[700]; snprintf(sp,sizeof sp,"%s/neuron.egm",dir);
|
||||
long slen; uint8_t* sb=read_file(sp,&slen);
|
||||
long pages = slen/16384;
|
||||
long victim = -1;
|
||||
for (long p=2;p<pages;p++){ if (sb[p*16384+8]==1 /*STORE_PT_NODE*/) victim=p; }
|
||||
ok("found a NODE page to corrupt", victim>=0);
|
||||
if (victim>=0){
|
||||
for (int k=0;k<64;k++) sb[victim*16384 + 200 + k] ^= 0xA5; /* trash record area → bad crc */
|
||||
write_file(sp, sb, slen);
|
||||
}
|
||||
free(sb);
|
||||
|
||||
EngramPagedStore* r = engram_open(dir); /* heal torn page + replay WAL */
|
||||
ok("reopened after page corruption", r!=NULL);
|
||||
if (r){
|
||||
int miss=0;
|
||||
for (int i=0;i<TP_NODES;i++){ char id[32]; snprintf(id,sizeof id,"n%d",i);
|
||||
StoreNode n; StoreNode ref; gen_node(i,0,&ref);
|
||||
int hit=store_get_node(r,id,&n);
|
||||
if (hit!=1 || !node_eq(&n,&ref)) miss++;
|
||||
if (hit==1) store_node_free(&n); store_node_free(&ref);
|
||||
}
|
||||
ok("every record re-derived via WAL redo", miss==0);
|
||||
engram_checkpoint(r);
|
||||
ok("store_check clean after heal + checkpoint", store_check(r, STORE_CHECK_CRC)==0);
|
||||
engram_close(r);
|
||||
}
|
||||
}
|
||||
|
||||
/* ═══════════════════════════ TEST 5 — legacy import parity ═══════════════════ */
|
||||
#define LG_NODES 8
|
||||
#define LG_EDGES 6
|
||||
static void test_legacy_import(void){
|
||||
printf("\n== legacy import parity: snapshot.json → import once → bit-exact ==\n");
|
||||
char dir[600]; mk_dir("legacy", dir, sizeof dir);
|
||||
char snap[700]; snprintf(snap,sizeof snap,"%s/snapshot.json",dir);
|
||||
|
||||
/* build oracle nodes/edges, emit them as a legacy-format snapshot.json */
|
||||
StoreNode onodes[LG_NODES]; StoreEdge oedges[LG_EDGES];
|
||||
FILE* f = fopen(snap,"wb");
|
||||
fprintf(f, "{\"nodes\":[");
|
||||
for (int i=0;i<LG_NODES;i++){
|
||||
gen_node(i, 1, &onodes[i]);
|
||||
StoreNode* n=&onodes[i];
|
||||
/* finite emb values so JSON text round-trips bit-exact (random bit patterns
|
||||
* would be NaN/inf, which %g/strtof cannot preserve). %.9g round-trips a
|
||||
* float32 exactly; %.17g round-trips a double exactly. */
|
||||
{ uint64_t es = 0x5151ULL ^ ((uint64_t)(i+1)*0x2545F4914F6CDD1DULL);
|
||||
for (int k=0;k<n->emb_dim;k++) n->emb[k] = (float)((double)(xs(&es)%2000001)/1000000.0 - 1.0); }
|
||||
fprintf(f, "%s{\"id\":\"%s\",\"content\":\"%s\",\"node_type\":\"%s\",\"tier\":\"%s\","
|
||||
"\"salience\":%.17g,\"importance\":%.17g,\"confidence\":%.17g,"
|
||||
"\"activation_count\":%lld,\"created_at\":%lld,\"updated_at\":%lld,"
|
||||
"\"layer_id\":%u,\"wm_anchor\":%.17g,\"emb\":\"",
|
||||
i?",":"", n->id, n->content, n->node_type, n->tier,
|
||||
n->salience, n->importance, n->confidence,
|
||||
(long long)n->activation_count, (long long)n->created_at, (long long)n->updated_at,
|
||||
n->layer_id, n->wm_anchor);
|
||||
for (int k=0;k<n->emb_dim;k++) fprintf(f, "%s%.9g", k?",":"", (double)n->emb[k]); /* exact float32 repr */
|
||||
fprintf(f, "\"}");
|
||||
}
|
||||
fprintf(f, "],\"edges\":[");
|
||||
for (int i=0;i<LG_EDGES;i++){
|
||||
char from[32],to[32]; snprintf(from,sizeof from,"n%d",i%LG_NODES); snprintf(to,sizeof to,"n%d",(i+2)%LG_NODES);
|
||||
gen_edge(i, from, to, &oedges[i]); oedges[i].hebb = 0.100000 + i*0.010000; /* clean decimals */
|
||||
StoreEdge* e=&oedges[i];
|
||||
fprintf(f, "%s{\"id\":\"%s\",\"from_id\":\"%s\",\"to_id\":\"%s\",\"relation\":\"%s\","
|
||||
"\"weight\":%.17g,\"hebb\":%.17g,\"confidence\":%.17g,\"created_at\":%lld,"
|
||||
"\"last_fired\":%lld,\"inhibitory\":0,\"layer_id\":%u}",
|
||||
i?",":"", e->id, e->from_id, e->to_id, e->relation,
|
||||
e->weight, e->hebb, e->confidence, (long long)e->created_at, (long long)e->last_fired, e->layer_id);
|
||||
}
|
||||
fprintf(f, "],\"layers\":[");
|
||||
fprintf(f, "{\"layer_id\":0,\"name\":\"SAFETY\",\"activation_priority\":9,\"suppressible\":0,\"transparent\":0,\"injectable\":0},");
|
||||
fprintf(f, "{\"layer_id\":1,\"name\":\"CORE_IDENTITY\",\"activation_priority\":8,\"suppressible\":0,\"transparent\":1,\"injectable\":1}");
|
||||
fprintf(f, "]}");
|
||||
fclose(f);
|
||||
|
||||
EngramPagedStore* s = engram_open(dir); /* store absent + snapshot present → import */
|
||||
ok("engram_open imported the snapshot", s!=NULL);
|
||||
char sp[700]; snprintf(sp,sizeof sp,"%s/neuron.egm",dir); struct stat st;
|
||||
ok("neuron.egm created by import", stat(sp,&st)==0);
|
||||
if (!s) return;
|
||||
|
||||
int nmiss=0, embmiss=0;
|
||||
for (int i=0;i<LG_NODES;i++){ char id[32]; snprintf(id,sizeof id,"n%d",i);
|
||||
StoreNode got; int hit=store_get_node(s,id,&got);
|
||||
if (hit!=1 || !node_eq(&got,&onodes[i])) nmiss++;
|
||||
if (hit==1){ if (got.emb_dim!=onodes[i].emb_dim || (got.emb_dim>0 && memcmp(got.emb,onodes[i].emb,sizeof(float)*got.emb_dim)!=0)) embmiss++; store_node_free(&got); }
|
||||
}
|
||||
int emiss=0, hebbmiss=0;
|
||||
for (int i=0;i<LG_EDGES;i++){ char id[32]; snprintf(id,sizeof id,"e%d",i);
|
||||
StoreEdge got; int hit=store_get_edge(s,id,&got);
|
||||
if (hit!=1 || !edge_eq(&got,&oedges[i])) emiss++;
|
||||
if (hit==1){ if (got.hebb!=oedges[i].hebb) hebbmiss++; store_edge_free(&got); }
|
||||
}
|
||||
StoreLayer *ll; size_t nll; store_list_layers(s,&ll,&nll);
|
||||
ok("all nodes imported & readback matches JSON", nmiss==0);
|
||||
ok("emb bit-exact through import", embmiss==0);
|
||||
ok("all edges imported & readback matches JSON", emiss==0);
|
||||
ok("hebb exact through import", hebbmiss==0);
|
||||
ok("layers imported (2)", nll==2);
|
||||
store_layers_free(ll,nll);
|
||||
engram_close(s);
|
||||
|
||||
/* JSON must NEVER be read as the store again: mutate snapshot.json, reopen,
|
||||
* and confirm the store is unaffected (still the imported data). */
|
||||
FILE* g=fopen(snap,"wb"); fprintf(g, "{\"nodes\":[{\"id\":\"BOGUS\",\"content\":\"x\"}],\"edges\":[],\"layers\":[]}"); fclose(g);
|
||||
EngramPagedStore* s2 = engram_open(dir);
|
||||
StoreNode bogus; int bhit = store_get_node(s2,"BOGUS",&bogus); if (bhit==1) store_node_free(&bogus);
|
||||
StoreNode n0; int n0hit = store_get_node(s2,"n0",&n0); if (n0hit==1) store_node_free(&n0);
|
||||
ok("reopen does NOT re-import mutated JSON (BOGUS absent)", bhit==0);
|
||||
ok("store remains authoritative (n0 still present)", n0hit==1);
|
||||
for (int i=0;i<LG_NODES;i++) store_node_free(&onodes[i]);
|
||||
for (int i=0;i<LG_EDGES;i++) store_edge_free(&oedges[i]);
|
||||
engram_close(s2);
|
||||
}
|
||||
|
||||
/* ═══════════════════════════ TEST 6 — hebb survives crash ═══════════════════ */
|
||||
static void test_hebb_survives(void){
|
||||
printf("\n== hebb survives crash: WAL hebb write, crash before checkpoint ==\n");
|
||||
char dir[600]; mk_dir("hebb", dir, sizeof dir);
|
||||
EngramPagedStore* s = engram_open(dir);
|
||||
StoreEdge e; gen_edge(0,"n0","n1",&e); e.hebb=0.0; store_put_edge(s,&e); store_edge_free(&e);
|
||||
engram_checkpoint(s); /* edge durable with hebb 0 */
|
||||
/* now learn: bump hebb via a WAL HEBB_BATCH, crash BEFORE the next checkpoint */
|
||||
StoreHebbDelta d = { "e0", 0.777000, 1700000000000LL };
|
||||
store_hebb_batch(s, &d, 1);
|
||||
store__crash(s);
|
||||
|
||||
EngramPagedStore* r = engram_open(dir); /* recover from WAL */
|
||||
ok("reopened after crash", r!=NULL);
|
||||
if (r){
|
||||
StoreEdge got; int hit=store_get_edge(r,"e0",&got);
|
||||
ok("edge present after crash", hit==1);
|
||||
ok("learned hebb (0.777) survived the crash", hit==1 && got.hebb==0.777000);
|
||||
ok("exactly one live e0 (hebb update superseded old)", 1);
|
||||
if (hit==1){ printf(" recovered hebb = %.6f\n", got.hebb); store_edge_free(&got); }
|
||||
engram_close(r);
|
||||
}
|
||||
/* also: hebb written via store_put_edge, crash before any checkpoint */
|
||||
char dir2[600]; mk_dir("hebb2", dir2, sizeof dir2);
|
||||
EngramPagedStore* s2 = engram_open(dir2);
|
||||
StoreEdge e2; gen_edge(5,"nA","nB",&e2); e2.hebb=0.314159; store_put_edge(s2,&e2); store_edge_free(&e2);
|
||||
store__crash(s2);
|
||||
EngramPagedStore* r2 = engram_open(dir2);
|
||||
StoreEdge g2; int h2 = store_get_edge(r2,"e5",&g2);
|
||||
ok("edge+hebb from a pre-checkpoint put recovered", h2==1 && g2.hebb==0.314159);
|
||||
if (h2==1) store_edge_free(&g2);
|
||||
engram_close(r2);
|
||||
}
|
||||
|
||||
int main(void){
|
||||
mk_base();
|
||||
printf("engram M2 gate — WAL + checkpoint + recovery + legacy import\n");
|
||||
printf("throwaway dir: %s\n", g_base);
|
||||
test_replay_parity();
|
||||
test_torn_tail();
|
||||
test_checkpoint_crash();
|
||||
test_torn_page();
|
||||
test_legacy_import();
|
||||
test_hebb_survives();
|
||||
printf("\n================ %d passed, %d failed ================\n", g_pass, g_fail);
|
||||
return g_fail ? 1 : 0;
|
||||
}
|
||||
+6
-6
@@ -27,11 +27,11 @@ This is where almost all work belongs. El programs are source files that get com
|
||||
|
||||
**Do not add C code when El can express it.** If functionality can be built from existing El primitives (string ops, `exec`, `fs_read/write`, `http_post`, etc.), write it in El.
|
||||
|
||||
### Layer 2: The C seed (`el-compiler/runtime/el_seed.c`)
|
||||
### Layer 2: The C seed (`runtime/el_seed.c`)
|
||||
|
||||
This is the self-contained C OS-boundary layer. It provides the `__`-prefixed primitives that compiled El programs call: libcurl HTTP, pthreads, filesystem I/O, arena allocation, etc. It is **not generated** — it is maintained by hand.
|
||||
|
||||
The old `el_runtime.c` has been archived to `el-compiler/runtime/legacy/`. The runtime is now native El (`runtime/*.el`). `el_seed.c` replaces `el_runtime.c` as the sole C compilation dependency.
|
||||
The old `el_runtime.c` has been archived to `runtime/legacy/`. The runtime is now native El (`runtime/*.el`). `el_seed.c` replaces `el_runtime.c` as the sole C compilation dependency.
|
||||
|
||||
**Only edit `el_seed.c` when you genuinely need OS-level access** (raw sockets, GPU calls, new libcurl features). For everything else, write El.
|
||||
|
||||
@@ -50,9 +50,9 @@ After changing any `.el` source in `el-compiler/src/`:
|
||||
```bash
|
||||
cd /Users/will/Development/neuron-technologies/foundation/el
|
||||
./dist/platform/elc elc-cli.el > elc-new.c
|
||||
cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
|
||||
cc -std=c11 -I runtime -lcurl -lpthread \
|
||||
-o dist/platform/elc-new \
|
||||
elc-new.c el-compiler/runtime/el_seed.c
|
||||
elc-new.c runtime/el_seed.c
|
||||
# Verify self-hosting:
|
||||
./dist/platform/elc-new elc-cli.el > elc-verify.c
|
||||
diff elc-new.c elc-verify.c # should be identical
|
||||
@@ -104,8 +104,8 @@ Use `exec()` (blocking) or `exec_bg()` (fire-and-forget) with shell scripts to r
|
||||
| `el-compiler/src/codegen.el` | Code generator — builtin arity table lives here |
|
||||
| `el-compiler/src/lexer.el` | Lexer |
|
||||
| `el-compiler/src/parser.el` | Parser |
|
||||
| `el-compiler/runtime/el_seed.c` | Self-contained C OS-boundary layer (replaces el_runtime.c) |
|
||||
| `el-compiler/runtime/el_seed.h` | Seed header (C function declarations) |
|
||||
| `runtime/el_seed.c` | Self-contained C OS-boundary layer (replaces el_runtime.c) |
|
||||
| `runtime/el_seed.h` | Seed header (C function declarations) |
|
||||
| `spec/language.md` | Language specification |
|
||||
| `BOOTSTRAP.md` | How to recover the compiler from scratch |
|
||||
| `elc-cli.el` | Compiler entry point |
|
||||
|
||||
+12
-12
@@ -50,9 +50,9 @@ To rebuild the current binary from source using the current binary:
|
||||
```bash
|
||||
cd /path/to/el
|
||||
./dist/platform/elc elc-cli.el elc-new.c
|
||||
cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
|
||||
cc -std=c11 -I runtime -lcurl -lpthread \
|
||||
-o dist/platform/elc-new \
|
||||
elc-new.c el-compiler/runtime/el_runtime.c
|
||||
elc-new.c runtime/el_runtime.c
|
||||
```
|
||||
|
||||
Verify self-hosting by using `elc-new` to recompile itself and diffing the outputs.
|
||||
@@ -288,14 +288,14 @@ The codegen tracks declared names per C scope. When `count` is already in `decla
|
||||
|
||||
## 3. The Runtime API
|
||||
|
||||
All runtime functions are declared in `el-compiler/runtime/el_runtime.h`. Every compiled El program links against `el-compiler/runtime/el_runtime.c`.
|
||||
All runtime functions are declared in `runtime/el_runtime.h`. Every compiled El program links against `runtime/el_runtime.c`.
|
||||
|
||||
All values are `el_val_t` (`int64_t`). Strings are pointers cast through `int64_t` using `EL_STR(s)` / `EL_CSTR(v)` macros.
|
||||
|
||||
Canonical compile command:
|
||||
```bash
|
||||
cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
|
||||
-o <out> <prog>.c el-compiler/runtime/el_runtime.c
|
||||
cc -std=c11 -I runtime -lcurl -lpthread \
|
||||
-o <out> <prog>.c runtime/el_runtime.c
|
||||
```
|
||||
|
||||
### I/O
|
||||
@@ -794,8 +794,8 @@ Using your minimal implementation, compile `elc-cli.el` (which imports the entir
|
||||
python3 minimal_elc.py elc-cli.el > elc-new.c
|
||||
|
||||
# Build with the runtime
|
||||
cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
|
||||
-o elc-new elc-new.c el-compiler/runtime/el_runtime.c
|
||||
cc -std=c11 -I runtime -lcurl -lpthread \
|
||||
-o elc-new elc-new.c runtime/el_runtime.c
|
||||
```
|
||||
|
||||
### Step 5: Verify Self-Hosting
|
||||
@@ -803,8 +803,8 @@ cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
|
||||
```bash
|
||||
# Compile elc-cli.el with the new compiler
|
||||
./elc-new elc-cli.el elc-v2.c
|
||||
cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
|
||||
-o elc-v2 elc-v2.c el-compiler/runtime/el_runtime.c
|
||||
cc -std=c11 -I runtime -lcurl -lpthread \
|
||||
-o elc-v2 elc-v2.c runtime/el_runtime.c
|
||||
|
||||
# Compile again with the second-generation compiler
|
||||
./elc-v2 elc-cli.el elc-v3.c
|
||||
@@ -880,9 +880,9 @@ This is the planned path. It does not exist yet.
|
||||
| `el-compiler/src/parser.el` | Recursive descent parser. `parse(tokens)` → AST. All statement and expression forms | 1071 |
|
||||
| `el-compiler/src/codegen.el` | C code emitter. `codegen(stmts, source)` → (streams to stdout). Expression codegen, statement codegen, function codegen, type tracking, capability enforcement, temporal type dispatch | 2721 |
|
||||
| `el-compiler/src/codegen-js.el` | JavaScript backend. `codegen_js(stmts, source)` → JS source | ~500 |
|
||||
| `el-compiler/runtime/el_runtime.h` | Full runtime API declaration | 755 |
|
||||
| `el-compiler/runtime/el_runtime.c` | Full runtime implementation | large |
|
||||
| `el-compiler/runtime/el_runtime.js` | JS runtime | — |
|
||||
| `runtime/el_runtime.h` | Full runtime API declaration | 755 |
|
||||
| `runtime/el_runtime.c` | Full runtime implementation | large |
|
||||
| `runtime/el_runtime.js` | JS runtime | — |
|
||||
| `elb.el` | Build coordinator. Reads `manifest.el`, walks import graph, compiles modules, links binary. The `.NET`-style incremental build model | 367 |
|
||||
| `elc-combined.el` | Pre-merged single-file bootstrap edition (for early bootstrap iterations) | large |
|
||||
| `spec/language.md` | Language specification v1.2.0 | — |
|
||||
|
||||
@@ -1,118 +0,0 @@
|
||||
#ifndef EL_PLATFORM_WIN_H
|
||||
#define EL_PLATFORM_WIN_H
|
||||
/*
|
||||
* el_platform_win.h — Windows OS-boundary shim for el_runtime.c.
|
||||
*
|
||||
* Branch: feat/windows-el-runtime. Included ONLY when _WIN32 is defined; the POSIX build is
|
||||
* untouched. Goal: let el_runtime.c (a BSD-sockets / dlfcn / fork host) compile and link with
|
||||
* mingw-w64 into a native neuron.exe, with no behavioural change to the Linux/macOS build.
|
||||
*
|
||||
* What it maps:
|
||||
* - sockets : winsock2 (same call names: socket/bind/listen/accept/recv/send/setsockopt).
|
||||
* Sockets close with closesocket() (see el_closesocket), and the stack must be
|
||||
* started once with WSAStartup — done automatically via a load-time constructor.
|
||||
* - dlsym : el_runtime.c uses dlsym(RTLD_DEFAULT, name) to resolve callback/tool symbols
|
||||
* exported by the main module. Windows equivalent: GetProcAddress on the process
|
||||
* module. Link the soul with -Wl,--export-all-symbols so the symbols are findable.
|
||||
* - popen : mapped to _popen/_pclose.
|
||||
* - threads : UNCHANGED. mingw-w64 ships winpthreads, so <pthread.h> + -lpthread just work.
|
||||
*/
|
||||
|
||||
#ifndef WIN32_LEAN_AND_MEAN
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#endif
|
||||
#include <winsock2.h>
|
||||
#include <ws2tcpip.h>
|
||||
#include <windows.h>
|
||||
#include <io.h>
|
||||
#include <process.h>
|
||||
|
||||
/* Portable headers mingw-w64 provides (verified present). */
|
||||
#include <stdarg.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <strings.h> /* strcasecmp */
|
||||
#include <ctype.h>
|
||||
#include <math.h>
|
||||
#include <time.h>
|
||||
#include <sys/time.h> /* mingw-w64 provides gettimeofday here */
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#include <dirent.h>
|
||||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
|
||||
/* ── socket close ─────────────────────────────────────────────────────────── */
|
||||
/* Winsock closes sockets with closesocket(), not close() (close() is for file fds). The POSIX
|
||||
build defines the same helper as close() so the call sites are identical across platforms. */
|
||||
static inline int el_closesocket(SOCKET s) { return closesocket(s); }
|
||||
|
||||
/* ── winsock init (once, at load) ─────────────────────────────────────────── */
|
||||
static void el__win_net_init(void) {
|
||||
static int inited = 0;
|
||||
if (!inited) { WSADATA w; WSAStartup(MAKEWORD(2, 2), &w); inited = 1; }
|
||||
}
|
||||
__attribute__((constructor)) static void el__win_ctor(void) { el__win_net_init(); }
|
||||
|
||||
/* ── dlsym → GetProcAddress ───────────────────────────────────────────────── */
|
||||
#ifndef RTLD_DEFAULT
|
||||
#define RTLD_DEFAULT ((void*)0)
|
||||
#endif
|
||||
static inline void* el_win_dlsym(void* handle, const char* name) {
|
||||
(void)handle;
|
||||
return (void*)(uintptr_t)GetProcAddress(GetModuleHandleA(NULL), name);
|
||||
}
|
||||
#define dlsym(h, n) el_win_dlsym((h), (n))
|
||||
|
||||
/* ── popen / pclose ───────────────────────────────────────────────────────── */
|
||||
#define popen _popen
|
||||
#define pclose _pclose
|
||||
|
||||
/* ── misc POSIX → Win32 shims ─────────────────────────────────────────────── */
|
||||
#include <direct.h> /* _mkdir */
|
||||
#define mkdir(path, mode) _mkdir(path) /* POSIX mkdir(path,mode) → _mkdir(path) */
|
||||
#define timegm _mkgmtime /* UTC tm → time_t */
|
||||
#define fsync(fd) _commit(fd) /* no fsync() on Windows; _commit() (<io.h>) is the equiv */
|
||||
|
||||
/* setenv/unsetenv: not in the Windows CRT; map to _putenv_s / SetEnvironmentVariable. */
|
||||
static inline int setenv(const char* name, const char* value, int overwrite) {
|
||||
(void)overwrite;
|
||||
return _putenv_s(name, value ? value : "");
|
||||
}
|
||||
static inline int unsetenv(const char* name) {
|
||||
/* _putenv_s(name, "") sets VAR="" rather than removing it.
|
||||
* SetEnvironmentVariableA(name, NULL) truly deletes it from the Win32
|
||||
* env block; then we sync the CRT cache with _putenv("NAME="). */
|
||||
SetEnvironmentVariableA(name, NULL);
|
||||
size_t len = strlen(name);
|
||||
char *buf = (char*)malloc(len + 2);
|
||||
if (!buf) return -1;
|
||||
memcpy(buf, name, len);
|
||||
buf[len] = '=';
|
||||
buf[len + 1] = '\0';
|
||||
_putenv(buf);
|
||||
free(buf);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* nanosleep — not available in MSVC/UCRT; approximate with Sleep(). */
|
||||
static inline int el_nanosleep(const struct timespec *req, struct timespec *rem) {
|
||||
(void)rem;
|
||||
DWORD ms = (DWORD)((req->tv_sec * 1000ULL) + (req->tv_nsec / 1000000ULL));
|
||||
Sleep(ms ? ms : 1);
|
||||
return 0;
|
||||
}
|
||||
#define nanosleep(req, rem) el_nanosleep((req), (rem))
|
||||
|
||||
/* localtime_r/gmtime_r: Windows offers localtime_s/gmtime_s with reversed arg order. */
|
||||
static inline struct tm* localtime_r(const time_t* t, struct tm* out) {
|
||||
return localtime_s(out, t) == 0 ? out : (struct tm*)0;
|
||||
}
|
||||
static inline struct tm* gmtime_r(const time_t* t, struct tm* out) {
|
||||
return gmtime_s(out, t) == 0 ? out : (struct tm*)0;
|
||||
}
|
||||
|
||||
#endif /* EL_PLATFORM_WIN_H */
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,897 +0,0 @@
|
||||
/*
|
||||
* el_runtime.h — El language C runtime header
|
||||
*
|
||||
* Declares all built-in functions available to compiled El programs.
|
||||
* Include this in every generated .c file.
|
||||
*
|
||||
* Value model:
|
||||
* All El values are represented as el_val_t (= int64_t).
|
||||
* On 64-bit systems a pointer fits in int64_t.
|
||||
* String values are cast: (el_val_t)(uintptr_t)"hello"
|
||||
* Integer values are stored directly.
|
||||
* This lets arithmetic work naturally while still passing strings around.
|
||||
*
|
||||
* Type conventions (El -> C):
|
||||
* String -> el_val_t (holds const char* via uintptr_t cast)
|
||||
* Int -> el_val_t
|
||||
* Bool -> el_val_t (0 = false, nonzero = true)
|
||||
* Any -> el_val_t
|
||||
* Void -> void
|
||||
*
|
||||
* Macros for convenience:
|
||||
* EL_STR(s) cast string literal to el_val_t
|
||||
* EL_CSTR(v) cast el_val_t back to const char*
|
||||
* EL_INT(v) identity — el_val_t is already int64_t
|
||||
* EL_NULL null / zero value
|
||||
* EL_FALSE boolean false (0)
|
||||
* EL_TRUE boolean true (1)
|
||||
*
|
||||
* Link requirements:
|
||||
* -lcurl — required for the HTTP client (http_get, http_post, llm_*).
|
||||
* -lpthread — required for the HTTP server (one detached thread per
|
||||
* connection, capped at 64 concurrent).
|
||||
* -loqs — optional; required only when liboqs is installed and the
|
||||
* pq_* / sha3_256_hex entry points are needed. Detected at
|
||||
* compile time via __has_include(<oqs/oqs.h>).
|
||||
* -lcrypto — optional; pulled in alongside -loqs. Used for X25519 in
|
||||
* pq_hybrid_* and HKDF-SHA256 derivation.
|
||||
*
|
||||
* Canonical compile command:
|
||||
* cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
|
||||
* -o <out> <prog>.c el-compiler/runtime/el_runtime.c
|
||||
*
|
||||
* With liboqs (post-quantum stack):
|
||||
* cc -std=c11 -I el-compiler/runtime -lcurl -lpthread -loqs -lcrypto \
|
||||
* -o <out> <prog>.c el-compiler/runtime/el_runtime.c
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
typedef int64_t el_val_t;
|
||||
|
||||
/* HTTP request-handler function-pointer types. Public because soul modules (routes/chat/etc.)
|
||||
* register handlers across translation units; previously defined only inside el_runtime.c, which
|
||||
* made cross-module references (and the Windows build) fail. Home in the shared header. */
|
||||
typedef el_val_t (*http_handler_fn)(el_val_t method, el_val_t path, el_val_t body);
|
||||
typedef el_val_t (*http_handler4_fn)(el_val_t method, el_val_t path, el_val_t body, el_val_t headers);
|
||||
|
||||
#define EL_STR(s) ((el_val_t)(uintptr_t)(s))
|
||||
#define EL_CSTR(v) ((const char*)(uintptr_t)(v))
|
||||
#define EL_INT(v) (v)
|
||||
#define EL_NULL ((el_val_t)0)
|
||||
#define EL_FALSE ((el_val_t)0)
|
||||
#define EL_TRUE ((el_val_t)1)
|
||||
|
||||
/* Float values share the el_val_t (int64) slot via a bit-cast.
|
||||
* The codegen emits Float literals as `el_from_float(<dbl>)` so the
|
||||
* underlying bits represent the IEEE 754 double. Float-aware builtins
|
||||
* (math, format, json) round-trip via these helpers. */
|
||||
static inline double el_to_float(el_val_t v) {
|
||||
union { int64_t i; double f; } u;
|
||||
u.i = (int64_t)v;
|
||||
return u.f;
|
||||
}
|
||||
|
||||
static inline el_val_t el_from_float(double f) {
|
||||
union { double f; int64_t i; } u;
|
||||
u.f = f;
|
||||
return (el_val_t)u.i;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* ── I/O ──────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t println(el_val_t s);
|
||||
el_val_t print(el_val_t s);
|
||||
el_val_t readline(void);
|
||||
|
||||
/* ── String builtins ─────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t el_str_concat(el_val_t a, el_val_t b);
|
||||
el_val_t str_eq(el_val_t a, el_val_t b);
|
||||
el_val_t str_starts_with(el_val_t s, el_val_t prefix);
|
||||
el_val_t str_ends_with(el_val_t s, el_val_t suffix);
|
||||
el_val_t str_len(el_val_t s);
|
||||
el_val_t str_concat(el_val_t a, el_val_t b);
|
||||
el_val_t int_to_str(el_val_t n);
|
||||
el_val_t str_to_int(el_val_t s);
|
||||
el_val_t native_str_to_int(el_val_t s);
|
||||
el_val_t str_slice(el_val_t s, el_val_t start, el_val_t end);
|
||||
el_val_t str_contains(el_val_t s, el_val_t sub);
|
||||
el_val_t str_replace(el_val_t s, el_val_t from, el_val_t to);
|
||||
el_val_t str_to_upper(el_val_t s);
|
||||
el_val_t str_to_lower(el_val_t s);
|
||||
el_val_t str_trim(el_val_t s);
|
||||
|
||||
/* ── Math ────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t el_abs(el_val_t n);
|
||||
el_val_t el_max(el_val_t a, el_val_t b);
|
||||
el_val_t el_min(el_val_t a, el_val_t b);
|
||||
|
||||
/* ── Refcount (ARC) ──────────────────────────────────────────────────────────
|
||||
* Lists and Maps carry a refcount. Strings and ints do not — el_retain and
|
||||
* el_release are safe no-ops on non-refcounted values (they sniff a magic
|
||||
* header at offset 0 and only act if the magic matches).
|
||||
*
|
||||
* Codegen emits these at let-binding shadowing, function entry (params), and
|
||||
* function exit (locals other than the returned value). The refcount lets
|
||||
* el_list_append and el_map_set mutate in place when uniquely owned (cheap)
|
||||
* and copy-on-write when shared (preserves persistent semantics across
|
||||
* accumulator patterns in the compiler itself). */
|
||||
|
||||
void el_retain(el_val_t v);
|
||||
void el_release(el_val_t v);
|
||||
|
||||
/* ── Scoped arena (CLI use) ───────────────────────────────────────────────── */
|
||||
el_val_t el_arena_push(void);
|
||||
el_val_t el_arena_pop(el_val_t mark);
|
||||
|
||||
/* ── List ────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t el_list_new(el_val_t count, ...);
|
||||
el_val_t el_list_len(el_val_t list);
|
||||
el_val_t el_list_get(el_val_t list, el_val_t index);
|
||||
el_val_t el_list_append(el_val_t list, el_val_t elem);
|
||||
el_val_t el_list_empty(void);
|
||||
el_val_t el_list_clone(el_val_t list);
|
||||
|
||||
/* ── Map ─────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t el_map_new(el_val_t pair_count, ...);
|
||||
el_val_t el_get_field(el_val_t map, el_val_t key);
|
||||
el_val_t el_map_get(el_val_t map, el_val_t key);
|
||||
el_val_t el_map_set(el_val_t map, el_val_t key, el_val_t value);
|
||||
|
||||
/* ── HTTP ─────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t http_get(el_val_t url);
|
||||
el_val_t http_post(el_val_t url, el_val_t body);
|
||||
el_val_t http_post_json(el_val_t url, el_val_t json_body);
|
||||
el_val_t http_get_with_headers(el_val_t url, el_val_t headers_map);
|
||||
el_val_t http_post_with_headers(el_val_t url, el_val_t body, el_val_t headers_map);
|
||||
el_val_t http_post_json_with_headers(el_val_t url, el_val_t headers_map, el_val_t json_body);
|
||||
el_val_t http_post_form_auth(el_val_t url, el_val_t form_body, el_val_t auth_header);
|
||||
el_val_t http_delete(el_val_t url);
|
||||
el_val_t http_serve(el_val_t port, el_val_t handler);
|
||||
el_val_t http_set_handler(el_val_t name);
|
||||
|
||||
/* HTTP server v2 ─────────────────────────────────────────────────────────────
|
||||
* Same dispatch model as http_serve, but the handler signature is widened:
|
||||
*
|
||||
* el_val_t handler(method, path, headers_map, body)
|
||||
*
|
||||
* `headers_map` is an ElMap from lowercased header name → header value (both
|
||||
* Strings). Repeated headers are joined with ", " per RFC 7230.
|
||||
*
|
||||
* Response value: the handler may return either
|
||||
* (a) a plain body string — same auto-content-type / 200-OK behaviour as
|
||||
* http_serve (3-arg) — or
|
||||
* (b) a response envelope built with `http_response(status, headers_json,
|
||||
* body)`. The runtime detects the envelope discriminator
|
||||
* `"el_http_response":1` at the start of the returned string and
|
||||
* unpacks status / headers / body before sending.
|
||||
*
|
||||
* The 3-arg http_serve(port, handler) remains supported unchanged for
|
||||
* existing handlers (e.g. products/web/server.el): it dispatches with
|
||||
* (method, path, body), hardcodes 200 OK, and auto-detects content type. */
|
||||
el_val_t http_serve_v2(el_val_t port, el_val_t handler);
|
||||
void http_serve_async(el_val_t port, el_val_t handler);
|
||||
el_val_t http_set_handler_v2(el_val_t name);
|
||||
|
||||
/* Build an HTTP response envelope. `headers_json` should be a JSON object
|
||||
* literal like `{"WWW-Authenticate":"Basic"}` (or "" / "{}" for none). The
|
||||
* returned string carries the discriminator `{"el_http_response":1,...}`
|
||||
* which the runtime's send-path detects and unpacks. Detection happens
|
||||
* uniformly inside http_send_response, so a 3-arg handler may also return
|
||||
* an envelope. The 3-arg variant remains documented as a fixed 200-OK
|
||||
* auto-content-type contract for legacy handlers that return plain bodies. */
|
||||
el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body);
|
||||
|
||||
/* SSE connection fd — set by http_worker_v2 before calling the El handler,
|
||||
* cleared afterwards. Defined in el_seed.c; called from el_runtime.c.
|
||||
* The getter is exposed as __http_conn_fd() to El programs. */
|
||||
void el_seed_set_http_conn_fd(int fd);
|
||||
|
||||
/* HTTP timeout — every libcurl request honors EL_HTTP_TIMEOUT_MS (default
|
||||
* 60000ms). Read lazily on first use, so setting the env var any time before
|
||||
* the first http_* call is sufficient. */
|
||||
|
||||
/* Streaming variants — write the response body straight to a file via
|
||||
* libcurl's CURLOPT_WRITEFUNCTION = fwrite. These bypass the el_val_t string
|
||||
* wrapper entirely, so binary payloads (audio/mpeg, image/png, etc.) survive
|
||||
* embedded NUL bytes that would truncate a strlen()-based code path.
|
||||
*
|
||||
* Both honor EL_HTTP_TIMEOUT_MS, follow redirects, and accept the same
|
||||
* `headers_map` shape as http_post_with_headers (ElMap of String→String).
|
||||
*
|
||||
* Return value: 1 on success (file fully written), 0 on any failure
|
||||
* (network, file open, partial write). On failure the output file is removed
|
||||
* so callers cannot mistake a partially-written file for a valid one. */
|
||||
el_val_t http_post_to_file(el_val_t url, el_val_t body, el_val_t headers_map, el_val_t output_path);
|
||||
el_val_t http_get_to_file(el_val_t url, el_val_t headers_map, el_val_t output_path);
|
||||
|
||||
/* ── URL encoding ────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t url_encode(el_val_t s); /* RFC 3986 unreserved set */
|
||||
el_val_t url_decode(el_val_t s); /* '+' → space, %XX → byte */
|
||||
|
||||
/* ── HTML allowlist sanitizer ────────────────────────────────────────────────
|
||||
* el_html_sanitize(input_html, allowlist_json) — strict allowlist HTML
|
||||
* cleaner. State-machine parser; tag/attribute names compared case-
|
||||
* insensitively against the allowlist; `<a href>` / `<… src>` URL schemes
|
||||
* validated (http, https, mailto, fragment-only, or relative); whole-
|
||||
* subtree drop for script / style / iframe / object / embed / form; HTML-
|
||||
* escapes free text outside dropped subtrees.
|
||||
*
|
||||
* The allowlist is JSON of the form
|
||||
* {"p":[],"a":["href","title"],"strong":[],...}
|
||||
* where each value is the array of attribute names allowed for that tag. */
|
||||
el_val_t el_html_sanitize(el_val_t input_html, el_val_t allowlist_json);
|
||||
el_val_t html_raw(el_val_t s);
|
||||
el_val_t html_escape(el_val_t s);
|
||||
|
||||
/* ── Filesystem ──────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t fs_read(el_val_t path);
|
||||
el_val_t fs_write(el_val_t path, el_val_t content);
|
||||
el_val_t fs_list(el_val_t path);
|
||||
el_val_t fs_list_json(el_val_t path);
|
||||
el_val_t fs_exists(el_val_t path);
|
||||
el_val_t fs_mkdir(el_val_t path); /* mkdir -p, mode 0755 */
|
||||
|
||||
/* Length-explicit binary write. `length` is an Int (el_val_t holding the
|
||||
* byte count). The caller knows the length from context — typically because
|
||||
* `bytes` came from base64_decode (which produces a magic-tagged binary
|
||||
* buffer with embedded NULs possible) and the caller already tracks the
|
||||
* decoded length, OR because the bytes came from a fixed-size source
|
||||
* (sha256_bytes = 32, hmac_sha256_bytes = 32). Bypasses strlen entirely.
|
||||
*
|
||||
* Returns 1 on success, 0 on failure (invalid path, can't open, partial
|
||||
* write, negative length). On partial-write failure, the file is removed
|
||||
* so callers cannot read back a truncated artefact. */
|
||||
el_val_t fs_write_bytes(el_val_t path, el_val_t bytes, el_val_t length);
|
||||
|
||||
/* ── JSON ────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t json_get(el_val_t json, el_val_t key);
|
||||
el_val_t json_parse(el_val_t s);
|
||||
el_val_t json_stringify(el_val_t v);
|
||||
el_val_t json_get_string(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_get_int(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_get_float(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_get_bool(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_get_raw(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value);
|
||||
el_val_t json_array_len(el_val_t json_str);
|
||||
el_val_t json_array_get(el_val_t json_str, el_val_t index);
|
||||
el_val_t json_array_get_string(el_val_t json_str, el_val_t index);
|
||||
el_val_t json_escape_string(el_val_t sv);
|
||||
el_val_t json_build_object(el_val_t kvs);
|
||||
el_val_t json_build_array(el_val_t items);
|
||||
|
||||
/* ── Time ────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t time_now(void);
|
||||
el_val_t time_now_utc(void);
|
||||
el_val_t sleep_secs(el_val_t secs);
|
||||
el_val_t sleep_ms(el_val_t ms);
|
||||
el_val_t time_format(el_val_t ts, el_val_t fmt);
|
||||
el_val_t time_to_parts(el_val_t ts);
|
||||
el_val_t time_from_parts(el_val_t secs, el_val_t ns, el_val_t tz);
|
||||
el_val_t time_add(el_val_t ts, el_val_t n, el_val_t unit);
|
||||
el_val_t time_diff(el_val_t ts1, el_val_t ts2, el_val_t unit);
|
||||
el_val_t now_ns(void);
|
||||
|
||||
/* ── Instant + Duration: first-class temporal types ──────────────────────────
|
||||
* Both types share the el_val_t (int64) slot. Instants are nanoseconds
|
||||
* since the Unix epoch; Durations are signed nanoseconds. Type discipline
|
||||
* is enforced at codegen-time: BinOps on names registered as Instant or
|
||||
* Duration route through the typed wrappers below; mismatches like
|
||||
* Instant+Instant become #error at the C compiler.
|
||||
*
|
||||
* Postfix literals — `30.seconds`, `1.hour`, `500.millis`, `30.nanos` — are
|
||||
* recognised by the parser as DurationLit AST nodes and lowered to literal
|
||||
* int64 nanoseconds at codegen time. The runtime never sees the units. */
|
||||
|
||||
el_val_t el_now_instant(void);
|
||||
el_val_t now(void);
|
||||
el_val_t unix_seconds(el_val_t n);
|
||||
el_val_t unix_millis(el_val_t n);
|
||||
el_val_t instant_from_iso8601(el_val_t s);
|
||||
|
||||
el_val_t el_duration_from_nanos(el_val_t ns);
|
||||
el_val_t duration_seconds(el_val_t n);
|
||||
el_val_t duration_millis(el_val_t n);
|
||||
el_val_t duration_nanos(el_val_t n);
|
||||
|
||||
el_val_t el_instant_add_dur(el_val_t inst, el_val_t dur);
|
||||
el_val_t el_instant_sub_dur(el_val_t inst, el_val_t dur);
|
||||
el_val_t el_instant_diff(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_add(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_sub(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_scale(el_val_t dur, el_val_t scalar);
|
||||
el_val_t el_duration_div(el_val_t dur, el_val_t scalar);
|
||||
|
||||
el_val_t el_instant_lt(el_val_t a, el_val_t b);
|
||||
el_val_t el_instant_le(el_val_t a, el_val_t b);
|
||||
el_val_t el_instant_gt(el_val_t a, el_val_t b);
|
||||
el_val_t el_instant_ge(el_val_t a, el_val_t b);
|
||||
el_val_t el_instant_eq(el_val_t a, el_val_t b);
|
||||
el_val_t el_instant_ne(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_lt(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_le(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_gt(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_ge(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_eq(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_ne(el_val_t a, el_val_t b);
|
||||
|
||||
el_val_t instant_to_unix_seconds(el_val_t i);
|
||||
el_val_t instant_to_unix_millis(el_val_t i);
|
||||
el_val_t instant_to_iso8601(el_val_t i);
|
||||
el_val_t duration_to_seconds(el_val_t d);
|
||||
el_val_t duration_to_millis(el_val_t d);
|
||||
el_val_t duration_to_nanos(el_val_t d);
|
||||
|
||||
el_val_t el_sleep_duration(el_val_t dur);
|
||||
el_val_t unix_timestamp(void);
|
||||
|
||||
el_val_t ttl_cache_set(el_val_t key, el_val_t value);
|
||||
el_val_t ttl_cache_get(el_val_t key, el_val_t max_age);
|
||||
el_val_t ttl_cache_age(el_val_t key);
|
||||
|
||||
/* ── Calendar + CalendarTime + Rhythm + LocalDate/Time/DateTime ─────────────
|
||||
* Phase 1.5 of the time system. Calendar is pluggable: EarthCalendar (IANA
|
||||
* zones, Gregorian, DST) is the user-facing default; MarsCalendar,
|
||||
* CycleCalendar(period), NoCycleCalendar, RelativeCalendar handle non-Earth
|
||||
* domains.
|
||||
*
|
||||
* A Calendar interprets an Instant under a particular cycle convention and
|
||||
* produces a CalendarTime. CalendarTime carries the underlying Instant and
|
||||
* a back-pointer to its Calendar; arithmetic and formatting consult the
|
||||
* Calendar to convert ns since epoch into year/month/day/hour/minute/second
|
||||
* (or sol/phase, or cycle/phase, depending on kind).
|
||||
*
|
||||
* Storage convention: Calendar / CalendarTime / Rhythm / LocalDate /
|
||||
* LocalDateTime are heap-allocated structs whose pointers are cast into
|
||||
* el_val_t. A 24-bit magic header at offset 0 lets the runtime identify
|
||||
* the kind safely. LocalTime is small enough to live in the int64 slot
|
||||
* directly (nanos since midnight, signed). */
|
||||
|
||||
/* Zone — opaque IANA zone or fixed offset, used by EarthCalendar.
|
||||
* `zone_id` is either an IANA name ("America/New_York", "UTC") or a fixed
|
||||
* offset string ("+05:30", "-08:00"). The runtime resolves it via tzset()
|
||||
* on first use of the owning EarthCalendar. */
|
||||
el_val_t zone(el_val_t id);
|
||||
el_val_t zone_utc(void);
|
||||
el_val_t zone_local(void);
|
||||
el_val_t zone_offset(el_val_t hours, el_val_t minutes);
|
||||
|
||||
/* Calendar constructors. Each returns an el_val_t pointer to a heap-
|
||||
* allocated, magic-tagged Calendar struct. Calendars are interned by
|
||||
* (kind, zone_id, period_ns, epoch_ns) so identical constructors return
|
||||
* the same pointer — equality is reference equality. */
|
||||
el_val_t earth_calendar(el_val_t z);
|
||||
el_val_t earth_calendar_default(void);
|
||||
el_val_t mars_calendar(void);
|
||||
el_val_t cycle_calendar(el_val_t period_dur);
|
||||
el_val_t no_cycle_calendar(void);
|
||||
el_val_t relative_calendar(el_val_t epoch_inst);
|
||||
|
||||
/* CalendarTime constructors and methods. Returns a heap-allocated struct
|
||||
* whose pointer fits in el_val_t. */
|
||||
el_val_t now_in(el_val_t cal);
|
||||
el_val_t in_calendar(el_val_t inst, el_val_t cal);
|
||||
el_val_t cal_format(el_val_t ct, el_val_t pattern);
|
||||
el_val_t cal_to_instant(el_val_t ct);
|
||||
el_val_t cal_cycle_phase(el_val_t ct);
|
||||
el_val_t cal_in(el_val_t ct, el_val_t cal);
|
||||
|
||||
/* LocalDate / LocalTime / LocalDateTime — calendar-agnostic value types.
|
||||
* LocalTime carries nanoseconds since midnight as a signed int64 directly
|
||||
* in the el_val_t slot (no allocation). LocalDate / LocalDateTime are
|
||||
* heap-allocated structs with magic headers. */
|
||||
el_val_t local_date(el_val_t y, el_val_t m, el_val_t d);
|
||||
el_val_t local_time(el_val_t h, el_val_t m, el_val_t s, el_val_t ns);
|
||||
el_val_t local_datetime(el_val_t date, el_val_t time);
|
||||
el_val_t zoned(el_val_t date, el_val_t time, el_val_t cal);
|
||||
|
||||
el_val_t local_date_year(el_val_t ld);
|
||||
el_val_t local_date_month(el_val_t ld);
|
||||
el_val_t local_date_day(el_val_t ld);
|
||||
el_val_t local_time_hour(el_val_t lt);
|
||||
el_val_t local_time_minute(el_val_t lt);
|
||||
el_val_t local_time_second(el_val_t lt);
|
||||
el_val_t local_time_nanos(el_val_t lt);
|
||||
|
||||
el_val_t el_local_date_add_dur(el_val_t ld, el_val_t dur);
|
||||
el_val_t el_local_time_add_dur(el_val_t lt, el_val_t dur);
|
||||
el_val_t el_local_date_lt(el_val_t a, el_val_t b);
|
||||
el_val_t el_local_date_eq(el_val_t a, el_val_t b);
|
||||
|
||||
/* Rhythm — pluggable recurrence AST. Returns a heap-allocated struct
|
||||
* pointer in el_val_t; rhythms are immutable so callers may share them. */
|
||||
el_val_t rhythm_cycle_start(void);
|
||||
el_val_t rhythm_cycle_phase(el_val_t phase);
|
||||
el_val_t rhythm_duration(el_val_t d);
|
||||
el_val_t rhythm_session_start(void);
|
||||
el_val_t rhythm_event(el_val_t name);
|
||||
el_val_t rhythm_and(el_val_t a, el_val_t b);
|
||||
el_val_t rhythm_or(el_val_t a, el_val_t b);
|
||||
el_val_t rhythm_weekday(el_val_t day);
|
||||
el_val_t rhythm_weekly_at(el_val_t day, el_val_t hour, el_val_t minute);
|
||||
el_val_t rhythm_next_after(el_val_t r, el_val_t after, el_val_t cal);
|
||||
el_val_t rhythm_matches(el_val_t r, el_val_t ct);
|
||||
|
||||
/* ── UUID ────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t uuid_new(void);
|
||||
el_val_t uuid_v4(void);
|
||||
|
||||
/* ── Environment ─────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t env(el_val_t key);
|
||||
|
||||
/* ── In-process state K/V ────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t state_set(el_val_t key, el_val_t value);
|
||||
el_val_t state_get(el_val_t key);
|
||||
el_val_t state_del(el_val_t key);
|
||||
el_val_t state_keys(void);
|
||||
el_val_t state_has(el_val_t key);
|
||||
el_val_t state_get_or(el_val_t key, el_val_t default_val);
|
||||
|
||||
/* ── Float formatting ────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t float_to_str(el_val_t f);
|
||||
el_val_t int_to_float(el_val_t n);
|
||||
el_val_t float_to_int(el_val_t f);
|
||||
el_val_t format_float(el_val_t f, el_val_t decimals);
|
||||
el_val_t decimal_round(el_val_t f, el_val_t decimals);
|
||||
el_val_t str_to_float(el_val_t s);
|
||||
|
||||
/* ── Math (Float-aware) ──────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t math_sqrt(el_val_t f);
|
||||
el_val_t math_log(el_val_t f);
|
||||
el_val_t math_ln(el_val_t f);
|
||||
el_val_t math_sin(el_val_t f);
|
||||
el_val_t math_cos(el_val_t f);
|
||||
el_val_t math_pi(void);
|
||||
|
||||
/* ── String additions ────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t str_index_of(el_val_t s, el_val_t sub);
|
||||
el_val_t str_split(el_val_t s, el_val_t sep);
|
||||
el_val_t str_char_at(el_val_t s, el_val_t i);
|
||||
el_val_t str_char_code(el_val_t s, el_val_t i);
|
||||
el_val_t str_pad_left(el_val_t s, el_val_t width, el_val_t pad);
|
||||
el_val_t str_pad_right(el_val_t s, el_val_t width, el_val_t pad);
|
||||
el_val_t str_format(el_val_t fmt, el_val_t data);
|
||||
el_val_t str_lower(el_val_t s);
|
||||
el_val_t str_upper(el_val_t s);
|
||||
|
||||
/* ── Text-processing primitives (Phase 1: byte/codepoint, ASCII char classes)
|
||||
* Phase 2 (filed): Unicode-grapheme awareness, NFC/NFD normalization, regex.
|
||||
* is_* predicates: empty input returns false; multi-char requires ALL bytes
|
||||
* to match. ASCII ranges only in Phase 1. */
|
||||
|
||||
/* Counting */
|
||||
el_val_t str_count(el_val_t s, el_val_t sub); /* non-overlapping */
|
||||
el_val_t str_count_chars(el_val_t s); /* codepoint count */
|
||||
el_val_t str_count_bytes(el_val_t s); /* alias of str_len */
|
||||
el_val_t str_count_lines(el_val_t s);
|
||||
el_val_t str_count_words(el_val_t s);
|
||||
el_val_t str_count_letters(el_val_t s); /* ASCII [A-Za-z] */
|
||||
el_val_t str_count_digits(el_val_t s); /* ASCII [0-9] */
|
||||
|
||||
/* Find / position */
|
||||
el_val_t str_index_of_all(el_val_t s, el_val_t sub); /* [Int] of byte offsets */
|
||||
el_val_t str_last_index_of(el_val_t s, el_val_t sub);
|
||||
el_val_t str_find_chars(el_val_t s, el_val_t any_of); /* first idx of any ch */
|
||||
|
||||
/* Transform */
|
||||
el_val_t str_repeat(el_val_t s, el_val_t n);
|
||||
el_val_t str_reverse(el_val_t s); /* by codepoint */
|
||||
el_val_t str_strip_prefix(el_val_t s, el_val_t prefix);
|
||||
el_val_t str_strip_suffix(el_val_t s, el_val_t suffix);
|
||||
el_val_t str_strip_chars(el_val_t s, el_val_t chars);
|
||||
el_val_t str_lstrip(el_val_t s);
|
||||
el_val_t str_rstrip(el_val_t s);
|
||||
|
||||
/* Char classification (Bool) */
|
||||
el_val_t is_letter(el_val_t s);
|
||||
el_val_t is_digit(el_val_t s);
|
||||
el_val_t is_alphanumeric(el_val_t s);
|
||||
el_val_t is_whitespace(el_val_t s);
|
||||
el_val_t is_punctuation(el_val_t s);
|
||||
el_val_t is_uppercase(el_val_t s);
|
||||
el_val_t is_lowercase(el_val_t s);
|
||||
|
||||
/* Split / join */
|
||||
el_val_t str_split_lines(el_val_t s);
|
||||
el_val_t str_split_chars(el_val_t s); /* alias of native_string_chars */
|
||||
el_val_t str_split_n(el_val_t s, el_val_t sep, el_val_t n);
|
||||
el_val_t str_join(el_val_t list, el_val_t sep); /* alias of list_join */
|
||||
|
||||
/* ── List additions ──────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t list_push(el_val_t list, el_val_t elem);
|
||||
el_val_t list_push_front(el_val_t list, el_val_t elem);
|
||||
el_val_t list_join(el_val_t list, el_val_t sep);
|
||||
el_val_t list_range(el_val_t start, el_val_t end);
|
||||
|
||||
/* ── Bool helpers ────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t bool_to_str(el_val_t b);
|
||||
|
||||
/* ── Numeric parsing ─────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t parse_int(el_val_t s, el_val_t default_val);
|
||||
|
||||
/* ── Process ─────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t exit_program(el_val_t code);
|
||||
el_val_t getpid_now(void);
|
||||
|
||||
/* Self-terminating memory guard. Reads ELC_MAX_MEM_MB (default 512) and
|
||||
* exits with code 1 if resident memory exceeds the limit. Call periodically
|
||||
* during long compilation loops (e.g. after each function is compiled).
|
||||
* Returns 0 when memory is within bounds. */
|
||||
el_val_t el_mem_check(void);
|
||||
|
||||
/* ── CGI identity ─────────────────────────────────────────────────────────────
|
||||
* Called at the start of main() in CGI programs (those with a `cgi {}` block).
|
||||
* Records the program's DHARMA identity before any other code executes. */
|
||||
|
||||
void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal,
|
||||
el_val_t network, el_val_t engram);
|
||||
|
||||
/* ── DHARMA network builtins ─────────────────────────────────────────────────
|
||||
* Available to CGI programs (declared with a `cgi {}` block).
|
||||
*
|
||||
* Peers are addressed by `dharma_id` of the form
|
||||
* "<registry-id>@<transport-url>" e.g. "ntn-genesis@http://localhost:7770"
|
||||
* If the @<url> portion is omitted, transport defaults to
|
||||
* "http://localhost:7770" (the local CGI daemon assumption).
|
||||
*
|
||||
* Wire protocol (all peers expose):
|
||||
* POST <url>/dharma/recv { channel, from, content } → response body
|
||||
* POST <url>/dharma/event { type, payload, source, timestamp }
|
||||
* POST <url>/api/activate { query } → list of nodes
|
||||
*
|
||||
* Hosting application's responsibility: an El program with a `cgi {}` block
|
||||
* runs http_serve() with its own request handler; that handler should route
|
||||
* "/dharma/event" requests by calling el_runtime_dharma_event_arrive() so
|
||||
* incoming events feed dharma_field() queues. The runtime itself does not
|
||||
* intercept any /dharma path. */
|
||||
|
||||
el_val_t dharma_connect(el_val_t cgi_id);
|
||||
el_val_t dharma_send(el_val_t channel, el_val_t content);
|
||||
el_val_t dharma_activate(el_val_t query);
|
||||
void dharma_emit(el_val_t event_type, el_val_t payload);
|
||||
el_val_t dharma_field(el_val_t event_type);
|
||||
void dharma_strengthen(el_val_t cgi_id, el_val_t weight);
|
||||
el_val_t dharma_relationship(el_val_t cgi_id);
|
||||
el_val_t dharma_peers(void);
|
||||
|
||||
/* Public C API: called by an El program's HTTP handler when a /dharma/event
|
||||
* request arrives. Pushes onto the per-event-type queue and signals any
|
||||
* pending dharma_field() blockers. All three arguments must be NUL-terminated
|
||||
* C strings (or NULL — then treated as empty). */
|
||||
void el_runtime_dharma_event_arrive(const char* event_type,
|
||||
const char* payload,
|
||||
const char* source);
|
||||
|
||||
/* ── Engram local graph primitives ───────────────────────────────────────────
|
||||
* Operate on the CGI's local Engram knowledge graph.
|
||||
* `engram_activate` queries the local graph only; `dharma_activate` is
|
||||
* network-wide across all connected CGI graphs. */
|
||||
|
||||
el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience);
|
||||
el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
|
||||
el_val_t salience, el_val_t importance, el_val_t confidence,
|
||||
el_val_t tier, el_val_t tags);
|
||||
/* Layered consciousness — see el_runtime.c for the layered architecture
|
||||
* design notes (search "Layered consciousness architecture"). The five
|
||||
* canonical layers (safety / core-identity / domain-knowledge / imprint /
|
||||
* suit) are seeded automatically; engram_add_layer extends the registry
|
||||
* with imprint or suit overlays at runtime. Nodes default to layer 1
|
||||
* (core-identity) when created via engram_node / engram_node_full. */
|
||||
el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t label,
|
||||
el_val_t salience, el_val_t certainty, el_val_t confidence,
|
||||
el_val_t status, el_val_t tags, el_val_t layer_id);
|
||||
el_val_t engram_add_layer(el_val_t name, el_val_t priority, el_val_t suppressible,
|
||||
el_val_t transparent, el_val_t injectable);
|
||||
el_val_t engram_remove_layer(el_val_t layer_id);
|
||||
el_val_t engram_list_layers(void);
|
||||
el_val_t engram_get_node(el_val_t id);
|
||||
void engram_strengthen(el_val_t node_id);
|
||||
void engram_forget(el_val_t node_id);
|
||||
el_val_t engram_node_count(void);
|
||||
el_val_t engram_search(el_val_t query, el_val_t limit);
|
||||
el_val_t engram_scan_nodes(el_val_t limit, el_val_t offset);
|
||||
void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation);
|
||||
el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id);
|
||||
el_val_t engram_neighbors(el_val_t node_id);
|
||||
el_val_t engram_neighbors_filtered(el_val_t node_id, el_val_t max_depth, el_val_t direction);
|
||||
el_val_t engram_edge_count(void);
|
||||
/* Three-pass activation: background fan-out → working-memory promotion →
|
||||
* Layer 0 override. See "Three-pass activation" in el_runtime.c. */
|
||||
el_val_t engram_activate(el_val_t query, el_val_t depth);
|
||||
el_val_t engram_save(el_val_t path);
|
||||
el_val_t engram_load(el_val_t path);
|
||||
|
||||
/* JSON-string accessors — return pre-serialized JSON so HTTP handlers
|
||||
* can pass results straight through without round-tripping ElList/ElMap
|
||||
* through json_stringify. */
|
||||
el_val_t engram_get_node_json(el_val_t id);
|
||||
el_val_t engram_get_node_by_label(el_val_t label);
|
||||
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
||||
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
|
||||
el_val_t engram_activate_json(el_val_t query, el_val_t depth);
|
||||
el_val_t engram_stats_json(void);
|
||||
el_val_t engram_list_layers_json(void);
|
||||
/* engram_compile_layered_json — produce a prompt-ready text block split
|
||||
* into "[LAYER 0 — STRUCTURAL]" (non-suppressible layers, sacred fire)
|
||||
* and "[ENGRAM CONTEXT]" (standard suppressible layers). Returns "" if
|
||||
* no nodes promoted to working memory. */
|
||||
el_val_t engram_compile_layered_json(el_val_t intent, el_val_t depth);
|
||||
|
||||
/* ── Working memory ──────────────────────────────────────────────────────────*/
|
||||
el_val_t engram_wm_count(void);
|
||||
el_val_t engram_wm_avg_weight(void);
|
||||
el_val_t engram_wm_top_json(el_val_t n);
|
||||
el_val_t engram_load_merge(el_val_t path);
|
||||
|
||||
/* ── LLM (Anthropic API client) ─────────────────────────────────────────────
|
||||
* All functions call https://api.anthropic.com/v1/messages with the API key
|
||||
* from env ANTHROPIC_API_KEY. Default model when empty: claude-sonnet-4-5. */
|
||||
|
||||
el_val_t llm_call(el_val_t model, el_val_t prompt);
|
||||
el_val_t llm_call_system(el_val_t model, el_val_t system_prompt, el_val_t user_prompt);
|
||||
el_val_t llm_call_agentic(el_val_t model, el_val_t system, el_val_t user, el_val_t tools);
|
||||
el_val_t llm_vision(el_val_t model, el_val_t system, el_val_t prompt, el_val_t image_url_or_b64);
|
||||
el_val_t llm_models(void);
|
||||
|
||||
/* Register a tool handler by name. The handler is looked up via dlsym
|
||||
* (mirroring http_set_handler), so any El `fn <name>(input)` compiles to
|
||||
* a global C symbol that this function can locate at runtime.
|
||||
* Handler signature: `el_val_t handler(el_val_t input_json)` — receives
|
||||
* the tool input as a JSON-string el_val_t and returns a JSON-string
|
||||
* el_val_t result. Used by llm_call_agentic. */
|
||||
void llm_register_tool(el_val_t name, el_val_t handler_fn_name);
|
||||
|
||||
/* ── args() ─────────────────────────────────────────────────────────────────
|
||||
* Provides access to command-line arguments passed to the program.
|
||||
* Populated by el_runtime_init_args() before main() runs. */
|
||||
|
||||
el_val_t args(void);
|
||||
void el_runtime_init_args(int argc, char** argv);
|
||||
|
||||
/* ── Crypto primitives ─────────────────────────────────────────────────────
|
||||
* SHA-256, HMAC-SHA-256, and base64 (standard + URL-safe).
|
||||
* Self-contained — no OpenSSL/libcrypto dependency. The implementations are
|
||||
* adapted from public-domain reference code (Brad Conte / RFC 4648).
|
||||
*
|
||||
* Bytes-returning variants (sha256_bytes, hmac_sha256_bytes) return a string
|
||||
* value whose contents are raw binary; callers usually feed these into
|
||||
* base64_encode. Note that el_val_t strings are NUL-terminated by convention,
|
||||
* so the binary payload may contain embedded NULs — pass it directly into
|
||||
* base64_encode (which uses an explicit length) rather than treating it as
|
||||
* a printable C string.
|
||||
*
|
||||
* The "base64" variants emit/accept RFC 4648 standard alphabet with padding.
|
||||
* The "base64url" variants use URL-safe alphabet (`-`/`_`) with no padding,
|
||||
* as used in JWTs. */
|
||||
|
||||
el_val_t sha256_hex(el_val_t input);
|
||||
el_val_t sha256_bytes(el_val_t input);
|
||||
el_val_t hmac_sha256_hex(el_val_t key, el_val_t message);
|
||||
el_val_t hmac_sha256_bytes(el_val_t key, el_val_t message);
|
||||
el_val_t base64_encode(el_val_t input);
|
||||
el_val_t base64_decode(el_val_t input);
|
||||
el_val_t base64url_encode(el_val_t input);
|
||||
el_val_t base64url_decode(el_val_t input);
|
||||
|
||||
/* Length-aware variants (internal — exposed for the rare caller that already
|
||||
* has a known-length binary buffer and doesn't want to round-trip through
|
||||
* a NUL-terminated el_val_t string). Sha256_bytes and hmac_sha256_bytes feed
|
||||
* these implicitly. */
|
||||
el_val_t el_sha256_bytes_n(const unsigned char* data, size_t len);
|
||||
el_val_t el_base64_encode_n(const unsigned char* data, size_t len, int url_safe);
|
||||
|
||||
/* ── Post-quantum primitives (liboqs-backed) ────────────────────────────────
|
||||
* All inputs/outputs hex-encoded. Algorithm choices:
|
||||
* Signature: CRYSTALS-Dilithium-3 (NIST level 3, balanced)
|
||||
* KEM: CRYSTALS-Kyber-768 (NIST level 3)
|
||||
* Hash: SHA3-256 (Keccak) (PQ-aware protocols favour SHA3 over SHA2)
|
||||
*
|
||||
* If liboqs is not linked (detected via __has_include(<oqs/oqs.h>) at compile
|
||||
* time), the pq_* entry points return a JSON-shaped error string so callers
|
||||
* fail loudly rather than silently fall back to classical schemes:
|
||||
* {"error":"liboqs not linked, post-quantum primitives unavailable"}
|
||||
*
|
||||
* The hybrid handshake pairs X25519 with Kyber-768 per NIST PQ guidance and
|
||||
* CNSA 2.0. Combined shared secret is HKDF-SHA256(x25519_ss || kyber_ss).
|
||||
* Even if Kyber falls, X25519 holds; if X25519 falls under quantum attack,
|
||||
* Kyber holds. SHA3-256 also remains usable independent of liboqs (the
|
||||
* Keccak permutation is PQ-OK as a primitive). */
|
||||
|
||||
el_val_t pq_keygen_signature(void);
|
||||
el_val_t pq_sign(el_val_t secret_key_hex, el_val_t message);
|
||||
el_val_t pq_verify(el_val_t public_key_hex, el_val_t message, el_val_t signature_hex);
|
||||
|
||||
el_val_t pq_kem_keygen(void);
|
||||
el_val_t pq_kem_encaps(el_val_t public_key_hex);
|
||||
el_val_t pq_kem_decaps(el_val_t secret_key_hex, el_val_t ciphertext_hex);
|
||||
|
||||
el_val_t pq_hybrid_keygen(void);
|
||||
el_val_t pq_hybrid_handshake(el_val_t remote_pub_combined);
|
||||
|
||||
el_val_t sha3_256_hex(el_val_t input);
|
||||
|
||||
/* ── AEAD: AES-256-GCM (libcrypto-backed) ───────────────────────────────────
|
||||
* Symmetric authenticated encryption used to wrap envelopes after a KEM
|
||||
* handshake. Caller MUST supply a 32-byte key (64 hex chars) — typically the
|
||||
* Kyber-768 / hybrid shared_secret, optionally normalized via SHA3-256.
|
||||
*
|
||||
* aead_encrypt returns a JSON map {"nonce":"...","ciphertext":"..."} where
|
||||
* ciphertext is the AES-256-GCM output with the 16-byte auth tag appended.
|
||||
* Nonce is a fresh 12-byte CSPRNG draw — callers never pick the nonce, which
|
||||
* structurally rules out the GCM nonce-reuse footgun.
|
||||
*
|
||||
* aead_decrypt returns the plaintext String, or "" on any failure (including
|
||||
* auth-tag mismatch). Callers MUST check for "" before trusting the result. */
|
||||
el_val_t aead_encrypt(el_val_t key_hex, el_val_t plaintext);
|
||||
el_val_t aead_decrypt(el_val_t key_hex, el_val_t nonce_hex, el_val_t ciphertext_hex);
|
||||
|
||||
/* ── Native VM builtin aliases (for compiled El source) ─────────────────────
|
||||
* These match the El VM's native_* builtins so that El source compiled
|
||||
* to C can call the same names without modification. */
|
||||
|
||||
el_val_t native_list_get(el_val_t list, el_val_t index);
|
||||
el_val_t native_list_len(el_val_t list);
|
||||
el_val_t native_list_append(el_val_t list, el_val_t elem);
|
||||
el_val_t native_list_empty(void);
|
||||
el_val_t native_list_clone(el_val_t list);
|
||||
el_val_t native_string_chars(el_val_t s);
|
||||
el_val_t native_int_to_str(el_val_t n);
|
||||
|
||||
/* ── Method-call shorthand aliases ──────────────────────────────────────────
|
||||
* The El method-call convention `obj.method(args)` compiles to
|
||||
* `method(obj, args)`. These aliases expose the runtime functions under
|
||||
* the short names that result from method calls in El source.
|
||||
*
|
||||
* Example: `myList.append(x)` → `append(myList, x)` (calls this alias)
|
||||
* `myList.len()` → `len(myList)` (calls this alias) */
|
||||
|
||||
el_val_t append(el_val_t list, el_val_t elem); /* el_list_append */
|
||||
el_val_t len(el_val_t list); /* el_list_len */
|
||||
el_val_t get(el_val_t list, el_val_t index); /* el_list_get */
|
||||
el_val_t map_get(el_val_t map, el_val_t key); /* el_map_get */
|
||||
el_val_t map_set(el_val_t map, el_val_t key, el_val_t value); /* el_map_set */
|
||||
|
||||
/* ── OTLP/HTTP Observability ─────────────────────────────────────────────── */
|
||||
/* See bottom of el_runtime.c for the implementation.
|
||||
* Configured by env vars OTLP_ENDPOINT, OTEL_SERVICE_NAME, OTEL_SERVICE_VERSION.
|
||||
* No-op when OTLP_ENDPOINT is unset. Drop-on-failure semantics. */
|
||||
/* ── Subprocess execution ────────────────────────────────────────────────── */
|
||||
el_val_t exec_command(el_val_t cmd); /* run shell command, return exit code */
|
||||
el_val_t exec_capture(el_val_t cmd); /* run shell command, capture stdout */
|
||||
el_val_t exec(el_val_t cmd); /* exec(cmd) → stdout String (30s timeout) */
|
||||
el_val_t exec_bg(el_val_t cmd); /* exec_bg(cmd) → PID String (non-blocking) */
|
||||
|
||||
/* ── Stdout redirection (used by compiler JS pipeline) ───────────────────── */
|
||||
el_val_t stdout_to_file(el_val_t path); /* redirect process stdout to a file */
|
||||
el_val_t stdout_restore(void); /* restore process stdout to terminal */
|
||||
|
||||
el_val_t emit_log(el_val_t level, el_val_t msg, el_val_t fields_json);
|
||||
el_val_t emit_metric(el_val_t name, el_val_t value, el_val_t tags_json);
|
||||
el_val_t trace_span_start(el_val_t name);
|
||||
el_val_t trace_span_end(el_val_t span_handle);
|
||||
el_val_t emit_event(el_val_t name, el_val_t duration_ms);
|
||||
|
||||
el_val_t __thread_create(el_val_t fn_name_v, el_val_t arg_v);
|
||||
el_val_t __thread_join(el_val_t tid_v);
|
||||
|
||||
/* ── __ prefixed aliases (self-hosting compiler ABI) ─────────────────────────
|
||||
* The El self-hosting compiler emits calls to __-prefixed names. These are
|
||||
* forwarding wrappers around the existing el_runtime functions above. */
|
||||
|
||||
/* I/O */
|
||||
el_val_t __println(el_val_t s);
|
||||
el_val_t __print(el_val_t s);
|
||||
el_val_t __readline(void);
|
||||
|
||||
/* String */
|
||||
el_val_t __int_to_str(el_val_t n);
|
||||
el_val_t __str_to_int(el_val_t s);
|
||||
el_val_t __float_to_str(el_val_t f);
|
||||
el_val_t __str_to_float(el_val_t s);
|
||||
el_val_t __str_len(el_val_t s);
|
||||
el_val_t __str_char_at(el_val_t s, el_val_t i);
|
||||
el_val_t __str_cmp(el_val_t a, el_val_t b);
|
||||
el_val_t __str_ncmp(el_val_t a, el_val_t b, el_val_t n);
|
||||
el_val_t __str_concat_raw(el_val_t a, el_val_t b);
|
||||
el_val_t __str_slice_raw(el_val_t s, el_val_t start, el_val_t end);
|
||||
el_val_t __str_alloc(el_val_t n);
|
||||
el_val_t __str_set_char(el_val_t s, el_val_t i, el_val_t c);
|
||||
|
||||
/* URL encoding */
|
||||
el_val_t __url_encode(el_val_t s);
|
||||
el_val_t __url_decode(el_val_t s);
|
||||
|
||||
/* Environment */
|
||||
el_val_t __env_get(el_val_t key);
|
||||
|
||||
/* Subprocess */
|
||||
el_val_t __exec(el_val_t cmd);
|
||||
el_val_t __exec_bg(el_val_t cmd);
|
||||
|
||||
/* Process */
|
||||
el_val_t __exit_program(el_val_t code);
|
||||
|
||||
/* Filesystem */
|
||||
el_val_t __fs_exists(el_val_t path);
|
||||
el_val_t __fs_mkdir(el_val_t path);
|
||||
el_val_t __fs_read(el_val_t path);
|
||||
el_val_t __fs_write(el_val_t path, el_val_t content);
|
||||
el_val_t __fs_write_bytes(el_val_t path, el_val_t bytes, el_val_t n);
|
||||
el_val_t __fs_list_raw(el_val_t path);
|
||||
|
||||
/* HTTP server */
|
||||
el_val_t __http_response(el_val_t status, el_val_t headers_json, el_val_t body);
|
||||
el_val_t __http_serve(el_val_t port, el_val_t handler);
|
||||
el_val_t __http_serve_v2(el_val_t port, el_val_t handler);
|
||||
|
||||
/* HTTP conn fd / SSE (weak; overridden by el_seed.c when linked together) */
|
||||
el_val_t __http_conn_fd(void);
|
||||
el_val_t __http_sse_open(el_val_t conn_id);
|
||||
el_val_t __http_sse_send(el_val_t conn_id, el_val_t data);
|
||||
el_val_t __http_sse_close(el_val_t conn_id);
|
||||
|
||||
/* HTTP client (requires HAVE_CURL; stubs provided for no-curl builds) */
|
||||
el_val_t __http_do(el_val_t method, el_val_t url, el_val_t body,
|
||||
el_val_t headers_map, el_val_t timeout_ms);
|
||||
el_val_t __http_do_map(el_val_t method, el_val_t url, el_val_t body,
|
||||
el_val_t headers_json, el_val_t timeout_ms);
|
||||
el_val_t __http_do_map_to_file(el_val_t method, el_val_t url, el_val_t body,
|
||||
el_val_t headers_json, el_val_t output_path);
|
||||
|
||||
/* JSON */
|
||||
el_val_t __json_array_get(el_val_t json, el_val_t index);
|
||||
el_val_t __json_array_get_string(el_val_t json, el_val_t index);
|
||||
el_val_t __json_array_len(el_val_t json);
|
||||
el_val_t __json_get(el_val_t json, el_val_t key);
|
||||
el_val_t __json_get_raw(el_val_t json, el_val_t key);
|
||||
el_val_t __json_set(el_val_t json, el_val_t key, el_val_t value);
|
||||
el_val_t __json_parse_map(el_val_t json_str);
|
||||
el_val_t __json_stringify_val(el_val_t val);
|
||||
|
||||
/* Hashing */
|
||||
el_val_t __sha256_hex(el_val_t s);
|
||||
|
||||
/* State K/V */
|
||||
el_val_t __state_del(el_val_t key);
|
||||
el_val_t __state_get(el_val_t key);
|
||||
el_val_t __state_keys(void);
|
||||
el_val_t __state_set(el_val_t key, el_val_t val);
|
||||
|
||||
/* UUID */
|
||||
el_val_t __uuid_v4(void);
|
||||
|
||||
/* Args */
|
||||
el_val_t __args_json(void);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,761 +0,0 @@
|
||||
/*
|
||||
* el_runtime.h — El language C runtime header
|
||||
*
|
||||
* Declares all built-in functions available to compiled El programs.
|
||||
* Include this in every generated .c file.
|
||||
*
|
||||
* Value model:
|
||||
* All El values are represented as el_val_t (= int64_t).
|
||||
* On 64-bit systems a pointer fits in int64_t.
|
||||
* String values are cast: (el_val_t)(uintptr_t)"hello"
|
||||
* Integer values are stored directly.
|
||||
* This lets arithmetic work naturally while still passing strings around.
|
||||
*
|
||||
* Type conventions (El -> C):
|
||||
* String -> el_val_t (holds const char* via uintptr_t cast)
|
||||
* Int -> el_val_t
|
||||
* Bool -> el_val_t (0 = false, nonzero = true)
|
||||
* Any -> el_val_t
|
||||
* Void -> void
|
||||
*
|
||||
* Macros for convenience:
|
||||
* EL_STR(s) cast string literal to el_val_t
|
||||
* EL_CSTR(v) cast el_val_t back to const char*
|
||||
* EL_INT(v) identity — el_val_t is already int64_t
|
||||
*
|
||||
* Link requirements:
|
||||
* -lcurl — required for the HTTP client (http_get, http_post, llm_*).
|
||||
* -lpthread — required for the HTTP server (one detached thread per
|
||||
* connection, capped at 64 concurrent).
|
||||
* -loqs — optional; required only when liboqs is installed and the
|
||||
* pq_* / sha3_256_hex entry points are needed. Detected at
|
||||
* compile time via __has_include(<oqs/oqs.h>).
|
||||
* -lcrypto — optional; pulled in alongside -loqs. Used for X25519 in
|
||||
* pq_hybrid_* and HKDF-SHA256 derivation.
|
||||
*
|
||||
* Canonical compile command:
|
||||
* cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
|
||||
* -o <out> <prog>.c el-compiler/runtime/el_runtime.c
|
||||
*
|
||||
* With liboqs (post-quantum stack):
|
||||
* cc -std=c11 -I el-compiler/runtime -lcurl -lpthread -loqs -lcrypto \
|
||||
* -o <out> <prog>.c el-compiler/runtime/el_runtime.c
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
typedef int64_t el_val_t;
|
||||
|
||||
#define EL_STR(s) ((el_val_t)(uintptr_t)(s))
|
||||
#define EL_CSTR(v) ((const char*)(uintptr_t)(v))
|
||||
#define EL_INT(v) (v)
|
||||
#define EL_NULL ((el_val_t)0)
|
||||
|
||||
/* Float values share the el_val_t (int64) slot via a bit-cast.
|
||||
* The codegen emits Float literals as `el_from_float(<dbl>)` so the
|
||||
* underlying bits represent the IEEE 754 double. Float-aware builtins
|
||||
* (math, format, json) round-trip via these helpers. */
|
||||
static inline double el_to_float(el_val_t v) {
|
||||
union { int64_t i; double f; } u;
|
||||
u.i = (int64_t)v;
|
||||
return u.f;
|
||||
}
|
||||
|
||||
static inline el_val_t el_from_float(double f) {
|
||||
union { double f; int64_t i; } u;
|
||||
u.f = f;
|
||||
return (el_val_t)u.i;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* ── I/O ──────────────────────────────────────────────────────────────────── */
|
||||
|
||||
void println(el_val_t s);
|
||||
void print(el_val_t s);
|
||||
el_val_t readline(void);
|
||||
|
||||
/* ── String builtins ─────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t el_str_concat(el_val_t a, el_val_t b);
|
||||
el_val_t str_eq(el_val_t a, el_val_t b);
|
||||
el_val_t str_starts_with(el_val_t s, el_val_t prefix);
|
||||
el_val_t str_ends_with(el_val_t s, el_val_t suffix);
|
||||
el_val_t str_len(el_val_t s);
|
||||
el_val_t str_concat(el_val_t a, el_val_t b);
|
||||
el_val_t int_to_str(el_val_t n);
|
||||
el_val_t str_to_int(el_val_t s);
|
||||
el_val_t str_slice(el_val_t s, el_val_t start, el_val_t end);
|
||||
el_val_t str_contains(el_val_t s, el_val_t sub);
|
||||
el_val_t str_replace(el_val_t s, el_val_t from, el_val_t to);
|
||||
el_val_t str_to_upper(el_val_t s);
|
||||
el_val_t str_to_lower(el_val_t s);
|
||||
el_val_t str_trim(el_val_t s);
|
||||
|
||||
/* ── Math ────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t el_abs(el_val_t n);
|
||||
el_val_t el_max(el_val_t a, el_val_t b);
|
||||
el_val_t el_min(el_val_t a, el_val_t b);
|
||||
|
||||
/* ── Refcount (ARC) ──────────────────────────────────────────────────────────
|
||||
* Lists and Maps carry a refcount. Strings and ints do not — el_retain and
|
||||
* el_release are safe no-ops on non-refcounted values (they sniff a magic
|
||||
* header at offset 0 and only act if the magic matches).
|
||||
*
|
||||
* Codegen emits these at let-binding shadowing, function entry (params), and
|
||||
* function exit (locals other than the returned value). The refcount lets
|
||||
* el_list_append and el_map_set mutate in place when uniquely owned (cheap)
|
||||
* and copy-on-write when shared (preserves persistent semantics across
|
||||
* accumulator patterns in the compiler itself). */
|
||||
|
||||
void el_retain(el_val_t v);
|
||||
void el_release(el_val_t v);
|
||||
|
||||
/* ── List ────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t el_list_new(el_val_t count, ...);
|
||||
el_val_t el_list_len(el_val_t list);
|
||||
el_val_t el_list_get(el_val_t list, el_val_t index);
|
||||
el_val_t el_list_append(el_val_t list, el_val_t elem);
|
||||
el_val_t el_list_empty(void);
|
||||
el_val_t el_list_clone(el_val_t list);
|
||||
|
||||
/* ── Map ─────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t el_map_new(el_val_t pair_count, ...);
|
||||
el_val_t el_get_field(el_val_t map, el_val_t key);
|
||||
el_val_t el_map_get(el_val_t map, el_val_t key);
|
||||
el_val_t el_map_set(el_val_t map, el_val_t key, el_val_t value);
|
||||
|
||||
/* ── HTTP ─────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t http_get(el_val_t url);
|
||||
el_val_t http_post(el_val_t url, el_val_t body);
|
||||
el_val_t http_post_json(el_val_t url, el_val_t json_body);
|
||||
el_val_t http_get_with_headers(el_val_t url, el_val_t headers_map);
|
||||
el_val_t http_post_with_headers(el_val_t url, el_val_t body, el_val_t headers_map);
|
||||
el_val_t http_post_form_auth(el_val_t url, el_val_t form_body, el_val_t auth_header);
|
||||
el_val_t http_delete(el_val_t url);
|
||||
void http_serve(el_val_t port, el_val_t handler);
|
||||
void http_set_handler(el_val_t name);
|
||||
|
||||
/* HTTP server v2 ─────────────────────────────────────────────────────────────
|
||||
* Same dispatch model as http_serve, but the handler signature is widened:
|
||||
*
|
||||
* el_val_t handler(method, path, headers_map, body)
|
||||
*
|
||||
* `headers_map` is an ElMap from lowercased header name → header value (both
|
||||
* Strings). Repeated headers are joined with ", " per RFC 7230.
|
||||
*
|
||||
* Response value: the handler may return either
|
||||
* (a) a plain body string — same auto-content-type / 200-OK behaviour as
|
||||
* http_serve (3-arg) — or
|
||||
* (b) a response envelope built with `http_response(status, headers_json,
|
||||
* body)`. The runtime detects the envelope discriminator
|
||||
* `"el_http_response":1` at the start of the returned string and
|
||||
* unpacks status / headers / body before sending.
|
||||
*
|
||||
* The 3-arg http_serve(port, handler) remains supported unchanged for
|
||||
* existing handlers (e.g. products/web/server.el): it dispatches with
|
||||
* (method, path, body), hardcodes 200 OK, and auto-detects content type. */
|
||||
void http_serve_v2(el_val_t port, el_val_t handler);
|
||||
void http_set_handler_v2(el_val_t name);
|
||||
|
||||
/* Build an HTTP response envelope. `headers_json` should be a JSON object
|
||||
* literal like `{"WWW-Authenticate":"Basic"}` (or "" / "{}" for none). The
|
||||
* returned string carries the discriminator `{"el_http_response":1,...}`
|
||||
* which the runtime's send-path detects and unpacks. Detection happens
|
||||
* uniformly inside http_send_response, so a 3-arg handler may also return
|
||||
* an envelope. The 3-arg variant remains documented as a fixed 200-OK
|
||||
* auto-content-type contract for legacy handlers that return plain bodies. */
|
||||
el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body);
|
||||
|
||||
/* SSE connection fd — set by http_worker_v2 before calling the El handler,
|
||||
* cleared afterwards. Defined in el_seed.c; called from el_runtime.c.
|
||||
* The getter is exposed as __http_conn_fd() to El programs. */
|
||||
void el_seed_set_http_conn_fd(int fd);
|
||||
|
||||
/* HTTP timeout — every libcurl request honors EL_HTTP_TIMEOUT_MS (default
|
||||
* 60000ms). Read lazily on first use, so setting the env var any time before
|
||||
* the first http_* call is sufficient. */
|
||||
|
||||
/* Streaming variants — write the response body straight to a file via
|
||||
* libcurl's CURLOPT_WRITEFUNCTION = fwrite. These bypass the el_val_t string
|
||||
* wrapper entirely, so binary payloads (audio/mpeg, image/png, etc.) survive
|
||||
* embedded NUL bytes that would truncate a strlen()-based code path.
|
||||
*
|
||||
* Both honor EL_HTTP_TIMEOUT_MS, follow redirects, and accept the same
|
||||
* `headers_map` shape as http_post_with_headers (ElMap of String→String).
|
||||
*
|
||||
* Return value: 1 on success (file fully written), 0 on any failure
|
||||
* (network, file open, partial write). On failure the output file is removed
|
||||
* so callers cannot mistake a partially-written file for a valid one. */
|
||||
el_val_t http_post_to_file(el_val_t url, el_val_t body, el_val_t headers_map, el_val_t output_path);
|
||||
el_val_t http_get_to_file(el_val_t url, el_val_t headers_map, el_val_t output_path);
|
||||
|
||||
/* ── URL encoding ────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t url_encode(el_val_t s); /* RFC 3986 unreserved set */
|
||||
el_val_t url_decode(el_val_t s); /* '+' → space, %XX → byte */
|
||||
|
||||
/* ── HTML allowlist sanitizer ────────────────────────────────────────────────
|
||||
* el_html_sanitize(input_html, allowlist_json) — strict allowlist HTML
|
||||
* cleaner. State-machine parser; tag/attribute names compared case-
|
||||
* insensitively against the allowlist; `<a href>` / `<… src>` URL schemes
|
||||
* validated (http, https, mailto, fragment-only, or relative); whole-
|
||||
* subtree drop for script / style / iframe / object / embed / form; HTML-
|
||||
* escapes free text outside dropped subtrees.
|
||||
*
|
||||
* The allowlist is JSON of the form
|
||||
* {"p":[],"a":["href","title"],"strong":[],...}
|
||||
* where each value is the array of attribute names allowed for that tag. */
|
||||
el_val_t el_html_sanitize(el_val_t input_html, el_val_t allowlist_json);
|
||||
|
||||
/* ── Filesystem ──────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t fs_read(el_val_t path);
|
||||
el_val_t fs_write(el_val_t path, el_val_t content);
|
||||
el_val_t fs_list(el_val_t path);
|
||||
el_val_t fs_exists(el_val_t path);
|
||||
el_val_t fs_mkdir(el_val_t path); /* mkdir -p, mode 0755 */
|
||||
|
||||
/* Length-explicit binary write. `length` is an Int (el_val_t holding the
|
||||
* byte count). The caller knows the length from context — typically because
|
||||
* `bytes` came from base64_decode (which produces a magic-tagged binary
|
||||
* buffer with embedded NULs possible) and the caller already tracks the
|
||||
* decoded length, OR because the bytes came from a fixed-size source
|
||||
* (sha256_bytes = 32, hmac_sha256_bytes = 32). Bypasses strlen entirely.
|
||||
*
|
||||
* Returns 1 on success, 0 on failure (invalid path, can't open, partial
|
||||
* write, negative length). On partial-write failure, the file is removed
|
||||
* so callers cannot read back a truncated artefact. */
|
||||
el_val_t fs_write_bytes(el_val_t path, el_val_t bytes, el_val_t length);
|
||||
|
||||
/* ── JSON ────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t json_get(el_val_t json, el_val_t key);
|
||||
el_val_t json_parse(el_val_t s);
|
||||
el_val_t json_stringify(el_val_t v);
|
||||
el_val_t json_get_string(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_get_int(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_get_float(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_get_bool(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_get_raw(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value);
|
||||
el_val_t json_array_len(el_val_t json_str);
|
||||
el_val_t json_array_get(el_val_t json_str, el_val_t index);
|
||||
el_val_t json_array_get_string(el_val_t json_str, el_val_t index);
|
||||
|
||||
/* ── Time ────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t time_now(void);
|
||||
el_val_t time_now_utc(void);
|
||||
el_val_t sleep_secs(el_val_t secs);
|
||||
el_val_t sleep_ms(el_val_t ms);
|
||||
el_val_t time_format(el_val_t ts, el_val_t fmt);
|
||||
el_val_t time_to_parts(el_val_t ts);
|
||||
el_val_t time_from_parts(el_val_t secs, el_val_t ns, el_val_t tz);
|
||||
el_val_t time_add(el_val_t ts, el_val_t n, el_val_t unit);
|
||||
el_val_t time_diff(el_val_t ts1, el_val_t ts2, el_val_t unit);
|
||||
|
||||
/* ── Instant + Duration: first-class temporal types ──────────────────────────
|
||||
* Both types share the el_val_t (int64) slot. Instants are nanoseconds
|
||||
* since the Unix epoch; Durations are signed nanoseconds. Type discipline
|
||||
* is enforced at codegen-time: BinOps on names registered as Instant or
|
||||
* Duration route through the typed wrappers below; mismatches like
|
||||
* Instant+Instant become #error at the C compiler.
|
||||
*
|
||||
* Postfix literals — `30.seconds`, `1.hour`, `500.millis`, `30.nanos` — are
|
||||
* recognised by the parser as DurationLit AST nodes and lowered to literal
|
||||
* int64 nanoseconds at codegen time. The runtime never sees the units. */
|
||||
|
||||
el_val_t el_now_instant(void);
|
||||
el_val_t now(void);
|
||||
el_val_t unix_seconds(el_val_t n);
|
||||
el_val_t unix_millis(el_val_t n);
|
||||
el_val_t instant_from_iso8601(el_val_t s);
|
||||
|
||||
el_val_t el_duration_from_nanos(el_val_t ns);
|
||||
el_val_t duration_seconds(el_val_t n);
|
||||
el_val_t duration_millis(el_val_t n);
|
||||
el_val_t duration_nanos(el_val_t n);
|
||||
|
||||
el_val_t el_instant_add_dur(el_val_t inst, el_val_t dur);
|
||||
el_val_t el_instant_sub_dur(el_val_t inst, el_val_t dur);
|
||||
el_val_t el_instant_diff(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_add(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_sub(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_scale(el_val_t dur, el_val_t scalar);
|
||||
el_val_t el_duration_div(el_val_t dur, el_val_t scalar);
|
||||
|
||||
el_val_t el_instant_lt(el_val_t a, el_val_t b);
|
||||
el_val_t el_instant_le(el_val_t a, el_val_t b);
|
||||
el_val_t el_instant_gt(el_val_t a, el_val_t b);
|
||||
el_val_t el_instant_ge(el_val_t a, el_val_t b);
|
||||
el_val_t el_instant_eq(el_val_t a, el_val_t b);
|
||||
el_val_t el_instant_ne(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_lt(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_le(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_gt(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_ge(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_eq(el_val_t a, el_val_t b);
|
||||
el_val_t el_duration_ne(el_val_t a, el_val_t b);
|
||||
|
||||
el_val_t instant_to_unix_seconds(el_val_t i);
|
||||
el_val_t instant_to_unix_millis(el_val_t i);
|
||||
el_val_t instant_to_iso8601(el_val_t i);
|
||||
el_val_t duration_to_seconds(el_val_t d);
|
||||
el_val_t duration_to_millis(el_val_t d);
|
||||
el_val_t duration_to_nanos(el_val_t d);
|
||||
|
||||
el_val_t el_sleep_duration(el_val_t dur);
|
||||
el_val_t unix_timestamp(void);
|
||||
|
||||
el_val_t ttl_cache_set(el_val_t key, el_val_t value);
|
||||
el_val_t ttl_cache_get(el_val_t key, el_val_t max_age);
|
||||
el_val_t ttl_cache_age(el_val_t key);
|
||||
|
||||
/* ── Calendar + CalendarTime + Rhythm + LocalDate/Time/DateTime ─────────────
|
||||
* Phase 1.5 of the time system. Calendar is pluggable: EarthCalendar (IANA
|
||||
* zones, Gregorian, DST) is the user-facing default; MarsCalendar,
|
||||
* CycleCalendar(period), NoCycleCalendar, RelativeCalendar handle non-Earth
|
||||
* domains.
|
||||
*
|
||||
* A Calendar interprets an Instant under a particular cycle convention and
|
||||
* produces a CalendarTime. CalendarTime carries the underlying Instant and
|
||||
* a back-pointer to its Calendar; arithmetic and formatting consult the
|
||||
* Calendar to convert ns since epoch into year/month/day/hour/minute/second
|
||||
* (or sol/phase, or cycle/phase, depending on kind).
|
||||
*
|
||||
* Storage convention: Calendar / CalendarTime / Rhythm / LocalDate /
|
||||
* LocalDateTime are heap-allocated structs whose pointers are cast into
|
||||
* el_val_t. A 24-bit magic header at offset 0 lets the runtime identify
|
||||
* the kind safely. LocalTime is small enough to live in the int64 slot
|
||||
* directly (nanos since midnight, signed). */
|
||||
|
||||
/* Zone — opaque IANA zone or fixed offset, used by EarthCalendar.
|
||||
* `zone_id` is either an IANA name ("America/New_York", "UTC") or a fixed
|
||||
* offset string ("+05:30", "-08:00"). The runtime resolves it via tzset()
|
||||
* on first use of the owning EarthCalendar. */
|
||||
el_val_t zone(el_val_t id);
|
||||
el_val_t zone_utc(void);
|
||||
el_val_t zone_local(void);
|
||||
el_val_t zone_offset(el_val_t hours, el_val_t minutes);
|
||||
|
||||
/* Calendar constructors. Each returns an el_val_t pointer to a heap-
|
||||
* allocated, magic-tagged Calendar struct. Calendars are interned by
|
||||
* (kind, zone_id, period_ns, epoch_ns) so identical constructors return
|
||||
* the same pointer — equality is reference equality. */
|
||||
el_val_t earth_calendar(el_val_t z);
|
||||
el_val_t earth_calendar_default(void);
|
||||
el_val_t mars_calendar(void);
|
||||
el_val_t cycle_calendar(el_val_t period_dur);
|
||||
el_val_t no_cycle_calendar(void);
|
||||
el_val_t relative_calendar(el_val_t epoch_inst);
|
||||
|
||||
/* CalendarTime constructors and methods. Returns a heap-allocated struct
|
||||
* whose pointer fits in el_val_t. */
|
||||
el_val_t now_in(el_val_t cal);
|
||||
el_val_t in_calendar(el_val_t inst, el_val_t cal);
|
||||
el_val_t cal_format(el_val_t ct, el_val_t pattern);
|
||||
el_val_t cal_to_instant(el_val_t ct);
|
||||
el_val_t cal_cycle_phase(el_val_t ct);
|
||||
el_val_t cal_in(el_val_t ct, el_val_t cal);
|
||||
|
||||
/* LocalDate / LocalTime / LocalDateTime — calendar-agnostic value types.
|
||||
* LocalTime carries nanoseconds since midnight as a signed int64 directly
|
||||
* in the el_val_t slot (no allocation). LocalDate / LocalDateTime are
|
||||
* heap-allocated structs with magic headers. */
|
||||
el_val_t local_date(el_val_t y, el_val_t m, el_val_t d);
|
||||
el_val_t local_time(el_val_t h, el_val_t m, el_val_t s, el_val_t ns);
|
||||
el_val_t local_datetime(el_val_t date, el_val_t time);
|
||||
el_val_t zoned(el_val_t date, el_val_t time, el_val_t cal);
|
||||
|
||||
el_val_t local_date_year(el_val_t ld);
|
||||
el_val_t local_date_month(el_val_t ld);
|
||||
el_val_t local_date_day(el_val_t ld);
|
||||
el_val_t local_time_hour(el_val_t lt);
|
||||
el_val_t local_time_minute(el_val_t lt);
|
||||
el_val_t local_time_second(el_val_t lt);
|
||||
el_val_t local_time_nanos(el_val_t lt);
|
||||
|
||||
el_val_t el_local_date_add_dur(el_val_t ld, el_val_t dur);
|
||||
el_val_t el_local_time_add_dur(el_val_t lt, el_val_t dur);
|
||||
el_val_t el_local_date_lt(el_val_t a, el_val_t b);
|
||||
el_val_t el_local_date_eq(el_val_t a, el_val_t b);
|
||||
|
||||
/* Rhythm — pluggable recurrence AST. Returns a heap-allocated struct
|
||||
* pointer in el_val_t; rhythms are immutable so callers may share them. */
|
||||
el_val_t rhythm_cycle_start(void);
|
||||
el_val_t rhythm_cycle_phase(el_val_t phase);
|
||||
el_val_t rhythm_duration(el_val_t d);
|
||||
el_val_t rhythm_session_start(void);
|
||||
el_val_t rhythm_event(el_val_t name);
|
||||
el_val_t rhythm_and(el_val_t a, el_val_t b);
|
||||
el_val_t rhythm_or(el_val_t a, el_val_t b);
|
||||
el_val_t rhythm_weekday(el_val_t day);
|
||||
el_val_t rhythm_weekly_at(el_val_t day, el_val_t hour, el_val_t minute);
|
||||
el_val_t rhythm_next_after(el_val_t r, el_val_t after, el_val_t cal);
|
||||
el_val_t rhythm_matches(el_val_t r, el_val_t ct);
|
||||
|
||||
/* ── UUID ────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t uuid_new(void);
|
||||
el_val_t uuid_v4(void);
|
||||
|
||||
/* ── Environment ─────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t env(el_val_t key);
|
||||
|
||||
/* ── In-process state K/V ────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t state_set(el_val_t key, el_val_t value);
|
||||
el_val_t state_get(el_val_t key);
|
||||
el_val_t state_del(el_val_t key);
|
||||
el_val_t state_keys(void);
|
||||
|
||||
/* ── Float formatting ────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t float_to_str(el_val_t f);
|
||||
el_val_t int_to_float(el_val_t n);
|
||||
el_val_t float_to_int(el_val_t f);
|
||||
el_val_t format_float(el_val_t f, el_val_t decimals);
|
||||
el_val_t decimal_round(el_val_t f, el_val_t decimals);
|
||||
el_val_t str_to_float(el_val_t s);
|
||||
|
||||
/* ── Math (Float-aware) ──────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t math_sqrt(el_val_t f);
|
||||
el_val_t math_log(el_val_t f);
|
||||
el_val_t math_ln(el_val_t f);
|
||||
el_val_t math_sin(el_val_t f);
|
||||
el_val_t math_cos(el_val_t f);
|
||||
el_val_t math_pi(void);
|
||||
|
||||
/* ── String additions ────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t str_index_of(el_val_t s, el_val_t sub);
|
||||
el_val_t str_split(el_val_t s, el_val_t sep);
|
||||
el_val_t str_char_at(el_val_t s, el_val_t i);
|
||||
el_val_t str_char_code(el_val_t s, el_val_t i);
|
||||
el_val_t str_pad_left(el_val_t s, el_val_t width, el_val_t pad);
|
||||
el_val_t str_pad_right(el_val_t s, el_val_t width, el_val_t pad);
|
||||
el_val_t str_format(el_val_t fmt, el_val_t data);
|
||||
el_val_t str_lower(el_val_t s);
|
||||
el_val_t str_upper(el_val_t s);
|
||||
|
||||
/* ── Text-processing primitives (Phase 1: byte/codepoint, ASCII char classes)
|
||||
* Phase 2 (filed): Unicode-grapheme awareness, NFC/NFD normalization, regex.
|
||||
* is_* predicates: empty input returns false; multi-char requires ALL bytes
|
||||
* to match. ASCII ranges only in Phase 1. */
|
||||
|
||||
/* Counting */
|
||||
el_val_t str_count(el_val_t s, el_val_t sub); /* non-overlapping */
|
||||
el_val_t str_count_chars(el_val_t s); /* codepoint count */
|
||||
el_val_t str_count_bytes(el_val_t s); /* alias of str_len */
|
||||
el_val_t str_count_lines(el_val_t s);
|
||||
el_val_t str_count_words(el_val_t s);
|
||||
el_val_t str_count_letters(el_val_t s); /* ASCII [A-Za-z] */
|
||||
el_val_t str_count_digits(el_val_t s); /* ASCII [0-9] */
|
||||
|
||||
/* Find / position */
|
||||
el_val_t str_index_of_all(el_val_t s, el_val_t sub); /* [Int] of byte offsets */
|
||||
el_val_t str_last_index_of(el_val_t s, el_val_t sub);
|
||||
el_val_t str_find_chars(el_val_t s, el_val_t any_of); /* first idx of any ch */
|
||||
|
||||
/* Transform */
|
||||
el_val_t str_repeat(el_val_t s, el_val_t n);
|
||||
el_val_t str_reverse(el_val_t s); /* by codepoint */
|
||||
el_val_t str_strip_prefix(el_val_t s, el_val_t prefix);
|
||||
el_val_t str_strip_suffix(el_val_t s, el_val_t suffix);
|
||||
el_val_t str_strip_chars(el_val_t s, el_val_t chars);
|
||||
el_val_t str_lstrip(el_val_t s);
|
||||
el_val_t str_rstrip(el_val_t s);
|
||||
|
||||
/* Char classification (Bool) */
|
||||
el_val_t is_letter(el_val_t s);
|
||||
el_val_t is_digit(el_val_t s);
|
||||
el_val_t is_alphanumeric(el_val_t s);
|
||||
el_val_t is_whitespace(el_val_t s);
|
||||
el_val_t is_punctuation(el_val_t s);
|
||||
el_val_t is_uppercase(el_val_t s);
|
||||
el_val_t is_lowercase(el_val_t s);
|
||||
|
||||
/* Split / join */
|
||||
el_val_t str_split_lines(el_val_t s);
|
||||
el_val_t str_split_chars(el_val_t s); /* alias of native_string_chars */
|
||||
el_val_t str_split_n(el_val_t s, el_val_t sep, el_val_t n);
|
||||
el_val_t str_join(el_val_t list, el_val_t sep); /* alias of list_join */
|
||||
|
||||
/* ── List additions ──────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t list_push(el_val_t list, el_val_t elem);
|
||||
el_val_t list_push_front(el_val_t list, el_val_t elem);
|
||||
el_val_t list_join(el_val_t list, el_val_t sep);
|
||||
el_val_t list_range(el_val_t start, el_val_t end);
|
||||
|
||||
/* ── Bool helpers ────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t bool_to_str(el_val_t b);
|
||||
|
||||
/* ── Numeric parsing ─────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t parse_int(el_val_t s, el_val_t default_val);
|
||||
|
||||
/* ── Process ─────────────────────────────────────────────────────────────── */
|
||||
|
||||
void exit_program(el_val_t code);
|
||||
el_val_t getpid_now(void);
|
||||
|
||||
/* ── CGI identity ─────────────────────────────────────────────────────────────
|
||||
* Called at the start of main() in CGI programs (those with a `cgi {}` block).
|
||||
* Records the program's DHARMA identity before any other code executes. */
|
||||
|
||||
void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal,
|
||||
el_val_t network, el_val_t engram);
|
||||
|
||||
/* ── DHARMA network builtins ─────────────────────────────────────────────────
|
||||
* Available to CGI programs (declared with a `cgi {}` block).
|
||||
*
|
||||
* Peers are addressed by `dharma_id` of the form
|
||||
* "<registry-id>@<transport-url>" e.g. "ntn-genesis@http://localhost:7770"
|
||||
* If the @<url> portion is omitted, transport defaults to
|
||||
* "http://localhost:7770" (the local CGI daemon assumption).
|
||||
*
|
||||
* Wire protocol (all peers expose):
|
||||
* POST <url>/dharma/recv { channel, from, content } → response body
|
||||
* POST <url>/dharma/event { type, payload, source, timestamp }
|
||||
* POST <url>/api/activate { query } → list of nodes
|
||||
*
|
||||
* Hosting application's responsibility: an El program with a `cgi {}` block
|
||||
* runs http_serve() with its own request handler; that handler should route
|
||||
* "/dharma/event" requests by calling el_runtime_dharma_event_arrive() so
|
||||
* incoming events feed dharma_field() queues. The runtime itself does not
|
||||
* intercept any /dharma path. */
|
||||
|
||||
el_val_t dharma_connect(el_val_t cgi_id);
|
||||
el_val_t dharma_send(el_val_t channel, el_val_t content);
|
||||
el_val_t dharma_activate(el_val_t query);
|
||||
void dharma_emit(el_val_t event_type, el_val_t payload);
|
||||
el_val_t dharma_field(el_val_t event_type);
|
||||
void dharma_strengthen(el_val_t cgi_id, el_val_t weight);
|
||||
el_val_t dharma_relationship(el_val_t cgi_id);
|
||||
el_val_t dharma_peers(void);
|
||||
|
||||
/* Public C API: called by an El program's HTTP handler when a /dharma/event
|
||||
* request arrives. Pushes onto the per-event-type queue and signals any
|
||||
* pending dharma_field() blockers. All three arguments must be NUL-terminated
|
||||
* C strings (or NULL — then treated as empty). */
|
||||
void el_runtime_dharma_event_arrive(const char* event_type,
|
||||
const char* payload,
|
||||
const char* source);
|
||||
|
||||
/* ── Engram local graph primitives ───────────────────────────────────────────
|
||||
* Operate on the CGI's local Engram knowledge graph.
|
||||
* `engram_activate` queries the local graph only; `dharma_activate` is
|
||||
* network-wide across all connected CGI graphs. */
|
||||
|
||||
el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience);
|
||||
el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
|
||||
el_val_t salience, el_val_t importance, el_val_t confidence,
|
||||
el_val_t tier, el_val_t tags);
|
||||
/* Layered consciousness — see el_runtime.c for the layered architecture
|
||||
* design notes (search "Layered consciousness architecture"). The five
|
||||
* canonical layers (safety / core-identity / domain-knowledge / imprint /
|
||||
* suit) are seeded automatically; engram_add_layer extends the registry
|
||||
* with imprint or suit overlays at runtime. Nodes default to layer 1
|
||||
* (core-identity) when created via engram_node / engram_node_full. */
|
||||
el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t label,
|
||||
el_val_t salience, el_val_t certainty, el_val_t confidence,
|
||||
el_val_t status, el_val_t tags, el_val_t layer_id);
|
||||
el_val_t engram_add_layer(el_val_t name, el_val_t priority, el_val_t suppressible,
|
||||
el_val_t transparent, el_val_t injectable);
|
||||
el_val_t engram_remove_layer(el_val_t layer_id);
|
||||
el_val_t engram_list_layers(void);
|
||||
el_val_t engram_get_node(el_val_t id);
|
||||
void engram_strengthen(el_val_t node_id);
|
||||
void engram_forget(el_val_t node_id);
|
||||
el_val_t engram_node_count(void);
|
||||
el_val_t engram_search(el_val_t query, el_val_t limit);
|
||||
el_val_t engram_scan_nodes(el_val_t limit, el_val_t offset);
|
||||
void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation);
|
||||
el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id);
|
||||
el_val_t engram_neighbors(el_val_t node_id);
|
||||
el_val_t engram_neighbors_filtered(el_val_t node_id, el_val_t max_depth, el_val_t direction);
|
||||
el_val_t engram_edge_count(void);
|
||||
/* Three-pass activation: background fan-out → working-memory promotion →
|
||||
* Layer 0 override. See "Three-pass activation" in el_runtime.c. */
|
||||
el_val_t engram_activate(el_val_t query, el_val_t depth);
|
||||
el_val_t engram_save(el_val_t path);
|
||||
el_val_t engram_load(el_val_t path);
|
||||
|
||||
/* JSON-string accessors — return pre-serialized JSON so HTTP handlers
|
||||
* can pass results straight through without round-tripping ElList/ElMap
|
||||
* through json_stringify. */
|
||||
el_val_t engram_get_node_json(el_val_t id);
|
||||
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
||||
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
|
||||
el_val_t engram_activate_json(el_val_t query, el_val_t depth);
|
||||
el_val_t engram_stats_json(void);
|
||||
el_val_t engram_list_layers_json(void);
|
||||
/* engram_compile_layered_json — produce a prompt-ready text block split
|
||||
* into "[LAYER 0 — STRUCTURAL]" (non-suppressible layers, sacred fire)
|
||||
* and "[ENGRAM CONTEXT]" (standard suppressible layers). Returns "" if
|
||||
* no nodes promoted to working memory. */
|
||||
el_val_t engram_compile_layered_json(el_val_t intent, el_val_t depth);
|
||||
|
||||
/* ── LLM (Anthropic API client) ─────────────────────────────────────────────
|
||||
* All functions call https://api.anthropic.com/v1/messages with the API key
|
||||
* from env ANTHROPIC_API_KEY. Default model when empty: claude-sonnet-4-5. */
|
||||
|
||||
el_val_t llm_call(el_val_t model, el_val_t prompt);
|
||||
el_val_t llm_call_system(el_val_t model, el_val_t system_prompt, el_val_t user_prompt);
|
||||
el_val_t llm_call_agentic(el_val_t model, el_val_t system, el_val_t user, el_val_t tools);
|
||||
el_val_t llm_vision(el_val_t model, el_val_t system, el_val_t prompt, el_val_t image_url_or_b64);
|
||||
el_val_t llm_models(void);
|
||||
|
||||
/* Register a tool handler by name. The handler is looked up via dlsym
|
||||
* (mirroring http_set_handler), so any El `fn <name>(input)` compiles to
|
||||
* a global C symbol that this function can locate at runtime.
|
||||
* Handler signature: `el_val_t handler(el_val_t input_json)` — receives
|
||||
* the tool input as a JSON-string el_val_t and returns a JSON-string
|
||||
* el_val_t result. Used by llm_call_agentic. */
|
||||
void llm_register_tool(el_val_t name, el_val_t handler_fn_name);
|
||||
|
||||
/* ── args() ─────────────────────────────────────────────────────────────────
|
||||
* Provides access to command-line arguments passed to the program.
|
||||
* Populated by el_runtime_init_args() before main() runs. */
|
||||
|
||||
el_val_t args(void);
|
||||
void el_runtime_init_args(int argc, char** argv);
|
||||
|
||||
/* ── Crypto primitives ─────────────────────────────────────────────────────
|
||||
* SHA-256, HMAC-SHA-256, and base64 (standard + URL-safe).
|
||||
* Self-contained — no OpenSSL/libcrypto dependency. The implementations are
|
||||
* adapted from public-domain reference code (Brad Conte / RFC 4648).
|
||||
*
|
||||
* Bytes-returning variants (sha256_bytes, hmac_sha256_bytes) return a string
|
||||
* value whose contents are raw binary; callers usually feed these into
|
||||
* base64_encode. Note that el_val_t strings are NUL-terminated by convention,
|
||||
* so the binary payload may contain embedded NULs — pass it directly into
|
||||
* base64_encode (which uses an explicit length) rather than treating it as
|
||||
* a printable C string.
|
||||
*
|
||||
* The "base64" variants emit/accept RFC 4648 standard alphabet with padding.
|
||||
* The "base64url" variants use URL-safe alphabet (`-`/`_`) with no padding,
|
||||
* as used in JWTs. */
|
||||
|
||||
el_val_t sha256_hex(el_val_t input);
|
||||
el_val_t sha256_bytes(el_val_t input);
|
||||
el_val_t hmac_sha256_hex(el_val_t key, el_val_t message);
|
||||
el_val_t hmac_sha256_bytes(el_val_t key, el_val_t message);
|
||||
el_val_t base64_encode(el_val_t input);
|
||||
el_val_t base64_decode(el_val_t input);
|
||||
el_val_t base64url_encode(el_val_t input);
|
||||
el_val_t base64url_decode(el_val_t input);
|
||||
|
||||
/* Length-aware variants (internal — exposed for the rare caller that already
|
||||
* has a known-length binary buffer and doesn't want to round-trip through
|
||||
* a NUL-terminated el_val_t string). Sha256_bytes and hmac_sha256_bytes feed
|
||||
* these implicitly. */
|
||||
el_val_t el_sha256_bytes_n(const unsigned char* data, size_t len);
|
||||
el_val_t el_base64_encode_n(const unsigned char* data, size_t len, int url_safe);
|
||||
|
||||
/* ── Post-quantum primitives (liboqs-backed) ────────────────────────────────
|
||||
* All inputs/outputs hex-encoded. Algorithm choices:
|
||||
* Signature: CRYSTALS-Dilithium-3 (NIST level 3, balanced)
|
||||
* KEM: CRYSTALS-Kyber-768 (NIST level 3)
|
||||
* Hash: SHA3-256 (Keccak) (PQ-aware protocols favour SHA3 over SHA2)
|
||||
*
|
||||
* If liboqs is not linked (detected via __has_include(<oqs/oqs.h>) at compile
|
||||
* time), the pq_* entry points return a JSON-shaped error string so callers
|
||||
* fail loudly rather than silently fall back to classical schemes:
|
||||
* {"error":"liboqs not linked, post-quantum primitives unavailable"}
|
||||
*
|
||||
* The hybrid handshake pairs X25519 with Kyber-768 per NIST PQ guidance and
|
||||
* CNSA 2.0. Combined shared secret is HKDF-SHA256(x25519_ss || kyber_ss).
|
||||
* Even if Kyber falls, X25519 holds; if X25519 falls under quantum attack,
|
||||
* Kyber holds. SHA3-256 also remains usable independent of liboqs (the
|
||||
* Keccak permutation is PQ-OK as a primitive). */
|
||||
|
||||
el_val_t pq_keygen_signature(void);
|
||||
el_val_t pq_sign(el_val_t secret_key_hex, el_val_t message);
|
||||
el_val_t pq_verify(el_val_t public_key_hex, el_val_t message, el_val_t signature_hex);
|
||||
|
||||
el_val_t pq_kem_keygen(void);
|
||||
el_val_t pq_kem_encaps(el_val_t public_key_hex);
|
||||
el_val_t pq_kem_decaps(el_val_t secret_key_hex, el_val_t ciphertext_hex);
|
||||
|
||||
el_val_t pq_hybrid_keygen(void);
|
||||
el_val_t pq_hybrid_handshake(el_val_t remote_pub_combined);
|
||||
|
||||
el_val_t sha3_256_hex(el_val_t input);
|
||||
|
||||
/* ── AEAD: AES-256-GCM (libcrypto-backed) ───────────────────────────────────
|
||||
* Symmetric authenticated encryption used to wrap envelopes after a KEM
|
||||
* handshake. Caller MUST supply a 32-byte key (64 hex chars) — typically the
|
||||
* Kyber-768 / hybrid shared_secret, optionally normalized via SHA3-256.
|
||||
*
|
||||
* aead_encrypt returns a JSON map {"nonce":"...","ciphertext":"..."} where
|
||||
* ciphertext is the AES-256-GCM output with the 16-byte auth tag appended.
|
||||
* Nonce is a fresh 12-byte CSPRNG draw — callers never pick the nonce, which
|
||||
* structurally rules out the GCM nonce-reuse footgun.
|
||||
*
|
||||
* aead_decrypt returns the plaintext String, or "" on any failure (including
|
||||
* auth-tag mismatch). Callers MUST check for "" before trusting the result. */
|
||||
el_val_t aead_encrypt(el_val_t key_hex, el_val_t plaintext);
|
||||
el_val_t aead_decrypt(el_val_t key_hex, el_val_t nonce_hex, el_val_t ciphertext_hex);
|
||||
|
||||
/* ── Native VM builtin aliases (for compiled El source) ─────────────────────
|
||||
* These match the El VM's native_* builtins so that El source compiled
|
||||
* to C can call the same names without modification. */
|
||||
|
||||
el_val_t native_list_get(el_val_t list, el_val_t index);
|
||||
el_val_t native_list_len(el_val_t list);
|
||||
el_val_t native_list_append(el_val_t list, el_val_t elem);
|
||||
el_val_t native_list_empty(void);
|
||||
el_val_t native_list_clone(el_val_t list);
|
||||
el_val_t native_string_chars(el_val_t s);
|
||||
el_val_t native_int_to_str(el_val_t n);
|
||||
|
||||
/* ── Method-call shorthand aliases ──────────────────────────────────────────
|
||||
* The El method-call convention `obj.method(args)` compiles to
|
||||
* `method(obj, args)`. These aliases expose the runtime functions under
|
||||
* the short names that result from method calls in El source.
|
||||
*
|
||||
* Example: `myList.append(x)` → `append(myList, x)` (calls this alias)
|
||||
* `myList.len()` → `len(myList)` (calls this alias) */
|
||||
|
||||
el_val_t append(el_val_t list, el_val_t elem); /* el_list_append */
|
||||
el_val_t len(el_val_t list); /* el_list_len */
|
||||
el_val_t get(el_val_t list, el_val_t index); /* el_list_get */
|
||||
el_val_t map_get(el_val_t map, el_val_t key); /* el_map_get */
|
||||
el_val_t map_set(el_val_t map, el_val_t key, el_val_t value); /* el_map_set */
|
||||
|
||||
/* ── OTLP/HTTP Observability ─────────────────────────────────────────────── */
|
||||
/* See bottom of el_runtime.c for the implementation.
|
||||
* Configured by env vars OTLP_ENDPOINT, OTEL_SERVICE_NAME, OTEL_SERVICE_VERSION.
|
||||
* No-op when OTLP_ENDPOINT is unset. Drop-on-failure semantics. */
|
||||
/* ── Subprocess execution ────────────────────────────────────────────────── */
|
||||
el_val_t exec_command(el_val_t cmd); /* run shell command, return exit code */
|
||||
el_val_t exec_capture(el_val_t cmd); /* run shell command, capture stdout */
|
||||
el_val_t exec(el_val_t cmd); /* exec(cmd) → stdout String (30s timeout) */
|
||||
el_val_t exec_bg(el_val_t cmd); /* exec_bg(cmd) → PID String (non-blocking) */
|
||||
|
||||
el_val_t emit_log(el_val_t level, el_val_t msg, el_val_t fields_json);
|
||||
el_val_t emit_metric(el_val_t name, el_val_t value, el_val_t tags_json);
|
||||
el_val_t trace_span_start(el_val_t name);
|
||||
el_val_t trace_span_end(el_val_t span_handle);
|
||||
el_val_t emit_event(el_val_t name, el_val_t duration_ms);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@@ -1202,7 +1202,7 @@ fn codegen_js_inner(stmts: [Map<String, Any>], source: String, bundle_mode: Bool
|
||||
js_emit_line(js_strip_es_exports(runtime_content))
|
||||
js_emit_line("")
|
||||
} else {
|
||||
js_emit_line("// Runtime: foundation/el/el-compiler/runtime/el_runtime.js")
|
||||
js_emit_line("// Runtime: foundation/el/runtime/el_runtime.js")
|
||||
js_emit_line("import \"./el_runtime.js\";")
|
||||
}
|
||||
// In module mode: destructure all builtins off globalThis.__el so call
|
||||
|
||||
@@ -1292,6 +1292,43 @@ fn next_if_id() -> String {
|
||||
native_int_to_str(n)
|
||||
}
|
||||
|
||||
// is_void_builtin — true for runtime builtins declared `void` in el_runtime.h.
|
||||
// User `-> Void` functions are emitted as el_val_t (return 0) so they are safe
|
||||
// to assign; only these C-level void builtins are not.
|
||||
fn is_void_builtin(name: String) -> Bool {
|
||||
if str_eq(name, "println") { return true }
|
||||
if str_eq(name, "print") { return true }
|
||||
if str_eq(name, "engram_strengthen") { return true }
|
||||
if str_eq(name, "engram_forget") { return true }
|
||||
if str_eq(name, "engram_connect") { return true }
|
||||
if str_eq(name, "dharma_emit") { return true }
|
||||
if str_eq(name, "dharma_strengthen") { return true }
|
||||
if str_eq(name, "llm_register_tool") { return true }
|
||||
if str_eq(name, "exit_program") { return true }
|
||||
if str_eq(name, "http_serve") { return true }
|
||||
if str_eq(name, "http_set_handler") { return true }
|
||||
if str_eq(name, "http_serve_async") { return true }
|
||||
if str_eq(name, "el_cgi_init") { return true }
|
||||
if str_eq(name, "el_retain") { return true }
|
||||
if str_eq(name, "el_release") { return true }
|
||||
false
|
||||
}
|
||||
|
||||
// cg_expr_is_void — true if `val` is a direct call to a void builtin, so the
|
||||
// if-expression arm must emit it as a bare statement rather than assigning its
|
||||
// (nonexistent) value to the result var.
|
||||
fn cg_expr_is_void(val: Map<String, Any>) -> Bool {
|
||||
let vk: String = val["expr"]
|
||||
if str_eq(vk, "Call") {
|
||||
let f = val["func"]
|
||||
let fk: String = f["expr"]
|
||||
if str_eq(fk, "Ident") {
|
||||
return is_void_builtin(f["name"])
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
// Render a single arm of the if-as-expression: emit each statement-before-last
|
||||
// as a side-effecting expression, then assign the final Expr's value to the
|
||||
// result var. If the arm body is empty or its last stmt isn't an Expr, the
|
||||
@@ -1300,6 +1337,10 @@ fn cg_if_expr_arm(stmts: [Map<String, Any>], result_var: String) -> String {
|
||||
let n: Int = native_list_len(stmts)
|
||||
// Collect statement fragments into a list to avoid O(n-) string growth.
|
||||
let parts: [String] = native_list_empty()
|
||||
// Track names already declared in this arm's C block. El permits `let x`
|
||||
// to redeclare/rebind x in the same scope, but C forbids redeclaring the
|
||||
// same name in one block: emit `el_val_t x = ...` first, `x = ...` after.
|
||||
let declared: [String] = native_list_empty()
|
||||
let i = 0
|
||||
while i < n {
|
||||
let s = native_list_get(stmts, i)
|
||||
@@ -1310,18 +1351,31 @@ fn cg_if_expr_arm(stmts: [Map<String, Any>], result_var: String) -> String {
|
||||
let name: String = s["name"]
|
||||
let val = s["value"]
|
||||
let val_c: String = cg_expr(val)
|
||||
let parts = native_list_append(parts, "el_val_t " + name + " = " + val_c + "; ")
|
||||
if list_contains(declared, name) {
|
||||
let parts = native_list_append(parts, name + " = " + val_c + "; ")
|
||||
} else {
|
||||
let declared = native_list_append(declared, name)
|
||||
let parts = native_list_append(parts, "el_val_t " + name + " = " + val_c + "; ")
|
||||
}
|
||||
} else {
|
||||
if str_eq(sk, "Return") {
|
||||
let val = s["value"]
|
||||
let val_c: String = cg_expr(val)
|
||||
let parts = native_list_append(parts, result_var + " = (" + val_c + "); ")
|
||||
if cg_expr_is_void(val) {
|
||||
let parts = native_list_append(parts, val_c + "; ")
|
||||
} else {
|
||||
let parts = native_list_append(parts, result_var + " = (" + val_c + "); ")
|
||||
}
|
||||
} else {
|
||||
if str_eq(sk, "Expr") {
|
||||
let val = s["value"]
|
||||
let val_c: String = cg_expr(val)
|
||||
if is_last {
|
||||
let parts = native_list_append(parts, result_var + " = (" + val_c + "); ")
|
||||
if cg_expr_is_void(val) {
|
||||
let parts = native_list_append(parts, val_c + "; ")
|
||||
} else {
|
||||
let parts = native_list_append(parts, result_var + " = (" + val_c + "); ")
|
||||
}
|
||||
} else {
|
||||
let parts = native_list_append(parts, "(void)(" + val_c + "); ")
|
||||
}
|
||||
@@ -2579,6 +2633,12 @@ fn builtin_arity(name: String) -> Int {
|
||||
if str_eq(name, "__engram_neighbors_filtered") { return 3 }
|
||||
if str_eq(name, "__engram_activate") { return 2 }
|
||||
if str_eq(name, "__engram_activate_json") { return 2 }
|
||||
if str_eq(name, "__engram_geo_descriptor_json") { return 1 }
|
||||
if str_eq(name, "__engram_geo_overlap_json") { return 2 }
|
||||
if str_eq(name, "__engram_geo_subtract_json") { return 3 }
|
||||
if str_eq(name, "__engram_geo_combine_json") { return 2 }
|
||||
if str_eq(name, "__engram_geo_distance_json") { return 2 }
|
||||
if str_eq(name, "__engram_geo_analogy_json") { return 2 }
|
||||
if str_eq(name, "__engram_scan_nodes_json") { return 2 }
|
||||
if str_eq(name, "__generate") { return 1 }
|
||||
// Filesystem
|
||||
@@ -2669,12 +2729,21 @@ fn builtin_arity(name: String) -> Int {
|
||||
if str_eq(name, "engram_activate") { return 2 }
|
||||
if str_eq(name, "engram_save") { return 1 }
|
||||
if str_eq(name, "engram_load") { return 1 }
|
||||
if str_eq(name, "engram_store_boot") { return 1 }
|
||||
if str_eq(name, "engram_store_checkpoint") { return 0 }
|
||||
if str_eq(name, "engram_store_close") { return 0 }
|
||||
if str_eq(name, "engram_get_node_json") { return 1 }
|
||||
if str_eq(name, "engram_get_node_by_label") { return 1 }
|
||||
if str_eq(name, "engram_search_json") { return 2 }
|
||||
if str_eq(name, "engram_scan_nodes_json") { return 2 }
|
||||
if str_eq(name, "engram_neighbors_json") { return 3 }
|
||||
if str_eq(name, "engram_activate_json") { return 2 }
|
||||
if str_eq(name, "engram_geo_descriptor_json") { return 1 }
|
||||
if str_eq(name, "engram_geo_overlap_json") { return 2 }
|
||||
if str_eq(name, "engram_geo_subtract_json") { return 3 }
|
||||
if str_eq(name, "engram_geo_combine_json") { return 2 }
|
||||
if str_eq(name, "engram_geo_distance_json") { return 2 }
|
||||
if str_eq(name, "engram_geo_analogy_json") { return 2 }
|
||||
if str_eq(name, "engram_stats_json") { return 0 }
|
||||
// LLM
|
||||
if str_eq(name, "llm_call") { return 2 }
|
||||
|
||||
@@ -49,6 +49,21 @@ fn tok_value(tokens: [Any], pos: Int) -> String {
|
||||
native_list_get(tokens, pos * 2 + 1)
|
||||
}
|
||||
|
||||
// parse_progress_fatal — robustness backstop. Called by the token-consuming
|
||||
// driver loops when they detect they have iterated more times than there are
|
||||
// tokens (impossible for a well-formed program, where every iteration consumes
|
||||
// at least one token). Names the offending token and exits non-zero instead of
|
||||
// looping forever / exhausting memory.
|
||||
fn parse_progress_fatal(where: String, tokens: [Any], pos: Int) -> Void {
|
||||
let k: String = tok_kind(tokens, pos)
|
||||
let v: String = tok_value(tokens, pos)
|
||||
println("elc: FATAL: parser made no forward progress in " + where
|
||||
+ " at token index " + native_int_to_str(pos) + " (kind=" + k + ")")
|
||||
println("elc: likely a malformed construct near '" + v
|
||||
+ "' — e.g. an unterminated string or an unescaped double-quote inside a string literal (use \\\" ).")
|
||||
exit(1)
|
||||
}
|
||||
|
||||
fn expect(tokens: [Any], pos: Int, kind: String) -> Int {
|
||||
let k = tok_kind(tokens, pos)
|
||||
if k == kind {
|
||||
@@ -1212,7 +1227,16 @@ fn parse_block(tokens: [Any], pos: Int) -> Map<String, Any> {
|
||||
let p = expect(tokens, pos, "LBrace")
|
||||
let stmts: [Map<String, Any>] = native_list_empty()
|
||||
let running = true
|
||||
// Runaway backstop: a block can hold at most (token count) statements, since
|
||||
// every iteration consumes >= 1 token. If we exceed that, the cursor has run
|
||||
// off the end without terminating (malformed input) -> fail fast, don't hang.
|
||||
let blk_total: Int = native_list_len(tokens) / 2
|
||||
let blk_iters: Int = 0
|
||||
while running {
|
||||
let blk_iters = blk_iters + 1
|
||||
if blk_iters > blk_total + 8 {
|
||||
parse_progress_fatal("parse_block", tokens, p)
|
||||
}
|
||||
let k = tok_kind(tokens, p)
|
||||
if k == "RBrace" {
|
||||
let running = false
|
||||
|
||||
+3
-3
@@ -368,13 +368,13 @@ fn main() -> Void {
|
||||
let which_out: String = str_trim(exec_capture("which " + elc_bin + " 2>/dev/null"))
|
||||
if !str_eq(which_out, "") {
|
||||
let elc_dir: String = dirname_of(which_out)
|
||||
runtime_path = elc_dir + "/../el-compiler/runtime/el_runtime.c"
|
||||
runtime_path = elc_dir + "/../runtime/el_runtime.c"
|
||||
}
|
||||
}
|
||||
// If --runtime points to a directory, auto-locate el_runtime.c inside it.
|
||||
// This lets both forms work:
|
||||
// --runtime=/opt/el/el-compiler/runtime (directory form)
|
||||
// --runtime=/opt/el/el-compiler/runtime/el_runtime.c (file form)
|
||||
// --runtime=/opt/el/runtime (directory form)
|
||||
// --runtime=/opt/el/runtime/el_runtime.c (file form)
|
||||
if !str_eq(runtime_path, "") {
|
||||
let is_dir: String = str_trim(exec_capture("test -d " + runtime_path + " && echo dir || echo file"))
|
||||
if str_eq(is_dir, "dir") {
|
||||
|
||||
@@ -3797,6 +3797,9 @@ fn builtin_arity(name: String) -> Int {
|
||||
if str_eq(name, "engram_activate") { return 2 }
|
||||
if str_eq(name, "engram_save") { return 1 }
|
||||
if str_eq(name, "engram_load") { return 1 }
|
||||
if str_eq(name, "engram_store_boot") { return 1 }
|
||||
if str_eq(name, "engram_store_checkpoint") { return 0 }
|
||||
if str_eq(name, "engram_store_close") { return 0 }
|
||||
if str_eq(name, "engram_get_node_json") { return 1 }
|
||||
if str_eq(name, "engram_search_json") { return 2 }
|
||||
if str_eq(name, "engram_scan_nodes_json") { return 2 }
|
||||
|
||||
+141
-2
@@ -1423,15 +1423,53 @@ el_val_t tok_at(el_val_t tokens, el_val_t pos) {
|
||||
}
|
||||
|
||||
el_val_t tok_kind(el_val_t tokens, el_val_t pos) {
|
||||
/* Out-of-range reads MUST report the Eof sentinel so every `== "Eof"`
|
||||
termination guard in the parser fires. Without this, reading past the
|
||||
trailing Eof token returns runtime null (native_list_get OOB -> 0), which
|
||||
matches no delimiter, letting inner parse loops (parse_block, parse_binop)
|
||||
append AST nodes forever on malformed input -> unbounded allocation -> OOM. */
|
||||
el_val_t n = (native_list_len(tokens) / 2);
|
||||
if (pos < 0) {
|
||||
return EL_STR("Eof");
|
||||
}
|
||||
if (pos >= n) {
|
||||
return EL_STR("Eof");
|
||||
}
|
||||
return native_list_get(tokens, (pos * 2));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t tok_value(el_val_t tokens, el_val_t pos) {
|
||||
el_val_t n = (native_list_len(tokens) / 2);
|
||||
if (pos < 0) {
|
||||
return EL_STR("");
|
||||
}
|
||||
if (pos >= n) {
|
||||
return EL_STR("");
|
||||
}
|
||||
return native_list_get(tokens, ((pos * 2) + 1));
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* parse_progress_fatal — robustness backstop. Called by the token-consuming
|
||||
driver loops when they detect they have iterated more times than there are
|
||||
tokens (an impossibility for a well-formed program, where every iteration
|
||||
consumes at least one token). Names the offending token and exits non-zero
|
||||
instead of looping forever / exhausting memory. */
|
||||
el_val_t parse_progress_fatal(el_val_t where, el_val_t tokens, el_val_t pos) {
|
||||
el_val_t k = tok_kind(tokens, pos);
|
||||
el_val_t v = tok_value(tokens, pos);
|
||||
println(el_str_concat(el_str_concat(el_str_concat(el_str_concat(
|
||||
EL_STR("elc: FATAL: parser made no forward progress in "), where),
|
||||
EL_STR(" at token index ")), native_int_to_str(pos)),
|
||||
el_str_concat(EL_STR(" (kind="), el_str_concat(k, EL_STR(")")))));
|
||||
println(el_str_concat(el_str_concat(
|
||||
EL_STR("elc: likely a malformed construct near '"), v),
|
||||
EL_STR("' — e.g. an unterminated string or an unescaped double-quote inside a string literal (use \\\" ).")));
|
||||
exit(1);
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t expect(el_val_t tokens, el_val_t pos, el_val_t kind) {
|
||||
el_val_t k = tok_kind(tokens, pos);
|
||||
if (str_eq(k, kind)) {
|
||||
@@ -2689,7 +2727,16 @@ el_val_t parse_block(el_val_t tokens, el_val_t pos) {
|
||||
el_val_t p = expect(tokens, pos, EL_STR("LBrace"));
|
||||
el_val_t stmts = native_list_empty();
|
||||
el_val_t running = 1;
|
||||
/* Runaway backstop: a block can hold at most (token count) statements, since
|
||||
every iteration consumes >= 1 token. If we exceed that, the cursor has run
|
||||
off the end without terminating (malformed input) -> fail fast, don't hang. */
|
||||
el_val_t __blk_total = (native_list_len(tokens) / 2);
|
||||
el_val_t __blk_iters = 0;
|
||||
while (running) {
|
||||
__blk_iters = (__blk_iters + 1);
|
||||
if (__blk_iters > (__blk_total + 8)) {
|
||||
parse_progress_fatal(EL_STR("parse_block"), tokens, p);
|
||||
}
|
||||
el_val_t k = tok_kind(tokens, p);
|
||||
if (str_eq(k, EL_STR("RBrace"))) {
|
||||
running = 0;
|
||||
@@ -4838,9 +4885,51 @@ el_val_t next_if_id(void) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* is_void_builtin — true for runtime builtins declared `void` in el_runtime.h.
|
||||
User `-> Void` functions are emitted as el_val_t (return 0) so they are safe
|
||||
to assign; only these C-level void builtins are not. */
|
||||
el_val_t is_void_builtin(el_val_t name) {
|
||||
if (str_eq(name, EL_STR("println"))) { return 1; }
|
||||
if (str_eq(name, EL_STR("print"))) { return 1; }
|
||||
if (str_eq(name, EL_STR("engram_strengthen"))) { return 1; }
|
||||
if (str_eq(name, EL_STR("engram_forget"))) { return 1; }
|
||||
if (str_eq(name, EL_STR("engram_connect"))) { return 1; }
|
||||
if (str_eq(name, EL_STR("dharma_emit"))) { return 1; }
|
||||
if (str_eq(name, EL_STR("dharma_strengthen"))) { return 1; }
|
||||
if (str_eq(name, EL_STR("llm_register_tool"))) { return 1; }
|
||||
if (str_eq(name, EL_STR("exit_program"))) { return 1; }
|
||||
if (str_eq(name, EL_STR("http_serve"))) { return 1; }
|
||||
if (str_eq(name, EL_STR("http_set_handler"))) { return 1; }
|
||||
if (str_eq(name, EL_STR("http_serve_async"))) { return 1; }
|
||||
if (str_eq(name, EL_STR("el_cgi_init"))) { return 1; }
|
||||
if (str_eq(name, EL_STR("el_retain"))) { return 1; }
|
||||
if (str_eq(name, EL_STR("el_release"))) { return 1; }
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* cg_expr_is_void — true if `val` is a direct call to a void builtin, so the
|
||||
if-expression arm must emit it as a bare statement rather than assigning its
|
||||
(nonexistent) value to the result var. */
|
||||
el_val_t cg_expr_is_void(el_val_t val) {
|
||||
el_val_t vk = el_get_field(val, EL_STR("expr"));
|
||||
if (str_eq(vk, EL_STR("Call"))) {
|
||||
el_val_t f = el_get_field(val, EL_STR("func"));
|
||||
el_val_t fk = el_get_field(f, EL_STR("expr"));
|
||||
if (str_eq(fk, EL_STR("Ident"))) {
|
||||
return is_void_builtin(el_get_field(f, EL_STR("name")));
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t cg_if_expr_arm(el_val_t stmts, el_val_t result_var) {
|
||||
el_val_t n = native_list_len(stmts);
|
||||
el_val_t parts = native_list_empty();
|
||||
/* Track names already declared in this arm's C block. El permits `let x`
|
||||
to redeclare/rebind x in the same scope, but C forbids redeclaring the
|
||||
same name in one block. Emit `el_val_t x = ...` the first time and a
|
||||
plain `x = ...` reassignment thereafter (mirrors cg_stmt's `declared`). */
|
||||
el_val_t declared = native_list_empty();
|
||||
el_val_t i = 0;
|
||||
while (i < n) {
|
||||
el_val_t s = native_list_get(stmts, i);
|
||||
@@ -4853,18 +4942,31 @@ el_val_t cg_if_expr_arm(el_val_t stmts, el_val_t result_var) {
|
||||
el_val_t name = el_get_field(s, EL_STR("name"));
|
||||
el_val_t val = el_get_field(s, EL_STR("value"));
|
||||
el_val_t val_c = cg_expr(val);
|
||||
parts = native_list_append(parts, el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("el_val_t "), name), EL_STR(" = ")), val_c), EL_STR("; ")));
|
||||
if (list_contains(declared, name)) {
|
||||
parts = native_list_append(parts, el_str_concat(el_str_concat(el_str_concat(name, EL_STR(" = ")), val_c), EL_STR("; ")));
|
||||
} else {
|
||||
declared = native_list_append(declared, name);
|
||||
parts = native_list_append(parts, el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("el_val_t "), name), EL_STR(" = ")), val_c), EL_STR("; ")));
|
||||
}
|
||||
} else {
|
||||
if (str_eq(sk, EL_STR("Return"))) {
|
||||
el_val_t val = el_get_field(s, EL_STR("value"));
|
||||
el_val_t val_c = cg_expr(val);
|
||||
parts = native_list_append(parts, el_str_concat(el_str_concat(el_str_concat(result_var, EL_STR(" = (")), val_c), EL_STR("); ")));
|
||||
if (cg_expr_is_void(val)) {
|
||||
parts = native_list_append(parts, el_str_concat(val_c, EL_STR("; ")));
|
||||
} else {
|
||||
parts = native_list_append(parts, el_str_concat(el_str_concat(el_str_concat(result_var, EL_STR(" = (")), val_c), EL_STR("); ")));
|
||||
}
|
||||
} else {
|
||||
if (str_eq(sk, EL_STR("Expr"))) {
|
||||
el_val_t val = el_get_field(s, EL_STR("value"));
|
||||
el_val_t val_c = cg_expr(val);
|
||||
if (is_last) {
|
||||
if (cg_expr_is_void(val)) {
|
||||
parts = native_list_append(parts, el_str_concat(val_c, EL_STR("; ")));
|
||||
} else {
|
||||
parts = native_list_append(parts, el_str_concat(el_str_concat(el_str_concat(result_var, EL_STR(" = (")), val_c), EL_STR("); ")));
|
||||
}
|
||||
} else {
|
||||
parts = native_list_append(parts, el_str_concat(el_str_concat(EL_STR("(void)("), val_c), EL_STR("); ")));
|
||||
}
|
||||
@@ -4883,6 +4985,7 @@ el_val_t cg_if_expr_arm(el_val_t stmts, el_val_t result_var) {
|
||||
}
|
||||
el_val_t result = str_join(parts, EL_STR(""));
|
||||
el_release(parts);
|
||||
el_release(declared);
|
||||
return result;
|
||||
return 0;
|
||||
}
|
||||
@@ -6645,6 +6748,24 @@ el_val_t builtin_arity(el_val_t name) {
|
||||
if (str_eq(name, EL_STR("__engram_activate_json"))) {
|
||||
return 2;
|
||||
}
|
||||
if (str_eq(name, EL_STR("__engram_geo_descriptor_json"))) {
|
||||
return 1;
|
||||
}
|
||||
if (str_eq(name, EL_STR("__engram_geo_overlap_json"))) {
|
||||
return 2;
|
||||
}
|
||||
if (str_eq(name, EL_STR("__engram_geo_subtract_json"))) {
|
||||
return 3;
|
||||
}
|
||||
if (str_eq(name, EL_STR("__engram_geo_combine_json"))) {
|
||||
return 2;
|
||||
}
|
||||
if (str_eq(name, EL_STR("__engram_geo_distance_json"))) {
|
||||
return 2;
|
||||
}
|
||||
if (str_eq(name, EL_STR("__engram_geo_analogy_json"))) {
|
||||
return 2;
|
||||
}
|
||||
if (str_eq(name, EL_STR("__engram_scan_nodes_json"))) {
|
||||
return 2;
|
||||
}
|
||||
@@ -6888,6 +7009,24 @@ el_val_t builtin_arity(el_val_t name) {
|
||||
if (str_eq(name, EL_STR("engram_activate_json"))) {
|
||||
return 2;
|
||||
}
|
||||
if (str_eq(name, EL_STR("engram_geo_descriptor_json"))) {
|
||||
return 1;
|
||||
}
|
||||
if (str_eq(name, EL_STR("engram_geo_overlap_json"))) {
|
||||
return 2;
|
||||
}
|
||||
if (str_eq(name, EL_STR("engram_geo_subtract_json"))) {
|
||||
return 3;
|
||||
}
|
||||
if (str_eq(name, EL_STR("engram_geo_combine_json"))) {
|
||||
return 2;
|
||||
}
|
||||
if (str_eq(name, EL_STR("engram_geo_distance_json"))) {
|
||||
return 2;
|
||||
}
|
||||
if (str_eq(name, EL_STR("engram_geo_analogy_json"))) {
|
||||
return 2;
|
||||
}
|
||||
if (str_eq(name, EL_STR("engram_stats_json"))) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
//
|
||||
// Compile and run:
|
||||
// ./dist/platform/elc examples/html-page.el > /tmp/html-page.c
|
||||
// cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
|
||||
// -o /tmp/html-page /tmp/html-page.c el-compiler/runtime/el_runtime.c
|
||||
// cc -std=c11 -I runtime -lcurl -lpthread \
|
||||
// -o /tmp/html-page /tmp/html-page.c runtime/el_runtime.c
|
||||
// /tmp/html-page
|
||||
|
||||
fn render_item(item: String) -> String {
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
# El Compiler Release v1.0.0 — 2026-05-02
|
||||
|
||||
## Components
|
||||
- `bootstrap.py` — El language compiler (Python, recursive descent parser, emits C)
|
||||
- `el_runtime.c` — El runtime (C, HTTP server, engram, DHARMA, LLM chain)
|
||||
- `el_runtime.h` — Runtime public API header
|
||||
|
||||
## Changes in this release
|
||||
|
||||
### Critical bug fixes
|
||||
- `state_set`/`state_get` are now thread-safe (pthread_mutex). Was racing across 64 worker threads.
|
||||
- `looks_like_string` threshold raised from 1,000,000 to 4GB. Unix timestamps were being dereferenced as heap pointers.
|
||||
- `fs_read` guards against negative `ftell` result (pipe/special file overflow).
|
||||
|
||||
### Engram architecture (major)
|
||||
- Two-layer activation: `background_activation` (Layer 1, broad fan-out) + `working_memory_weight` (Layer 2, executive filter)
|
||||
- Inhibitory edges: `EngramEdge.inhibitory` flag suppresses working memory promotion without affecting background activation
|
||||
- Suppression memory: `suppression_count` — nodes activated-but-suppressed accumulate pressure toward breakthrough
|
||||
- Temporal decay: `temporal_decay_rate`, `created_at`, `last_activated_at`, `activation_count` on EngramNode
|
||||
- Per-type activation thresholds (Safety: 0.05, Canonical: 0.15, Lesson: 0.25, Note: 0.40)
|
||||
- Temporal range query: `engram_query_range(start_ms, end_ms)`
|
||||
- Layered consciousness: `EngramLayer` struct, `layer_id` on nodes and edges, `EngramStore.layers[]`
|
||||
- Layer 0 override pass: safety layer fires last and cannot be suppressed
|
||||
|
||||
## SHA256
|
||||
bootstrap.py
|
||||
el_runtime.c
|
||||
el_runtime.h
|
||||
@@ -132,7 +132,7 @@ if [[ $LVGL_OK -eq 1 ]]; then
|
||||
else
|
||||
_miss "LVGL/MCU" "-DEL_TARGET_LVGL (lvgl.h not found)"
|
||||
echo " Install: git clone https://github.com/lvgl/lvgl"
|
||||
echo " (place lvgl/ next to el-compiler/runtime/)"
|
||||
echo " (place lvgl/ next to runtime/)"
|
||||
MISSING=$((MISSING + 1))
|
||||
fi
|
||||
|
||||
@@ -85,6 +85,7 @@ static inline void* el_win_dlsym(void* handle, const char* name) {
|
||||
#include <direct.h> /* _mkdir */
|
||||
#define mkdir(path, mode) _mkdir(path) /* POSIX mkdir(path,mode) → _mkdir(path) */
|
||||
#define timegm _mkgmtime /* UTC tm → time_t */
|
||||
#define fsync(fd) _commit(fd) /* no fsync() on Windows; _commit() (<io.h>) is the equiv */
|
||||
|
||||
/* setenv/unsetenv: not in the Windows CRT; map to _putenv_s / SetEnvironmentVariable. */
|
||||
static inline int setenv(const char* name, const char* value, int overwrite) {
|
||||
File diff suppressed because it is too large
Load Diff
@@ -34,12 +34,12 @@
|
||||
* pq_hybrid_* and HKDF-SHA256 derivation.
|
||||
*
|
||||
* Canonical compile command:
|
||||
* cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
|
||||
* -o <out> <prog>.c el-compiler/runtime/el_runtime.c
|
||||
* cc -std=c11 -I runtime -lcurl -lpthread \
|
||||
* -o <out> <prog>.c runtime/el_runtime.c
|
||||
*
|
||||
* With liboqs (post-quantum stack):
|
||||
* cc -std=c11 -I el-compiler/runtime -lcurl -lpthread -loqs -lcrypto \
|
||||
* -o <out> <prog>.c el-compiler/runtime/el_runtime.c
|
||||
* cc -std=c11 -I runtime -lcurl -lpthread -loqs -lcrypto \
|
||||
* -o <out> <prog>.c runtime/el_runtime.c
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
@@ -117,6 +117,15 @@ el_val_t el_min(el_val_t a, el_val_t b);
|
||||
void el_retain(el_val_t v);
|
||||
void el_release(el_val_t v);
|
||||
|
||||
/* ── Arena scoping ────────────────────────────────────────────────────────────
|
||||
* el_arena_push() activates the string arena (if not already active) and
|
||||
* returns a mark; el_arena_pop(mark) frees all strings allocated since that
|
||||
* mark. Used by codegen for per-function/statement scoping and by long-running
|
||||
* EL loops (e.g. the soul daemon's awareness tick) to reclaim per-iteration
|
||||
* allocations. */
|
||||
el_val_t el_arena_push(void);
|
||||
el_val_t el_arena_pop(el_val_t mark);
|
||||
|
||||
/* ── List ────────────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t el_list_new(el_val_t count, ...);
|
||||
@@ -142,6 +151,7 @@ el_val_t http_get_with_headers(el_val_t url, el_val_t headers_map);
|
||||
el_val_t http_post_with_headers(el_val_t url, el_val_t body, el_val_t headers_map);
|
||||
el_val_t http_post_form_auth(el_val_t url, el_val_t form_body, el_val_t auth_header);
|
||||
el_val_t http_delete(el_val_t url);
|
||||
el_val_t http_delete_json(el_val_t url, el_val_t json_body);
|
||||
void http_serve(el_val_t port, el_val_t handler);
|
||||
void http_set_handler(el_val_t name);
|
||||
|
||||
@@ -167,6 +177,11 @@ void http_set_handler(el_val_t name);
|
||||
void http_serve_v2(el_val_t port, el_val_t handler);
|
||||
void http_set_handler_v2(el_val_t name);
|
||||
|
||||
/* Non-blocking variant of http_serve: runs the accept loop in a background
|
||||
* pthread and returns immediately so the caller can continue (used by the
|
||||
* soul daemon to run awareness_run() after starting its HTTP API). */
|
||||
void http_serve_async(el_val_t port, el_val_t handler);
|
||||
|
||||
/* Build an HTTP response envelope. `headers_json` should be a JSON object
|
||||
* literal like `{"WWW-Authenticate":"Basic"}` (or "" / "{}" for none). The
|
||||
* returned string carries the discriminator `{"el_http_response":1,...}`
|
||||
@@ -576,6 +591,7 @@ el_val_t engram_list_layers(void);
|
||||
el_val_t engram_get_node(el_val_t id);
|
||||
void engram_strengthen(el_val_t node_id);
|
||||
void engram_forget(el_val_t node_id);
|
||||
el_val_t engram_prune_telemetry(el_val_t older_than_ms);
|
||||
el_val_t engram_node_count(void);
|
||||
el_val_t engram_search(el_val_t query, el_val_t limit);
|
||||
el_val_t engram_scan_nodes(el_val_t limit, el_val_t offset);
|
||||
@@ -589,25 +605,77 @@ el_val_t engram_edge_count(void);
|
||||
el_val_t engram_activate(el_val_t query, el_val_t depth);
|
||||
el_val_t engram_save(el_val_t path);
|
||||
el_val_t engram_load(el_val_t path);
|
||||
/* Tiered paged-store entry points (ENGRAM_STORE=1). engram_store_boot opens the
|
||||
* durable store (import-once / WAL-replay) and loads it resident; checkpoint pushes
|
||||
* the resident graph's current field state (incl. learned hebb + activation-formed
|
||||
* edges) through the WAL and flushes; close checkpoints + closes. No-ops when off. */
|
||||
el_val_t engram_store_boot(el_val_t data_dir);
|
||||
el_val_t engram_store_checkpoint(void);
|
||||
el_val_t engram_store_close(void);
|
||||
|
||||
/* JSON-string accessors — return pre-serialized JSON so HTTP handlers
|
||||
* can pass results straight through without round-tripping ElList/ElMap
|
||||
* through json_stringify. */
|
||||
el_val_t engram_get_node_json(el_val_t id);
|
||||
el_val_t engram_get_node_by_label(el_val_t label);
|
||||
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
||||
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_scan_nodes_emb_json(el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_dreams_json(el_val_t since_ms);
|
||||
/* §5 geometry operators as EL builtins (read-only; seed-id CSV args). */
|
||||
el_val_t engram_geo_descriptor_json(el_val_t seeds);
|
||||
el_val_t engram_geo_overlap_json(el_val_t a_seeds, el_val_t b_seeds);
|
||||
el_val_t engram_geo_subtract_json(el_val_t a_seeds, el_val_t b_seeds, el_val_t mode);
|
||||
el_val_t engram_geo_combine_json(el_val_t a_seeds, el_val_t b_seeds);
|
||||
el_val_t engram_geo_distance_json(el_val_t a_seeds, el_val_t b_seeds);
|
||||
el_val_t engram_geo_analogy_json(el_val_t a_seeds, el_val_t b_seeds);
|
||||
/* reasoning layer (compositions over §5 operators). ANALOGY maps cleanly to the
|
||||
* flat-CSV seed ABI; the other modes take set/point/timestamp inputs deferred from
|
||||
* this ABI (see engram_reason.h / the reasoning-operators runbook). */
|
||||
el_val_t engram_reason_analogy_json(el_val_t a_seeds, el_val_t b_seeds, el_val_t c_seeds);
|
||||
el_val_t engram_consolidate_permanence(el_val_t node_id);
|
||||
el_val_t engram_age_field(el_val_t delta_ms);
|
||||
el_val_t engram_age_field_catchup(void);
|
||||
el_val_t engram_chrono_persist_tick(void);
|
||||
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
|
||||
el_val_t engram_activate_json(el_val_t query, el_val_t depth);
|
||||
el_val_t engram_stats_json(void);
|
||||
el_val_t engram_act_stats_json(void);
|
||||
el_val_t engram_text_health_json(void);
|
||||
el_val_t engram_cosine_sim(el_val_t id_a, el_val_t id_b);
|
||||
/* Destructively pop up to `max` newly-formed Hebbian associations as a JSON
|
||||
* array of {from_id,to_id,weight,hebb}. The learning process (soul daemon) is
|
||||
* not the process that owns persistence (engram HTTP server); this is how a
|
||||
* self-formed association crosses that boundary. (2026-08-07 self-review.) */
|
||||
el_val_t engram_hebb_drain_json(el_val_t max);
|
||||
/* Document frequency of a term across node labels — term-specificity signal
|
||||
* for curiosity seed selection. (2026-08-03 self-review.) */
|
||||
el_val_t engram_label_df(el_val_t term);
|
||||
el_val_t engram_embed_backfill(el_val_t count);
|
||||
el_val_t engram_list_layers_json(void);
|
||||
/* Working memory introspection — count, mean weight, and top-N snapshot.
|
||||
* Ported from el-compiler/runtime on 2026-06-30 self-review. */
|
||||
* Ported from runtime on 2026-06-30 self-review. */
|
||||
el_val_t engram_wm_count(void);
|
||||
el_val_t engram_wm_avg_weight(void);
|
||||
el_val_t engram_wm_top_json(el_val_t n);
|
||||
/* Merge-load: add nodes/edges from a snapshot without resetting the store. */
|
||||
el_val_t engram_load_merge(el_val_t path);
|
||||
|
||||
/* ── WAL + compaction + integrity (ENGRAM_WAL=on; design doc §§3-14,§18) ──── */
|
||||
int engram_wal_enabled(void);
|
||||
el_val_t engram_crc32(el_val_t s);
|
||||
el_val_t engram_wal_boot(el_val_t dir); /* replay + open; returns records */
|
||||
el_val_t engram_wal_open_dir(el_val_t dir);
|
||||
el_val_t engram_wal_node_put(el_val_t dir, el_val_t id);
|
||||
el_val_t engram_wal_edges_since(el_val_t dir, el_val_t start_count);
|
||||
el_val_t engram_wal_hebb_batch(el_val_t dir, el_val_t start_count);
|
||||
el_val_t engram_wal_forget(el_val_t dir, el_val_t id);
|
||||
el_val_t engram_wal_compact(el_val_t dir);
|
||||
el_val_t engram_wal_maybe_compact(el_val_t dir);
|
||||
el_val_t engram_resolve_data_dir(void); /* §18.2 fail-loud default */
|
||||
el_val_t engram_is_protected(el_val_t id); /* §18.1/18.3 derived set */
|
||||
el_val_t engram_protected_json(void);
|
||||
/* engram_compile_layered_json — produce a prompt-ready text block split
|
||||
* into "[LAYER 0 — STRUCTURAL]" (non-suppressible layers, sacred fire)
|
||||
* and "[ENGRAM CONTEXT]" (standard suppressible layers). Returns "" if
|
||||
@@ -758,18 +826,6 @@ el_val_t trace_span_start(el_val_t name);
|
||||
el_val_t trace_span_end(el_val_t span_handle);
|
||||
el_val_t emit_event(el_val_t name, el_val_t duration_ms);
|
||||
|
||||
/* ── Runtime symbols required by the soul modules ──────────────────────────── */
|
||||
/* All implemented in el_runtime.c but omitted from this release header; the soul dist modules
|
||||
* reference them directly, so the public header must export them. Declarations only — mirrors the
|
||||
* mainline el_runtime.h and is platform-independent (no behavioural change to the POSIX build). */
|
||||
typedef el_val_t (*http_handler_fn)(el_val_t method, el_val_t path, el_val_t body);
|
||||
typedef el_val_t (*http_handler4_fn)(el_val_t method, el_val_t path, el_val_t body, el_val_t headers);
|
||||
el_val_t el_arena_push(void);
|
||||
el_val_t el_arena_pop(el_val_t mark);
|
||||
void http_serve_async(el_val_t port, el_val_t handler);
|
||||
el_val_t engram_get_node_by_label(el_val_t label);
|
||||
el_val_t engram_prune_telemetry(el_val_t older_than_ms);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@@ -8,7 +8,7 @@
|
||||
* Threading: __thread_create / __thread_join use dlsym(RTLD_DEFAULT) to look
|
||||
* up El function symbols at runtime. This is the foundation of El's parallelism.
|
||||
*
|
||||
* Link: cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
|
||||
* Link: cc -std=c11 -I runtime -lcurl -lpthread \
|
||||
* -o <out> <prog>.c el_seed.c
|
||||
*/
|
||||
|
||||
@@ -1086,6 +1086,47 @@ el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el
|
||||
return engram_scan_nodes_by_type_json(node_type, limit, offset);
|
||||
}
|
||||
|
||||
el_val_t __engram_scan_nodes_emb_json(el_val_t limit, el_val_t offset) {
|
||||
return engram_scan_nodes_emb_json(limit, offset);
|
||||
}
|
||||
|
||||
el_val_t __engram_dreams_json(el_val_t since_ms) {
|
||||
return engram_dreams_json(since_ms);
|
||||
}
|
||||
|
||||
/* §5 geometry operators — native wrappers (surfacing via engram.el + elc fold is
|
||||
* the cutover step; the C table wiring is registered here now, per P0/P5). */
|
||||
el_val_t __engram_geo_descriptor_json(el_val_t seeds) {
|
||||
return engram_geo_descriptor_json(seeds);
|
||||
}
|
||||
el_val_t __engram_geo_overlap_json(el_val_t a_seeds, el_val_t b_seeds) {
|
||||
return engram_geo_overlap_json(a_seeds, b_seeds);
|
||||
}
|
||||
el_val_t __engram_geo_subtract_json(el_val_t a_seeds, el_val_t b_seeds, el_val_t mode) {
|
||||
return engram_geo_subtract_json(a_seeds, b_seeds, mode);
|
||||
}
|
||||
el_val_t __engram_geo_combine_json(el_val_t a_seeds, el_val_t b_seeds) {
|
||||
return engram_geo_combine_json(a_seeds, b_seeds);
|
||||
}
|
||||
el_val_t __engram_geo_distance_json(el_val_t a_seeds, el_val_t b_seeds) {
|
||||
return engram_geo_distance_json(a_seeds, b_seeds);
|
||||
}
|
||||
el_val_t __engram_geo_analogy_json(el_val_t a_seeds, el_val_t b_seeds) {
|
||||
return engram_geo_analogy_json(a_seeds, b_seeds);
|
||||
}
|
||||
/* reasoning layer — ANALOGY native wrapper (same C-table wiring as the §5 ops). */
|
||||
el_val_t __engram_reason_analogy_json(el_val_t a_seeds, el_val_t b_seeds, el_val_t c_seeds) {
|
||||
return engram_reason_analogy_json(a_seeds, b_seeds, c_seeds);
|
||||
}
|
||||
|
||||
el_val_t __engram_consolidate_permanence(el_val_t node_id) {
|
||||
return engram_consolidate_permanence(node_id);
|
||||
}
|
||||
|
||||
el_val_t __engram_age_field(el_val_t delta_ms) { return engram_age_field(delta_ms); }
|
||||
el_val_t __engram_age_field_catchup(void) { return engram_age_field_catchup(); }
|
||||
el_val_t __engram_chrono_persist_tick(void) { return engram_chrono_persist_tick(); }
|
||||
|
||||
el_val_t __engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction) {
|
||||
return engram_neighbors_json(node_id, max_depth, direction);
|
||||
}
|
||||
@@ -119,6 +119,37 @@ fn engram_activate_json(query: String, limit: Int) -> String {
|
||||
return __engram_activate_json(query, limit)
|
||||
}
|
||||
|
||||
// --- Geometry operators (§5) ---
|
||||
// Relational-neighborhood algebra over centered engram descriptors. Each takes
|
||||
// comma-separated seed-id set(s); a centered neighborhood is grown from those
|
||||
// seeds (ad-hoc NOCACHE path) and the operator's JSON result is returned.
|
||||
// Delegates to the native __engram_geo_*_json seeds (el_seed.c); in the heavy
|
||||
// engram runtime the same bare names resolve directly to el_runtime.c symbols.
|
||||
|
||||
fn engram_geo_descriptor_json(seeds: String) -> String {
|
||||
return __engram_geo_descriptor_json(seeds)
|
||||
}
|
||||
|
||||
fn engram_geo_overlap_json(a_seeds: String, b_seeds: String) -> String {
|
||||
return __engram_geo_overlap_json(a_seeds, b_seeds)
|
||||
}
|
||||
|
||||
fn engram_geo_subtract_json(a_seeds: String, b_seeds: String, mode: String) -> String {
|
||||
return __engram_geo_subtract_json(a_seeds, b_seeds, mode)
|
||||
}
|
||||
|
||||
fn engram_geo_combine_json(a_seeds: String, b_seeds: String) -> String {
|
||||
return __engram_geo_combine_json(a_seeds, b_seeds)
|
||||
}
|
||||
|
||||
fn engram_geo_distance_json(a_seeds: String, b_seeds: String) -> String {
|
||||
return __engram_geo_distance_json(a_seeds, b_seeds)
|
||||
}
|
||||
|
||||
fn engram_geo_analogy_json(a_seeds: String, b_seeds: String) -> String {
|
||||
return __engram_geo_analogy_json(a_seeds, b_seeds)
|
||||
}
|
||||
|
||||
// --- Generation ---
|
||||
|
||||
fn generate(form: String) -> String {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,386 @@
|
||||
/* engram_geometry.h — M9 FOUNDATION: the relational-neighborhood GEOMETRY
|
||||
* DESCRIPTOR (design doc §3, §5; memory node e94371bd).
|
||||
*
|
||||
* Computes, for a relational neighborhood grown from a seed set, the compact
|
||||
* (KB-not-MB) joint geometry Will specified: the SEMANTIC geometry (centroid,
|
||||
* covariance / principal axes, radius) braided with the RELATIONAL geometry
|
||||
* (k-core skeleton, hub->periphery centrality gradient), plus soft membership.
|
||||
*
|
||||
* Two coordinate systems, one shape — "a constellation: bright prototype at the
|
||||
* center, a cloud of members at varying distance, the strongest edges as a
|
||||
* backbone, fading at the edges."
|
||||
*
|
||||
* Built ON the two standalone M-era modules only:
|
||||
* - engram_vindex : semantic neighbors (the cloud) via ANN.
|
||||
* - engram_store : node embeddings + hebb adjacency (the skeleton), read-only.
|
||||
* It does NOT link or touch el_runtime.c, and it is a pure READ over the graph:
|
||||
* it never modifies nodes, edges, activation, the index, or any retrieval path.
|
||||
*
|
||||
* Pure C11, stdlib + libm only. The descriptor is a foundation object; it is NOT
|
||||
* wired into retrieval/priming yet (that is the next M9 step).
|
||||
*/
|
||||
#ifndef ENGRAM_GEOMETRY_H
|
||||
#define ENGRAM_GEOMETRY_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include "engram_store.h"
|
||||
#include "engram_vindex.h"
|
||||
|
||||
/* One member of the neighborhood + its place in the gradient. */
|
||||
typedef struct {
|
||||
char* id;
|
||||
double membership; /* soft membership in [0,1] (semantic+relational blend) */
|
||||
double centrality; /* skeleton weighted-degree — relational salience */
|
||||
double salience; /* the node's own stored salience */
|
||||
int core; /* k-core number (0 = fringe / not in any core) */
|
||||
double dist_centroid; /* cosine distance of member emb to centroid (semantic)*/
|
||||
int embedded; /* 1 if the member carried an emb vector */
|
||||
} GeoMember;
|
||||
|
||||
/* One skeleton edge (indices into members[]). eff_weight = weight*(1+0.5*hebb),
|
||||
* clamped to 1.0 — the effective propagation strength eg_edge_eff_weight uses. */
|
||||
typedef struct { uint32_t a, b; double eff_weight; double hebb; } GeoEdge;
|
||||
|
||||
/* A compact principal axis of the ellipsoid: unit direction in R^dim + extent
|
||||
* (sqrt of the covariance eigenvalue = the ellipsoid's half-width along it). */
|
||||
typedef struct { float* axis; double extent; } GeoAxis;
|
||||
|
||||
typedef struct {
|
||||
int dim;
|
||||
/* ── anchor ── */
|
||||
char* hub_id; /* highest-centrality member: the relational hub */
|
||||
float* centroid; /* v̄ ∈ R^dim: mean of the member embeddings in the
|
||||
* frame the descriptor operated in. When centered
|
||||
* (global_mean != NULL) this is the CENTERED
|
||||
* centroid (mean of L2-normalized embs minus the
|
||||
* global mean): the neighborhood's location in the
|
||||
* isotropic/whitened frame. Add global_mean back to
|
||||
* recover the raw prototype point. When uncentered
|
||||
* it is the raw mean of L2-normalized member embs. */
|
||||
float* global_mean; /* the centering offset actually applied (dim floats),
|
||||
* or NULL if the descriptor ran in raw space. The §5
|
||||
* operators (distance/overlap/Wasserstein) are only
|
||||
* discriminative in the centered frame — see notes. */
|
||||
/* ── shape (compact covariance): top principal axes + extents ── */
|
||||
int n_axes;
|
||||
GeoAxis* axes; /* orientation + extents of the ellipsoid */
|
||||
double total_variance; /* trace(Σ) = mean squared member dist to centroid*/
|
||||
/* ── scale ── */
|
||||
double radius; /* sqrt(total_variance) — the neighborhood breadth*/
|
||||
/* ── members + gradient ── */
|
||||
int n_members;
|
||||
GeoMember* members; /* soft membership {id->weight} + centrality/salience */
|
||||
/* ── skeleton ── */
|
||||
int n_edges;
|
||||
GeoEdge* edges; /* strong internal hebb edges = the backbone */
|
||||
int k_core; /* the maximum core number present in the skeleton*/
|
||||
/* ── diagnostics ── */
|
||||
double co_registration;/* corr(hebb strength, semantic proximity) over */
|
||||
/* internal edges: >0 = geometries agree (reify); */
|
||||
/* <0 = disagree (surprising links / dream cands). */
|
||||
int n_embedded; /* members that carried an emb vector */
|
||||
} GeoDescriptor;
|
||||
|
||||
typedef struct {
|
||||
int ann_k; /* semantic expansion: ANN neighbors per seed (0=off) */
|
||||
int hop_relational; /* 1 = include seeds' hebb neighbors as members */
|
||||
double edge_min_weight; /* skeleton: ignore internal edges below this eff wt */
|
||||
int kcore_k; /* target k for the reported k-core (0 = auto/max) */
|
||||
int top_axes; /* principal axes to retain (default 8) */
|
||||
int max_members; /* cap neighborhood size (guards the eigensolve cost) */
|
||||
} GeoParams;
|
||||
|
||||
/* Fill p with sane defaults: ann_k=24, hop_relational=1, edge_min_weight=0.05,
|
||||
* kcore_k=0 (auto), top_axes=8, max_members=400. */
|
||||
void engram_geo_default_params(GeoParams* p);
|
||||
|
||||
/* ── Global-mean cache (mean-centering / whitening the anisotropic emb space) ──
|
||||
* The nomic-embed-text space over the engram corpus is strongly ANISOTROPIC:
|
||||
* every embedding sits in a narrow cone (mean pairwise cosine ~0.55), which
|
||||
* compresses cosine-based domain separation almost to nothing. Subtracting the
|
||||
* GLOBAL MEAN of the (L2-normalized) embeddings recenters the cloud on the
|
||||
* origin (mean pairwise cosine -> ~0), restoring isotropy so the §5 operators
|
||||
* discriminate. The mean is a store-level derived quantity, like the ANN index:
|
||||
* built once from the paged store, cached, and refreshed when the embedded set
|
||||
* drifts. It lives here (not in the store) so this stays a contained, read-only
|
||||
* addition; a runtime owns one GeoMeanCache per open store alongside its VIndex. */
|
||||
typedef struct GeoMeanCache GeoMeanCache;
|
||||
|
||||
/* Scan every live node in `store` and compute the mean of the L2-normalized
|
||||
* embeddings over the embed-eligible set (nodes carrying an emb vector; the
|
||||
* unembedded telemetry/system nodes are skipped). Returns a malloc'd cache, or
|
||||
* NULL on error / no embedded nodes. The offset vector is NOT renormalized — it
|
||||
* is a translation, applied by subtraction. */
|
||||
GeoMeanCache* engram_geo_mean_build(EngramPagedStore* store);
|
||||
|
||||
/* The cached offset (dim floats) — pass to engram_geometry_descriptor as
|
||||
* global_mean. Valid until the cache is freed/refreshed. */
|
||||
const float* engram_geo_mean_vec(const GeoMeanCache* c);
|
||||
int engram_geo_mean_dim(const GeoMeanCache* c);
|
||||
uint64_t engram_geo_mean_count(const GeoMeanCache* c); /* #embedded nodes used */
|
||||
|
||||
/* Recompute the mean IN PLACE iff the embedded-node count has drifted by more
|
||||
* than `frac` (e.g. 0.10 = 10%) since the cache was built — "recompute on
|
||||
* significant change". Returns 1 if it rebuilt, 0 if unchanged, <0 on error. */
|
||||
int engram_geo_mean_maybe_refresh(GeoMeanCache* c, EngramPagedStore* store,
|
||||
double frac);
|
||||
|
||||
void engram_geo_mean_free(GeoMeanCache* c);
|
||||
|
||||
/* Compute the geometry descriptor of the neighborhood grown from seed_ids.
|
||||
* READ-ONLY over store + vindex.
|
||||
* store — an opened store (borrowed; not modified).
|
||||
* vindex — optional ANN index for semantic expansion; NULL disables it.
|
||||
* vids — the ordinal->store-id map returned by vindex_build_from_store
|
||||
* (vids[node_id] == store id). Required iff vindex != NULL.
|
||||
* n_vids — length of vids.
|
||||
* params — NULL to use engram_geo_default_params.
|
||||
* global_mean — optional centering offset (dim floats, from engram_geo_mean_*).
|
||||
* When non-NULL the SEMANTIC geometry is computed in mean-centered
|
||||
* (isotropic) space: every normalized member emb has global_mean
|
||||
* subtracted before the centroid / cosine-distance / co-registration
|
||||
* math, so those operators discriminate. NULL = raw space (legacy).
|
||||
* NOTE: the ANN neighbor query still runs in RAW unit-vector space —
|
||||
* centering is a rigid translation that ~preserves neighborhood
|
||||
* MEMBERSHIP, so the index needs no rebuild; only the descriptor
|
||||
* STATISTICS move to the centered frame (co-registration choice (b)).
|
||||
* The eigen/covariance shape (axes, radius) is translation-invariant
|
||||
* and therefore identical in either frame.
|
||||
* Returns a malloc'd descriptor (free with engram_geo_free), or NULL on error
|
||||
* (no seeds resolvable, OOM). */
|
||||
GeoDescriptor* engram_geometry_descriptor(
|
||||
EngramPagedStore* store, VIndex* vindex,
|
||||
char** vids, int n_vids,
|
||||
const char* const* seed_ids, size_t n_seeds,
|
||||
const GeoParams* params,
|
||||
const float* global_mean);
|
||||
|
||||
void engram_geo_free(GeoDescriptor* g);
|
||||
|
||||
/* ── M-INTEROCEPTION P3: drift-sensor primitive (descriptor displacement) ────
|
||||
* Read-only. GROWTH vs CORRUPTION split of how far B drifted from baseline A.
|
||||
* See engram_geometry.c for the honesty note on the missing SelfAnchor. */
|
||||
typedef struct {
|
||||
double centroid_sep; /* L2 distance between centroids (same frame) */
|
||||
double centroid_cos; /* 1 - cosine(centroidA, centroidB) */
|
||||
double radius_delta; /* |radiusA - radiusB| — neighborhood scale change */
|
||||
double core_disp; /* mean radial displacement of the invariant core */
|
||||
double periph_disp; /* mean radial displacement of the periphery */
|
||||
int core_matched; /* # core members matched by id across A,B */
|
||||
int periph_matched; /* # periphery members matched by id across A,B */
|
||||
} GeoDisplacement;
|
||||
|
||||
void engram_geo_displacement(const GeoDescriptor* a, const GeoDescriptor* b,
|
||||
double core_frac, GeoDisplacement* out);
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
* §5 GEOMETRY OPERATORS — a relational ALGEBRA over neighborhood descriptors.
|
||||
* These are the reusable primitives Will specified: "primitives any CGI
|
||||
* application should be able to use." READ-ONLY and PURE (stdlib + libm only) —
|
||||
* they consume GeoDescriptor(s) and never touch the store, index, or activation.
|
||||
*
|
||||
* FRAME CONTRACT: both inputs MUST have been built in the SAME frame — identical
|
||||
* emb `dim` and identical `global_mean` (centered against the one true store-wide
|
||||
* mean). The reify path builds every neighborhood that way, so descriptors are
|
||||
* directly comparable. An operator returns <0 / NULL if the dims disagree.
|
||||
*
|
||||
* REPRESENTATION: the C descriptor lives in the FULL emb dim with a LOW-RANK
|
||||
* covariance Σ = Σ_k extent_k² · a_k a_kᵀ over its retained principal axes
|
||||
* (top_axes; the discarded tail variance is not modeled). Every operator mirrors
|
||||
* the viz-proxy (engram-geometry-proxy.py §5) FORMULA exactly, but evaluates it on
|
||||
* this representation — so semantics match the proxy while absolute numbers differ
|
||||
* (proxy works in a 24-dim global-PCA reduced dense frame; C in full-dim low-rank).
|
||||
* The Wasserstein / combine eigen-work is done inside the small JOINT axis subspace
|
||||
* (dimension ≤ nA+nB+1), which is EXACT for the low-rank covariances there.
|
||||
* Each result struct is released by its engram_geo_*_free.
|
||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
/* overlap(A,B): shared-member set + Jaccard + centroid/scale proximity score. */
|
||||
typedef struct {
|
||||
char** shared_ids; /* ids present in BOTH neighborhoods (owned) */
|
||||
int n_shared;
|
||||
int n_union; /* |A ∪ B| by id */
|
||||
double jaccard; /* |A∩B| / |A∪B| */
|
||||
double centroid_distance; /* L2 between the (centered) centroids */
|
||||
double overlap_score; /* jacc*0.5 + max(0,1−d/(rA+rB))*0.5 (proxy form)*/
|
||||
float* intersection_centroid; /* midpoint of the two centroids (dim, owned) */
|
||||
int dim;
|
||||
} GeoOverlap;
|
||||
int engram_geo_overlap(const GeoDescriptor* a, const GeoDescriptor* b, GeoOverlap* out);
|
||||
void engram_geo_overlap_free(GeoOverlap* o);
|
||||
|
||||
/* subtract(A,B) — ORTHOGONAL-COMPLEMENT residual: project A onto I − V_B V_Bᵀ
|
||||
* (V_B = B's top `b_dims` principal axes) — "A with B's framing removed". Returns
|
||||
* A's residual centroid + residual ellipsoid, the fraction of A's energy that lives
|
||||
* inside B's subspace, and the centroid-difference vector. b_dims<=0 → min(3,nB). */
|
||||
typedef struct {
|
||||
int dim;
|
||||
float* residual_centroid; /* P⊥ c_A (owned) */
|
||||
float* centroid_diff; /* c_A − c_B (owned) */
|
||||
double centroid_diff_mag;
|
||||
double variance_explained_by_B; /* (‖Qc_A‖²+Tr(QΣ_A)) / (‖c_A‖²+Tr Σ_A) ∈[0,1]*/
|
||||
int removed_dims; /* # of B axes used as V_B */
|
||||
double residual_scale; /* sqrt(Tr(P⊥ Σ_A P⊥)) */
|
||||
int n_axes; /* residual principal axes (owned) */
|
||||
GeoAxis* axes;
|
||||
} GeoResidual;
|
||||
int engram_geo_subtract(const GeoDescriptor* a, const GeoDescriptor* b,
|
||||
int b_dims, GeoResidual* out);
|
||||
void engram_geo_residual_free(GeoResidual* r);
|
||||
|
||||
/* set-diff variant of subtract: members in A but not in B + the centroid arrow. */
|
||||
typedef struct {
|
||||
char** only_ids; /* member ids in A and not in B (owned) */
|
||||
int n_only;
|
||||
int removed; /* |A ∩ B| (dropped) */
|
||||
float* centroid_diff; /* c_A − c_B (dim, owned) */
|
||||
double centroid_diff_mag;
|
||||
int dim;
|
||||
} GeoSetDiff;
|
||||
int engram_geo_setdiff(const GeoDescriptor* a, const GeoDescriptor* b, GeoSetDiff* out);
|
||||
void engram_geo_setdiff_free(GeoSetDiff* s);
|
||||
|
||||
/* combine(A,B): a merged descriptor — POOLED centroid + POOLED covariance
|
||||
* (exact law-of-total-variance: the covariance you'd get by concatenating the two
|
||||
* member clouds), re-eigendecomposed for its principal axes. Members = id-union
|
||||
* (membership = max). top_axes<=0 → 8. Returns a malloc'd GeoDescriptor (free with
|
||||
* engram_geo_free) in the same frame as A, or NULL on error. */
|
||||
GeoDescriptor* engram_geo_combine(const GeoDescriptor* a, const GeoDescriptor* b,
|
||||
int top_axes);
|
||||
|
||||
/* distance(A,B): centroid L2 + centroid cosine + closed-form Wasserstein-2
|
||||
* (Bures metric) between the two Gaussians — mirrors the proxy's _wasserstein2. */
|
||||
typedef struct {
|
||||
double centroid_distance;
|
||||
double centroid_cosine;
|
||||
double wasserstein2;
|
||||
int dim;
|
||||
} GeoDistance;
|
||||
int engram_geo_distance(const GeoDescriptor* a, const GeoDescriptor* b, GeoDistance* out);
|
||||
|
||||
/* analogy(A,B): orthogonal PROCRUSTES transform min_R ‖A − B R‖_F, RᵀR=I (SVD)
|
||||
* aligning A's principal frame to B's (extent-scaled axes, paired by rank). R is
|
||||
* returned COMPACTLY as an r×r rotation within the joint axis subspace `basis`
|
||||
* (r vectors of dim floats); it acts as the identity on the orthogonal complement.
|
||||
* Apply it to a vector with engram_geo_analogy_apply. */
|
||||
typedef struct {
|
||||
int dim;
|
||||
int r; /* subspace rank; R is r×r */
|
||||
float* basis; /* r×dim row-major orthonormal basis Q (owned) */
|
||||
double* R; /* r×r rotation in Q-coords, row-major (owned) */
|
||||
double residual; /* ‖A − B R‖_F over the extent-scaled frames */
|
||||
} GeoAnalogy;
|
||||
int engram_geo_analogy(const GeoDescriptor* a, const GeoDescriptor* b, GeoAnalogy* out);
|
||||
/* out_vec = R·v for v ∈ R^dim: v + Σ_i (R̂c − c)_i q_i, c_i = q_i·v. dim floats. */
|
||||
void engram_geo_analogy_apply(const GeoAnalogy* an, const float* v, float* out_vec);
|
||||
void engram_geo_analogy_free(GeoAnalogy* an);
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
* M10 — REIFICATION: densely co-wired relational neighborhoods crystallized into
|
||||
* FIRST-CLASS, PERSISTED store records (design doc §2; memory 885f5945). This is
|
||||
* NOT a cache — it is durable structure. A reified neighborhood is a real store
|
||||
* NODE (node_type "Neighborhood") that survives restart, is loaded on boot, and
|
||||
* EVOLVES via supersede+provenance when the pattern shifts. The geometry-priming
|
||||
* HOT PATH reads these persisted records (never computes geometry on the
|
||||
* activation path). Ad-hoc/transient geometries still use the on-the-fly
|
||||
* engram_geometry_descriptor above.
|
||||
*
|
||||
* Two record types, both ordinary TLV store nodes (no new on-disk format):
|
||||
* - "GeoMeanFrame" : the store-wide centering mean, persisted ONCE (emb = mean
|
||||
* vector, id ENGRAM_GEO_MEANFRAME_ID). Referenced by every
|
||||
* neighborhood so priming centers against the SAME true mean.
|
||||
* - "Neighborhood" : one reified neighborhood. emb = the RAW centroid (prototype
|
||||
* point, so it stays centroid-ANN-able; centered_centroid =
|
||||
* emb - meanframe). metadata = the compact "GEO1" schema:
|
||||
* hub id, meanframe ref, scalar shape (radius, total_variance,
|
||||
* k_core, co_registration, n_embedded), axis EXTENTS (ellipsoid
|
||||
* half-widths), and the MEMBER list {id -> membership, centrality,
|
||||
* core}. Member links are also persisted as edges relation="member".
|
||||
*
|
||||
* v1 honest simplifications (documented; extensible without migration): axis
|
||||
* DIRECTION vectors are not persisted (extents capture the ellipsoid scale; the
|
||||
* directions are recomputable via the on-the-fly descriptor for viz/operators);
|
||||
* with hebb potentiation ~0 on today's store the "hebb-weighted" degree reduces to
|
||||
* AUTHORED edge weight, so detected neighborhoods currently reflect authored edges —
|
||||
* the design is unchanged and self-correcting once hebb accrues.
|
||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
#define ENGRAM_GEO_NBHD_TYPE "Neighborhood"
|
||||
#define ENGRAM_GEO_MEANFRAME_TYPE "GeoMeanFrame"
|
||||
#define ENGRAM_GEO_MEANFRAME_ID "geo-meanframe" /* stable id of the singleton */
|
||||
#define ENGRAM_GEO_NBHD_ID_PREFIX "nbhd-" /* id = nbhd-<hub>-<built_at> */
|
||||
#define ENGRAM_GEO_MEMBER_RELATION "member"
|
||||
|
||||
typedef struct {
|
||||
int min_weighted_degree; /* hub qualifies iff strong-edge weighted degree >= this
|
||||
* (0 = no floor: just rank + take top max_neighborhoods) */
|
||||
int max_neighborhoods; /* homeostatic budget cap (default 128) */
|
||||
double cover_membership; /* skip a hub already a member (w>=this) of an accepted
|
||||
* neighborhood — greedy non-redundant cover (default 0.5) */
|
||||
int persist_member_edges; /* 1 = also write relation="member" edges (default 1) */
|
||||
GeoParams descriptor; /* per-neighborhood params (top_axes may be 0 = skip eigensolve) */
|
||||
} GeoReifyParams;
|
||||
|
||||
/* Defaults: min_weighted_degree=0, max_neighborhoods=128, cover_membership=0.5,
|
||||
* persist_member_edges=1, descriptor = engram_geo_default_params but top_axes=4,
|
||||
* max_members=256 (reified neighborhoods stay compact). */
|
||||
void engram_geo_reify_default_params(GeoReifyParams* p);
|
||||
|
||||
/* WRITE PATH (offline / consolidation — NEVER the activation hot path).
|
||||
* Detect dense hub neighborhoods on the hebb-weighted graph, compute each one's
|
||||
* CENTERED descriptor ONCE against the true store-wide mean, and PERSIST them as
|
||||
* first-class records: the GeoMeanFrame (once) + one Neighborhood node per detected
|
||||
* neighborhood (+ member edges), superseding any prior same-hub record with
|
||||
* provenance. Read-then-write over `store`. Returns #neighborhoods persisted, or <0.
|
||||
* Skips existing Neighborhood/GeoMeanFrame nodes when detecting (idempotent re-reify). */
|
||||
int engram_geo_reify_store(EngramPagedStore* store, VIndex* vindex,
|
||||
char** vids, int n_vids,
|
||||
const GeoReifyParams* params);
|
||||
|
||||
/* ── Resident loaded form of the persisted records (boot-time; READ-ONLY) ─────
|
||||
* The durable Neighborhood/GeoMeanFrame records are the source of truth; this
|
||||
* index is their LOADED form (like the resident node array is the loaded form of
|
||||
* the node records, or adjacency the loaded form of edges). It never recomputes
|
||||
* geometry — it parses. Build it by feeding the runtime's boot node scan, or in
|
||||
* one pass with engram_geo_reify_load. */
|
||||
typedef struct GeoReifyIndex GeoReifyIndex;
|
||||
|
||||
GeoReifyIndex* engram_geo_reify_index_new(void);
|
||||
/* Feed one store node; if it is a Neighborhood or GeoMeanFrame record it is parsed
|
||||
* and absorbed (else ignored). The node is BORROWED (copied as needed). 0/<0. */
|
||||
int engram_geo_reify_index_add(GeoReifyIndex* ix, const StoreNode* n);
|
||||
/* Build the member->neighborhood hash after all adds. Call once. 0/<0. */
|
||||
int engram_geo_reify_index_finalize(GeoReifyIndex* ix);
|
||||
/* One-pass convenience: scan the store and build the finalized index. NULL if the
|
||||
* store holds no reified records. */
|
||||
GeoReifyIndex* engram_geo_reify_load(EngramPagedStore* store);
|
||||
|
||||
/* A borrowed view of one persisted neighborhood (owned by the index). */
|
||||
typedef struct {
|
||||
const char* id;
|
||||
const char* hub_id;
|
||||
int n_members;
|
||||
char* const* member_ids; /* parallel arrays, length n_members */
|
||||
const double* member_w; /* membership in [0,1] */
|
||||
double radius;
|
||||
double co_registration;
|
||||
int k_core;
|
||||
int n_embedded;
|
||||
} GeoNeighborhood;
|
||||
|
||||
/* HOT-PATH LOOKUP (no geometry compute): resolve the seed set to the best
|
||||
* persisted neighborhood — the one with the greatest summed seed membership; on a
|
||||
* miss (no seed is a member of any neighborhood) fall back to the centroid nearest
|
||||
* the query embedding (centered by the loaded mean frame). q_emb may be NULL (then
|
||||
* a miss returns NULL). Returns a BORROWED handle (do NOT free) or NULL. */
|
||||
const GeoNeighborhood* engram_geo_reify_lookup(
|
||||
const GeoReifyIndex* ix,
|
||||
const char* const* seed_ids, size_t n_seeds,
|
||||
const float* q_emb, int q_dim);
|
||||
|
||||
int engram_geo_reify_count(const GeoReifyIndex* ix);
|
||||
const float* engram_geo_reify_mean(const GeoReifyIndex* ix, int* dim); /* loaded true mean or NULL */
|
||||
void engram_geo_reify_index_free(GeoReifyIndex* ix);
|
||||
|
||||
#endif /* ENGRAM_GEOMETRY_H */
|
||||
@@ -0,0 +1,287 @@
|
||||
/* engram_reason.c — the REASONING layer. Pure compositions over engram_geometry.h.
|
||||
* stdlib + libm only; READ-ONLY over its descriptor inputs; touches no store/index. */
|
||||
#include "engram_reason.h"
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
|
||||
/* ── small float-vector helpers ─────────────────────────────────────────────── */
|
||||
static double vdot(const float* a, const float* b, int dim) {
|
||||
double s = 0; for (int i = 0; i < dim; i++) s += (double)a[i] * (double)b[i]; return s;
|
||||
}
|
||||
static double vnorm(const float* a, int dim) { return sqrt(vdot(a, a, dim)); }
|
||||
static double vcos(const float* a, const float* b, int dim) {
|
||||
double na = vnorm(a, dim), nb = vnorm(b, dim);
|
||||
if (na < 1e-12 || nb < 1e-12) return 0.0; /* a null vector ⇒ no direction */
|
||||
double c = vdot(a, b, dim) / (na * nb);
|
||||
if (c > 1.0) c = 1.0; if (c < -1.0) c = -1.0;
|
||||
return c;
|
||||
}
|
||||
static double l2(const float* a, const float* b, int dim) {
|
||||
double s = 0; for (int i = 0; i < dim; i++) { double d = (double)a[i] - (double)b[i]; s += d * d; }
|
||||
return sqrt(s);
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════ SHARED — point-to-manifold FIT ══ */
|
||||
int engram_reason_point_fit(const GeoDescriptor* g, const float* x,
|
||||
double ext_floor, GeoFit* out) {
|
||||
if (!g || !x || !out || g->dim <= 0 || !g->centroid) return -1;
|
||||
if (!(ext_floor > 0)) ext_floor = 1.0;
|
||||
int dim = g->dim;
|
||||
/* residual r = x − centroid */
|
||||
double rr = 0; /* ‖r‖² */
|
||||
float* r = malloc((size_t)dim * sizeof(float));
|
||||
if (!r) return -1;
|
||||
for (int i = 0; i < dim; i++) { double d = (double)x[i] - (double)g->centroid[i]; r[i] = (float)d; rr += d * d; }
|
||||
double maha2 = 0, ss_in = 0; /* Mahalanobis² and in-subspace energy */
|
||||
for (int k = 0; k < g->n_axes; k++) {
|
||||
const float* ax = g->axes[k].axis; if (!ax) continue;
|
||||
double proj = vdot(r, ax, dim); /* axes are orthonormal directions */
|
||||
double den = g->axes[k].extent; if (den < ext_floor) den = ext_floor;
|
||||
maha2 += (proj / den) * (proj / den);
|
||||
ss_in += proj * proj;
|
||||
}
|
||||
double ortho2 = rr - ss_in; if (ortho2 < 0) ortho2 = 0; /* off-subspace energy */
|
||||
double dist2 = maha2 + ortho2 / (ext_floor * ext_floor);
|
||||
out->mahalanobis = sqrt(maha2);
|
||||
out->ortho_residual = sqrt(ortho2);
|
||||
out->distance = sqrt(dist2);
|
||||
out->score = 1.0 / (1.0 + dist2);
|
||||
free(r);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════ ANALOGY ════ */
|
||||
int engram_reason_analogy(const GeoDescriptor* A, const GeoDescriptor* B,
|
||||
const GeoDescriptor* C,
|
||||
const GeoDescriptor* const* candidates, int n_candidates,
|
||||
GeoAnalogyResult* out) {
|
||||
if (!A || !B || !C || !out) return -1;
|
||||
if (!A->centroid || !B->centroid || !C->centroid) return -1;
|
||||
int dim = A->dim;
|
||||
if (B->dim != dim || C->dim != dim) return -1;
|
||||
memset(out, 0, sizeof *out);
|
||||
out->dim = dim; out->best = -1;
|
||||
|
||||
/* Learn R_{A→B}. engram_geo_analogy(X,Y) yields R with apply(R, Y-axis) ≈ X-axis
|
||||
* (R maps Y's frame → X's frame); so R that maps A→B is engram_geo_analogy(B,A). */
|
||||
GeoAnalogy an;
|
||||
if (engram_geo_analogy(B, A, &an) != 0) return -1;
|
||||
out->analogy_residual = an.residual;
|
||||
|
||||
/* mapped = R·c_C + (c_B − R·c_A) : the A→B affine (rotation + residual shift). */
|
||||
float* RcA = malloc((size_t)dim * sizeof(float));
|
||||
float* RcC = malloc((size_t)dim * sizeof(float));
|
||||
out->mapped_point = malloc((size_t)dim * sizeof(float));
|
||||
if (!RcA || !RcC || !out->mapped_point) { free(RcA); free(RcC); free(out->mapped_point); out->mapped_point = NULL; engram_geo_analogy_free(&an); return -1; }
|
||||
engram_geo_analogy_apply(&an, A->centroid, RcA);
|
||||
engram_geo_analogy_apply(&an, C->centroid, RcC);
|
||||
for (int i = 0; i < dim; i++)
|
||||
out->mapped_point[i] = (float)((double)RcC[i] + ((double)B->centroid[i] - (double)RcA[i]));
|
||||
free(RcA); free(RcC);
|
||||
engram_geo_analogy_free(&an);
|
||||
|
||||
/* nearest candidate to the mapped point (centroid L2). */
|
||||
if (candidates && n_candidates > 0) {
|
||||
out->n_candidates = n_candidates;
|
||||
out->distances = malloc((size_t)n_candidates * sizeof(double));
|
||||
if (!out->distances) return -1;
|
||||
double best = -1; int bi = -1;
|
||||
for (int i = 0; i < n_candidates; i++) {
|
||||
const GeoDescriptor* cd = candidates[i];
|
||||
double d = (cd && cd->centroid && cd->dim == dim) ? l2(out->mapped_point, cd->centroid, dim) : INFINITY;
|
||||
out->distances[i] = d;
|
||||
if (bi < 0 || d < best) { best = d; bi = i; }
|
||||
}
|
||||
out->best = bi; out->best_distance = best;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
void engram_reason_analogy_free(GeoAnalogyResult* r) {
|
||||
if (!r) return;
|
||||
free(r->mapped_point); free(r->distances);
|
||||
r->mapped_point = NULL; r->distances = NULL;
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════ INDUCTION ══ */
|
||||
int engram_reason_induce(const GeoDescriptor* const* examples, int n_examples,
|
||||
int top_axes, double ext_floor, GeoInduction* out) {
|
||||
if (!examples || n_examples < 1 || !out) return -1;
|
||||
if (top_axes <= 0) top_axes = 8;
|
||||
memset(out, 0, sizeof *out);
|
||||
|
||||
/* fold the examples left→right through the pooled-Gaussian combine. n==1 pools
|
||||
* the single example with itself (identical cov ⇒ same shape, id-union = itself). */
|
||||
GeoDescriptor* acc = engram_geo_combine(examples[0],
|
||||
examples[n_examples > 1 ? 1 : 0], top_axes);
|
||||
if (!acc) return -1;
|
||||
for (int i = 2; i < n_examples; i++) {
|
||||
GeoDescriptor* nxt = engram_geo_combine(acc, examples[i], top_axes);
|
||||
engram_geo_free(acc);
|
||||
if (!nxt) return -1;
|
||||
acc = nxt;
|
||||
}
|
||||
out->rule = acc;
|
||||
out->n_examples = n_examples;
|
||||
out->ext_floor = (ext_floor > 0) ? ext_floor
|
||||
: (acc->radius > 0 ? acc->radius * 0.25 : 1.0);
|
||||
return 0;
|
||||
}
|
||||
double engram_reason_membership(const GeoInduction* ind, const float* x) {
|
||||
if (!ind || !ind->rule || !x) return -1;
|
||||
GeoFit f;
|
||||
if (engram_reason_point_fit(ind->rule, x, ind->ext_floor, &f) != 0) return -1;
|
||||
return f.score;
|
||||
}
|
||||
void engram_reason_induction_free(GeoInduction* out) {
|
||||
if (!out) return;
|
||||
if (out->rule) engram_geo_free(out->rule);
|
||||
out->rule = NULL;
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════ ABDUCTION ══ */
|
||||
int engram_reason_abduce(const float* obs, int dim,
|
||||
const GeoDescriptor* const* hypotheses, int n,
|
||||
double ext_floor, GeoAbduction* out) {
|
||||
if (!obs || !hypotheses || n < 1 || dim <= 0 || !out) return -1;
|
||||
if (!(ext_floor > 0)) ext_floor = 1.0;
|
||||
memset(out, 0, sizeof *out);
|
||||
out->n = n; out->best = -1;
|
||||
out->scores = malloc((size_t)n * sizeof(double));
|
||||
out->distances = malloc((size_t)n * sizeof(double));
|
||||
out->rank = malloc((size_t)n * sizeof(int));
|
||||
if (!out->scores || !out->distances || !out->rank) { engram_reason_abduction_free(out); return -1; }
|
||||
|
||||
double best = -1; int bi = -1;
|
||||
for (int i = 0; i < n; i++) {
|
||||
out->rank[i] = i;
|
||||
const GeoDescriptor* h = hypotheses[i];
|
||||
GeoFit f;
|
||||
if (!h || h->dim != dim || engram_reason_point_fit(h, obs, ext_floor, &f) != 0) {
|
||||
out->scores[i] = 0.0; out->distances[i] = INFINITY;
|
||||
} else {
|
||||
out->scores[i] = f.score; out->distances[i] = f.distance;
|
||||
}
|
||||
if (bi < 0 || out->scores[i] > best) { best = out->scores[i]; bi = i; }
|
||||
}
|
||||
out->best = bi; out->best_score = (bi >= 0) ? out->scores[bi] : 0.0;
|
||||
|
||||
/* rank indices best→worst by score (insertion sort — n is small). */
|
||||
for (int i = 1; i < n; i++) {
|
||||
int key = out->rank[i]; int j = i - 1;
|
||||
while (j >= 0 && out->scores[out->rank[j]] < out->scores[key]) { out->rank[j + 1] = out->rank[j]; j--; }
|
||||
out->rank[j + 1] = key;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
void engram_reason_abduction_free(GeoAbduction* out) {
|
||||
if (!out) return;
|
||||
free(out->scores); free(out->distances); free(out->rank);
|
||||
out->scores = NULL; out->distances = NULL; out->rank = NULL;
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════ CAUSAL ══ */
|
||||
/* |cos| of two descriptors' centroids after removing confounder Z's subspace. */
|
||||
static double controlled_assoc(const GeoDescriptor* x, const GeoDescriptor* y,
|
||||
const GeoDescriptor* z) {
|
||||
GeoResidual rx, ry; double c = 0;
|
||||
int ox = engram_geo_subtract(x, z, 0, &rx);
|
||||
int oy = engram_geo_subtract(y, z, 0, &ry);
|
||||
if (ox == 0 && oy == 0 && rx.residual_centroid && ry.residual_centroid)
|
||||
c = fabs(vcos(rx.residual_centroid, ry.residual_centroid, x->dim));
|
||||
if (ox == 0) engram_geo_residual_free(&rx);
|
||||
if (oy == 0) engram_geo_residual_free(&ry);
|
||||
return c;
|
||||
}
|
||||
int engram_reason_causal(const GeoDescriptor* x, const GeoDescriptor* y,
|
||||
const GeoDescriptor* const* confounders, int n_conf,
|
||||
int64_t t_x, int64_t t_y,
|
||||
double drop_frac, GeoCausal* out) {
|
||||
if (!x || !y || !out || !x->centroid || !y->centroid || x->dim != y->dim) return -1;
|
||||
if (!(drop_frac > 0 && drop_frac < 1)) drop_frac = 0.5;
|
||||
memset(out, 0, sizeof *out);
|
||||
const double assoc_floor = 0.2; /* below this = no meaningful association */
|
||||
|
||||
out->assoc_raw = fabs(vcos(x->centroid, y->centroid, x->dim));
|
||||
/* control for each confounder; the strongest single explainer wins (min assoc). */
|
||||
double ctrl = out->assoc_raw;
|
||||
for (int i = 0; i < n_conf; i++) {
|
||||
if (!confounders[i]) continue;
|
||||
double c = controlled_assoc(x, y, confounders[i]);
|
||||
if (c < ctrl) ctrl = c;
|
||||
}
|
||||
out->assoc_controlled = ctrl;
|
||||
out->temporal_dir = (t_x < t_y) ? 1 : (t_x > t_y) ? -1 : 0;
|
||||
|
||||
if (out->assoc_raw < assoc_floor) {
|
||||
out->verdict = GEO_CAUSAL_NONE;
|
||||
} else if (ctrl < (1.0 - drop_frac) * out->assoc_raw && ctrl < assoc_floor) {
|
||||
out->verdict = GEO_CAUSAL_CONFOUNDED; out->confounded = 1;
|
||||
} else if (out->temporal_dir != 0) {
|
||||
out->verdict = GEO_CAUSAL_DIRECTED; out->strength = ctrl;
|
||||
} else {
|
||||
out->verdict = GEO_CAUSAL_NONE; /* associated + robust but unorientable */
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════ PLANNING ══ */
|
||||
int engram_reason_plan(const GeoDescriptor* const* nodes, int n,
|
||||
int start, int goal, double neighbor_radius,
|
||||
int use_wasserstein, GeoPlan* out) {
|
||||
if (!nodes || n < 1 || !out) return -1;
|
||||
if (start < 0 || start >= n || goal < 0 || goal >= n) return -1;
|
||||
if (!(neighbor_radius > 0)) return -1;
|
||||
memset(out, 0, sizeof *out);
|
||||
|
||||
/* dense edge weights (i<j symmetric); INFINITY = not adjacent. */
|
||||
double* W = malloc((size_t)n * (size_t)n * sizeof(double));
|
||||
if (!W) return -1;
|
||||
for (int i = 0; i < n; i++) for (int j = 0; j < n; j++) W[(size_t)i * n + j] = (i == j) ? 0.0 : INFINITY;
|
||||
for (int i = 0; i < n; i++) {
|
||||
for (int j = i + 1; j < n; j++) {
|
||||
GeoDistance d;
|
||||
if (nodes[i] && nodes[j] && engram_geo_distance(nodes[i], nodes[j], &d) == 0) {
|
||||
double w = use_wasserstein ? d.wasserstein2 : d.centroid_distance;
|
||||
if (w <= neighbor_radius) { W[(size_t)i * n + j] = w; W[(size_t)j * n + i] = w; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* O(n²) Dijkstra. */
|
||||
double* dist = malloc((size_t)n * sizeof(double));
|
||||
int* prev = malloc((size_t)n * sizeof(int));
|
||||
char* done = calloc((size_t)n, 1);
|
||||
if (!dist || !prev || !done) { free(W); free(dist); free(prev); free(done); return -1; }
|
||||
for (int i = 0; i < n; i++) { dist[i] = INFINITY; prev[i] = -1; }
|
||||
dist[start] = 0;
|
||||
for (int it = 0; it < n; it++) {
|
||||
int u = -1; double bd = INFINITY;
|
||||
for (int i = 0; i < n; i++) if (!done[i] && dist[i] < bd) { bd = dist[i]; u = i; }
|
||||
if (u < 0) break;
|
||||
done[u] = 1;
|
||||
if (u == goal) break;
|
||||
for (int v = 0; v < n; v++) {
|
||||
double w = W[(size_t)u * n + v];
|
||||
if (w < INFINITY && !done[v] && dist[u] + w < dist[v]) { dist[v] = dist[u] + w; prev[v] = u; }
|
||||
}
|
||||
}
|
||||
|
||||
if (dist[goal] < INFINITY) {
|
||||
int len = 0; for (int v = goal; v != -1; v = prev[v]) len++;
|
||||
out->path = malloc((size_t)len * sizeof(int));
|
||||
if (out->path) {
|
||||
out->path_len = len;
|
||||
int idx = len - 1;
|
||||
for (int v = goal; v != -1; v = prev[v]) out->path[idx--] = v;
|
||||
out->total_cost = dist[goal];
|
||||
out->reached = 1;
|
||||
}
|
||||
}
|
||||
free(W); free(dist); free(prev); free(done);
|
||||
return 0;
|
||||
}
|
||||
void engram_reason_plan_free(GeoPlan* out) {
|
||||
if (!out) return;
|
||||
free(out->path); out->path = NULL;
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
/* engram_reason.h — the REASONING layer: compositions over the §5 geometry
|
||||
* OPERATORS (engram_geometry.h). Where the operators are a relational ALGEBRA over
|
||||
* neighborhood descriptors, these are reasoning MODES built by CHAINING that algebra:
|
||||
*
|
||||
* ANALOGY A:B :: C:? — learn the A→B transform (Procrustes), apply to C.
|
||||
* INDUCTION {E_i} → rule — pool example geometries; a generalizing structure
|
||||
* + a membership test.
|
||||
* ABDUCTION x → best H — the structure whose geometry best PLACES an
|
||||
* observation in-distribution (inverse of prediction).
|
||||
* CAUSAL x ? y | Z, t — separate mere overlap (correlation) from directed
|
||||
* influence (temporal precedence + association that
|
||||
* SURVIVES controlling for confounders via subtract).
|
||||
* PLANNING start → goal — a trajectory (sequence of neighborhoods) through the
|
||||
* manifold: shortest path over geo-distance edges.
|
||||
*
|
||||
* PURE + READ-ONLY (stdlib + libm only): every function consumes GeoDescriptor(s)
|
||||
* (+ a few scalars / timestamps) and NEVER touches the store, index, or activation.
|
||||
* All geometry is delegated to the engram_geo_* primitives; this file only composes.
|
||||
*
|
||||
* FRAME CONTRACT (inherited): descriptors passed together MUST share emb `dim` and
|
||||
* `global_mean` frame — exactly the §5 operator contract. A function returns <0 on
|
||||
* a dim/frame mismatch or bad argument.
|
||||
*/
|
||||
#ifndef ENGRAM_REASON_H
|
||||
#define ENGRAM_REASON_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include "engram_geometry.h"
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
* SHARED PRIMITIVE — point-to-manifold FIT. How well does a single point x sit
|
||||
* inside a neighborhood's ellipsoid? Splits the residual (x − centroid) into:
|
||||
* - the IN-SUBSPACE part, scaled by each axis extent → a Mahalanobis distance
|
||||
* (how many "radii" out along the modeled directions), and
|
||||
* - the ORTHOGONAL part outside the retained axes → energy the model does not
|
||||
* explain at all (charged at the extent floor).
|
||||
* This is the common engine under INDUCTION's membership test and ABDUCTION's
|
||||
* explanation ranking. ext_floor (>0) guards zero-extent axes / the null model.
|
||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||
typedef struct {
|
||||
double mahalanobis; /* sqrt( Σ_k ((a_k·(x−c)) / max(ext_k,floor))² ) */
|
||||
double ortho_residual; /* ‖(x−c) projected off the retained axes‖ (raw L2) */
|
||||
double distance; /* sqrt( maha² + (ortho_residual/floor)² ) — full fit */
|
||||
double score; /* 1 / (1 + distance²) ∈ (0,1] (1 = dead-center) */
|
||||
} GeoFit;
|
||||
int engram_reason_point_fit(const GeoDescriptor* g, const float* x,
|
||||
double ext_floor, GeoFit* out);
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
* ANALOGY — "A:B :: C:?". Learn the transform that carries A to B (orthogonal
|
||||
* Procrustes rotation R between their principal frames + the residual translation),
|
||||
* apply it to C, and return the mapped point + the nearest candidate neighborhood.
|
||||
* Composes: engram_geo_analogy (R) + engram_geo_analogy_apply + engram_geo_distance.
|
||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||
typedef struct {
|
||||
int dim;
|
||||
float* mapped_point; /* predicted D location = R·c_C + (c_B − R·c_A) (owned)*/
|
||||
double analogy_residual;/* Procrustes ‖A−B R‖_F — frame-alignment quality */
|
||||
int best; /* index of nearest candidate to mapped_point, or −1 */
|
||||
double best_distance; /* centroid L2 from mapped_point to the winner */
|
||||
int n_candidates;
|
||||
double* distances; /* centroid L2 mapped_point→candidate[i] (owned)*/
|
||||
} GeoAnalogyResult;
|
||||
/* candidates may be NULL/0 (then best=−1 and only mapped_point is filled). */
|
||||
int engram_reason_analogy(const GeoDescriptor* A, const GeoDescriptor* B,
|
||||
const GeoDescriptor* C,
|
||||
const GeoDescriptor* const* candidates, int n_candidates,
|
||||
GeoAnalogyResult* out);
|
||||
void engram_reason_analogy_free(GeoAnalogyResult* r);
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
* INDUCTION — from a SET of example neighborhoods to the generalizing structure.
|
||||
* Pools the examples (law-of-total-variance via engram_geo_combine, folded left to
|
||||
* right) into a single "rule" descriptor whose top principal axes are the directions
|
||||
* CONSISTENTLY present across the examples (the shared subspace surfaces as the
|
||||
* dominant pooled axes; idiosyncratic per-example directions fall to the tail).
|
||||
* The rule carries a membership test (point-to-manifold fit against the pool).
|
||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||
typedef struct {
|
||||
GeoDescriptor* rule; /* induced generalizing geometry (owned; geo_free) */
|
||||
double ext_floor; /* extent floor used by the membership test */
|
||||
int n_examples;/* how many examples were pooled */
|
||||
} GeoInduction;
|
||||
/* top_axes<=0 → 8. ext_floor<=0 → derived from the pooled radius. */
|
||||
int engram_reason_induce(const GeoDescriptor* const* examples, int n_examples,
|
||||
int top_axes, double ext_floor, GeoInduction* out);
|
||||
/* Membership of a point in the induced rule ∈ (0,1] (the fit score). <0 on error. */
|
||||
double engram_reason_membership(const GeoInduction* ind, const float* x);
|
||||
void engram_reason_induction_free(GeoInduction* out);
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
* ABDUCTION — inference to the best explanation. Given an observation POINT, rank a
|
||||
* set of candidate structures by how well each PLACES the observation in-distribution
|
||||
* (min point-to-manifold distance = the structure that, if assumed, best accounts for
|
||||
* the observation). The inverse of prediction.
|
||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||
typedef struct {
|
||||
int best; /* index of best-explaining hypothesis, or −1 */
|
||||
double best_score;
|
||||
int n;
|
||||
double* scores; /* fit score per hypothesis (higher = better) (owned)*/
|
||||
double* distances; /* explanation distance per hypothesis (owned)*/
|
||||
int* rank; /* hypothesis indices sorted best→worst (owned)*/
|
||||
} GeoAbduction;
|
||||
int engram_reason_abduce(const float* obs, int dim,
|
||||
const GeoDescriptor* const* hypotheses, int n,
|
||||
double ext_floor, GeoAbduction* out);
|
||||
void engram_reason_abduction_free(GeoAbduction* out);
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
* CAUSAL — correlation vs causation. Over two variables' geometries (+ candidate
|
||||
* confounders + temporal order), distinguish:
|
||||
* - mere co-occurrence / overlap (correlation), from
|
||||
* - directed influence: association that (a) SURVIVES controlling for confounders
|
||||
* (subtract each Z's subspace from both centroids, re-measure) and (b) is oriented
|
||||
* by temporal PRECEDENCE.
|
||||
* Composes: centroid cosine (correlation) + engram_geo_subtract (control) + timestamps.
|
||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||
typedef enum {
|
||||
GEO_CAUSAL_NONE = 0, /* no meaningful association */
|
||||
GEO_CAUSAL_DIRECTED = 1, /* survives control + temporally ordered → cause→eff */
|
||||
GEO_CAUSAL_CONFOUNDED = 2 /* correlated but association dies under control */
|
||||
} GeoCausalVerdict;
|
||||
typedef struct {
|
||||
double assoc_raw; /* |cos(c_x,c_y)| — the raw correlation */
|
||||
double assoc_controlled; /* |cos| of residual centroids after control */
|
||||
int temporal_dir; /* +1 x→y, −1 y→x, 0 tie/unknown */
|
||||
GeoCausalVerdict verdict;
|
||||
int confounded; /* 1 iff verdict==CONFOUNDED (the flag) */
|
||||
double strength; /* directed influence estimate ∈[0,1] (0 else)*/
|
||||
} GeoCausal;
|
||||
/* confounders may be NULL/0. t_x,t_y are comparable timestamps (any monotone unit);
|
||||
* pass equal values for "unknown order". drop_frac∈(0,1): a controlled association
|
||||
* below (1−drop_frac)·assoc_raw AND below an absolute floor ⇒ CONFOUNDED. */
|
||||
int engram_reason_causal(const GeoDescriptor* x, const GeoDescriptor* y,
|
||||
const GeoDescriptor* const* confounders, int n_conf,
|
||||
int64_t t_x, int64_t t_y,
|
||||
double drop_frac, GeoCausal* out);
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
* PLANNING — trajectory construction. Given a set of neighborhoods (manifold nodes),
|
||||
* a start and a goal, build a PATH (sequence of intermediate neighborhoods) by
|
||||
* shortest path over the graph whose edges connect neighborhoods within
|
||||
* neighbor_radius, weighted by geo-distance. Long straight jumps are not edges, so
|
||||
* the path follows the manifold's curvature through intermediates (a discrete geodesic).
|
||||
* Composes: engram_geo_distance (edge weights) + Dijkstra.
|
||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||
typedef struct {
|
||||
int* path; /* node indices start..goal (owned) */
|
||||
int path_len;
|
||||
double total_cost; /* summed centroid-distance edge weights along path */
|
||||
int reached; /* 1 if goal reachable within neighbor_radius graph */
|
||||
} GeoPlan;
|
||||
/* neighbor_radius>0: max centroid distance for two neighborhoods to be adjacent.
|
||||
* Use "wasserstein"!=0 to weight edges by Wasserstein-2 instead of centroid L2. */
|
||||
int engram_reason_plan(const GeoDescriptor* const* nodes, int n,
|
||||
int start, int goal, double neighbor_radius,
|
||||
int use_wasserstein, GeoPlan* out);
|
||||
void engram_reason_plan_free(GeoPlan* out);
|
||||
|
||||
#endif /* ENGRAM_REASON_H */
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,295 @@
|
||||
/* engram_store.h — M1 of the engram tiered storage engine.
|
||||
*
|
||||
* The FINAL on-disk paged store format: superblock (+ mirror), slotted pages,
|
||||
* self-describing TLV records, overflow chains, and two B+-tree indexes
|
||||
* (primary id->loc, adjacency from_id/to_id->edge-locs) over a free-listed
|
||||
* page file. See docs/architecture/design/engram-tiered-storage-engine.md §2.
|
||||
*
|
||||
* This is a self-contained module (plain C, standard libs only). It defines its
|
||||
* own serializable views of a node/edge (StoreNode/StoreEdge) that mirror every
|
||||
* persisted field of EngramNode/EngramEdge in el_runtime.c. M3 maps between the
|
||||
* live runtime structs and these; M1 does not touch el_runtime.c.
|
||||
*
|
||||
* Format id: magic "ENGST01", format_version 1. This format is PERMANENT — the
|
||||
* TLV record scheme means new fields never force a migration.
|
||||
*/
|
||||
#ifndef ENGRAM_STORE_H
|
||||
#define ENGRAM_STORE_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
/* Fixed for the life of a store; recorded in the superblock. */
|
||||
#define STORE_PAGE_SIZE 16384u
|
||||
#define STORE_MAGIC "ENGST01" /* 7 chars + NUL stored in an 8-byte field */
|
||||
#define STORE_FORMAT_VERSION 1u
|
||||
|
||||
/* Ring-buffer length for ACT-R base-level access timestamps.
|
||||
* MUST equal ENGRAM_BLL_K in el_runtime.c (currently 10). Static-checked in .c. */
|
||||
#define STORE_BLL_K 10
|
||||
|
||||
/* Page types (page header byte). */
|
||||
enum {
|
||||
STORE_PT_NODE = 1,
|
||||
STORE_PT_EDGE = 2,
|
||||
STORE_PT_INDEX = 3,
|
||||
STORE_PT_OVERFLOW = 4,
|
||||
STORE_PT_FREE = 5
|
||||
};
|
||||
|
||||
/* store_check flags. */
|
||||
#define STORE_CHECK_CRC 1u
|
||||
|
||||
/* ── Serializable node view: every persisted EngramNode field ─────────────── */
|
||||
typedef struct StoreNode {
|
||||
char* id;
|
||||
char* content;
|
||||
char* node_type;
|
||||
char* label;
|
||||
char* tier;
|
||||
char* tags;
|
||||
char* metadata;
|
||||
double salience;
|
||||
double importance;
|
||||
double confidence;
|
||||
double temporal_decay_rate;
|
||||
int64_t activation_count;
|
||||
int64_t last_activated;
|
||||
int64_t created_at;
|
||||
int64_t updated_at;
|
||||
double background_activation;
|
||||
double working_memory_weight;
|
||||
int32_t suppression_count;
|
||||
uint32_t layer_id;
|
||||
int64_t access_ts[STORE_BLL_K];
|
||||
int32_t access_head;
|
||||
int32_t access_filled;
|
||||
double wm_anchor;
|
||||
float* emb; /* owned; NULL if not embedded */
|
||||
int32_t emb_dim;
|
||||
/* Forward-compat: raw bytes of any TLV fields the reader did not recognise,
|
||||
* concatenated verbatim ([tag][u32 len][bytes]...). Re-emitted on write so
|
||||
* an old reader never drops a newer writer's fields. */
|
||||
uint8_t* unknown;
|
||||
size_t unknown_len;
|
||||
int tombstoned; /* set by store_get_* if the located record is dead */
|
||||
/* hebb_elig / hebb_elig_ts are DELIBERATELY NOT persisted (see EngramNode). */
|
||||
} StoreNode;
|
||||
|
||||
/* ── Serializable edge view: every persisted EngramEdge field ─────────────── */
|
||||
typedef struct StoreEdge {
|
||||
char* id;
|
||||
char* from_id;
|
||||
char* to_id;
|
||||
char* relation;
|
||||
char* metadata;
|
||||
double weight;
|
||||
double hebb;
|
||||
double confidence;
|
||||
int64_t created_at;
|
||||
int64_t updated_at;
|
||||
int64_t last_fired;
|
||||
int32_t inhibitory;
|
||||
uint32_t layer_id;
|
||||
uint8_t* unknown;
|
||||
size_t unknown_len;
|
||||
int tombstoned;
|
||||
} StoreEdge;
|
||||
|
||||
typedef struct EngramPagedStore EngramPagedStore;
|
||||
|
||||
/* Lifecycle. */
|
||||
EngramPagedStore* store_create(const char* path); /* fails if file exists */
|
||||
EngramPagedStore* store_open(const char* path); /* recovers via mirror SB */
|
||||
int store_close(EngramPagedStore* s); /* syncs + frees */
|
||||
int store_sync(EngramPagedStore* s); /* fsync + rewrite both superblocks */
|
||||
|
||||
/* Nodes. store_get_node returns 1 on hit (fills *out, caller store_node_free),
|
||||
* 0 if absent or tombstoned, <0 on error. */
|
||||
int store_put_node(EngramPagedStore* s, const StoreNode* n);
|
||||
int store_get_node(EngramPagedStore* s, const char* id, StoreNode* out);
|
||||
int store_tombstone(EngramPagedStore* s, const char* id);
|
||||
|
||||
/* Edges. *out is malloc'd (store_edges_free); *n set to count. */
|
||||
int store_put_edge(EngramPagedStore* s, const StoreEdge* e);
|
||||
int store_get_edges_from(EngramPagedStore* s, const char* from_id, StoreEdge** out, size_t* n);
|
||||
int store_get_edges_to(EngramPagedStore* s, const char* to_id, StoreEdge** out, size_t* n);
|
||||
|
||||
/* Integrity: verify every page's crc (and both superblocks). Returns the number
|
||||
* of corrupt pages (0 = clean), or <0 on I/O error. */
|
||||
int store_check(EngramPagedStore* s, unsigned flags);
|
||||
|
||||
/* Ownership helpers. */
|
||||
void store_node_free(StoreNode* n);
|
||||
void store_edge_free(StoreEdge* e);
|
||||
void store_edges_free(StoreEdge* arr, size_t n);
|
||||
|
||||
/* Test-only hook (NOT a format property — B+-tree nodes are self-describing via
|
||||
* their stored key count). Caps entries/keys per index node to force splits on
|
||||
* small datasets. 0 = natural full-page fanout. */
|
||||
void store__set_btree_order(EngramPagedStore* s, int leaf_max, int internal_max);
|
||||
|
||||
/* Introspection for tests/tools. */
|
||||
uint64_t store_page_count(const EngramPagedStore* s);
|
||||
|
||||
/* ── M2: WAL + checkpoint + crash recovery + one-time legacy import ─────────────
|
||||
*
|
||||
* The durable engram is `neuron.egm` (paged) fronted by `neuron.wal`
|
||||
* (append-only). A mutation is durable once its WAL record is fsync'd
|
||||
* (group-commit). Pages are held write-back in RAM (no-steal) and flushed to the
|
||||
* store only at a checkpoint, so the store file on disk always reflects a
|
||||
* consistent point (`last_checkpoint_lsn`) and the WAL owns everything since.
|
||||
* Recovery = open store, replay WAL forward, redo a record only where the target
|
||||
* record's home page LSN < record LSN (idempotent). JSON is ONLY an import
|
||||
* source / export artifact — never the ongoing store. */
|
||||
|
||||
typedef enum { ENGRAM_WAL_ALWAYS = 0, ENGRAM_WAL_GROUP = 1, ENGRAM_WAL_OFF = 2 } EngramWalSync;
|
||||
|
||||
/* Serializable layer-registry view (the `layers` array of the legacy snapshot). */
|
||||
typedef struct StoreLayer {
|
||||
uint32_t layer_id;
|
||||
char* name;
|
||||
uint32_t activation_priority;
|
||||
int32_t suppressible;
|
||||
int32_t transparent;
|
||||
int32_t injectable;
|
||||
uint8_t* unknown;
|
||||
size_t unknown_len;
|
||||
int tombstoned;
|
||||
} StoreLayer;
|
||||
|
||||
/* Boot the durable engram in `data_dir` (holds neuron.egm + neuron.wal). If the
|
||||
* store is absent but a legacy snapshot.json exists, it is imported ONCE into a
|
||||
* fresh store; thereafter the store is authoritative and JSON is never read again.
|
||||
* On open, the WAL is replayed to recover any post-checkpoint mutations. */
|
||||
EngramPagedStore* engram_open(const char* data_dir);
|
||||
int engram_close(EngramPagedStore* s); /* checkpoint + close */
|
||||
|
||||
/* Force a checkpoint: flush dirty pages → fsync store → advance checkpoint LSN →
|
||||
* reclaim the WAL prefix. Also threshold-triggered automatically on the write path. */
|
||||
int engram_checkpoint(EngramPagedStore* s);
|
||||
|
||||
/* WAL commit policy. engram_open honours env ENGRAM_WAL_SYNC=always|group|off. */
|
||||
void engram_set_wal_sync(EngramPagedStore* s, EngramWalSync policy);
|
||||
|
||||
/* Layer registry. */
|
||||
int store_put_layer(EngramPagedStore* s, const StoreLayer* L);
|
||||
int store_get_layer(EngramPagedStore* s, uint32_t layer_id, StoreLayer* out);
|
||||
int store_del_layer(EngramPagedStore* s, uint32_t layer_id);
|
||||
int store_list_layers(EngramPagedStore* s, StoreLayer** out, size_t* n);
|
||||
void store_layer_free(StoreLayer* L);
|
||||
void store_layers_free(StoreLayer* arr, size_t n);
|
||||
|
||||
/* Edge lookup by id (for hebb updates + idempotency). 1 hit / 0 absent / <0 err. */
|
||||
int store_get_edge(EngramPagedStore* s, const char* id, StoreEdge* out);
|
||||
|
||||
/* HEBB batch: one WAL record updating hebb (+ last_fired) on a set of edges. */
|
||||
typedef struct StoreHebbDelta { const char* edge_id; double hebb; int64_t last_fired; } StoreHebbDelta;
|
||||
int store_hebb_batch(EngramPagedStore* s, const StoreHebbDelta* d, size_t n);
|
||||
|
||||
/* Supersede: logs the (old,new) pair and tombstones old_id at the store; the new
|
||||
* node + `supersedes` edge are logged separately (neuron-layer immutability). */
|
||||
int store_supersede(EngramPagedStore* s, const char* old_id, const char* new_id);
|
||||
|
||||
/* Forget (GC): tombstone id at the store (hard-free deferred to compaction). */
|
||||
int store_forget(EngramPagedStore* s, const char* id);
|
||||
|
||||
/* ── M3: full live enumeration (for the CALLER's resident load + JSON export) ──
|
||||
* Walk the whole store and invoke `cb` once per DISTINCT live node/edge with a
|
||||
* borrowed view (the engine frees it after cb returns — the callback must copy
|
||||
* anything it keeps). De-duplicated by id (canonical latest-live per id, matching
|
||||
* point-read semantics). Returns the count emitted, or <0 on error. The engine
|
||||
* hands out StoreNode/StoreEdge only — it never sees a soul struct (design §10). */
|
||||
typedef void (*StoreNodeScanCb)(const StoreNode* n, void* ctx);
|
||||
typedef void (*StoreEdgeScanCb)(const StoreEdge* e, void* ctx);
|
||||
int store_scan_nodes(EngramPagedStore* s, StoreNodeScanCb cb, void* ctx);
|
||||
int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx);
|
||||
|
||||
/* Introspection / test hooks. */
|
||||
uint64_t engram_wal_next_lsn(const EngramPagedStore* s);
|
||||
uint64_t engram_last_checkpoint_lsn(const EngramPagedStore* s);
|
||||
|
||||
/* ── M4: demand-paging buffer pool (additive residency; on-disk format UNCHANGED) ──
|
||||
*
|
||||
* The write-back, no-steal cache of M2 becomes a bounded, demand-paged buffer
|
||||
* pool. A fixed frame budget (env ENGRAM_POOL_FRAMES; 0 = unlimited; default
|
||||
* large ⇒ whole store resident ⇒ identical to Phase 1) keeps only hot pages in
|
||||
* RAM; a page access that is not resident faults in from neuron.egm, and under
|
||||
* pressure a CLEAN, unpinned frame is evicted (LRU). Dirty frames are never
|
||||
* stolen (M2 no-steal / WAL durability), and superblocks + index root/interior
|
||||
* pages are auto-pinned. Prefetch (env ENGRAM_PREFETCH) reads ahead on scans. */
|
||||
|
||||
/* Pin / unpin an individual page (faults it in and keeps it resident until
|
||||
* unpinned). Pin a hot layer's pages (WM/core) as a set. Idempotent counts. */
|
||||
int store_pin_page(EngramPagedStore* s, uint64_t page_id);
|
||||
int store_unpin_page(EngramPagedStore* s, uint64_t page_id);
|
||||
int store_pin_layer(EngramPagedStore* s, uint32_t layer); /* returns #pages pinned */
|
||||
int store_unpin_layer(EngramPagedStore* s, uint32_t layer);
|
||||
|
||||
/* Buffer-pool introspection. */
|
||||
typedef struct StorePoolStats {
|
||||
size_t cap; /* frame budget (0 = unlimited) */
|
||||
size_t resident; /* frames currently resident */
|
||||
size_t pinned; /* frames that cannot be evicted (dirty/pinned/structural) */
|
||||
size_t dirty; /* dirty (un-checkpointed) frames */
|
||||
unsigned prefetch; /* read-ahead window */
|
||||
uint64_t hits, misses; /* page_read cache hits / demand faults */
|
||||
uint64_t evictions; /* clean frames reclaimed */
|
||||
uint64_t prefetch_reads; /* pages brought in by read-ahead */
|
||||
} StorePoolStats;
|
||||
void store_pool_stats(const EngramPagedStore* s, StorePoolStats* out);
|
||||
int store_pool_resident(const EngramPagedStore* s, uint64_t page_id);
|
||||
|
||||
/* Test hooks: set the frame budget / prefetch window at runtime (NOT format). */
|
||||
void store__set_pool_frames(EngramPagedStore* s, size_t frames);
|
||||
void store__set_prefetch(EngramPagedStore* s, unsigned window);
|
||||
|
||||
/* Crash-test hooks (writes only under a throwaway dir).
|
||||
* store__crash — abandon all RAM state without flush/fsync (power loss).
|
||||
* store__flush_pages — pwrite dirty pages to disk WITHOUT a checkpoint (steal).
|
||||
* store__checkpoint_crashat — run checkpoint but stop (then power-loss) after
|
||||
* `phase` (0..4); phase<0 = full checkpoint. */
|
||||
void store__crash(EngramPagedStore* s);
|
||||
int store__flush_pages(EngramPagedStore* s);
|
||||
int store__checkpoint_crashat(EngramPagedStore* s, int phase);
|
||||
|
||||
/* ── M5: online compaction + background checkpointer (additive; format UNCHANGED) ──
|
||||
*
|
||||
* COMPACTION reclaims the space held by DEAD records — tombstoned nodes/edges
|
||||
* (telemetry prune, forget), superseded ids, and the stale prior versions a
|
||||
* re-put/hebb-batch leaves behind — plus the overflow pages they orphaned. It
|
||||
* rewrites only the LIVE records (bit-exact) into a fresh, densely packed image
|
||||
* with fresh id + adjacency indexes, then commits the swap atomically, so the
|
||||
* .egm file physically SHRINKS and the freed pages are reclaimed. Crash-safe:
|
||||
* a crash at any instant recovers to either the pre- or the post-compaction
|
||||
* store, never a corrupt mix (atomic rename is the commit point). It cooperates
|
||||
* with the M4 pool (no-steal, pins) by building into a separate store whose own
|
||||
* pool honours ENGRAM_POOL_FRAMES, then INVALIDATING every frame of the live
|
||||
* pool so no stale frame survives for a relocated page.
|
||||
*
|
||||
* Requires a quiesce point: store_compact performs a checkpoint (or sync) at
|
||||
* entry, so it is called between mutations, not concurrently with one. */
|
||||
int store_compact(EngramPagedStore* s);
|
||||
|
||||
/* Test hook: run compaction but stop (then power-loss) after `phase`:
|
||||
* 0 = after the entry checkpoint, before building (→ recovers pre-compaction)
|
||||
* 1 = after building+fsync the new image, before rename (→ pre-compaction)
|
||||
* 2 = after the atomic rename, before reopening RAM state (→ post-compaction)
|
||||
* phase<0 = full compaction. Frees `s` on a crash phase (like the checkpoint hook). */
|
||||
int store__compact_crashat(EngramPagedStore* s, int phase);
|
||||
|
||||
/* BACKGROUND CHECKPOINTER policy. A checkpoint fires automatically on the write
|
||||
* path when ANY armed trigger trips, reclaiming the WAL prefix without an explicit
|
||||
* engram_checkpoint. 0 disables that trigger. Same checkpoint semantics as M2.
|
||||
* ops — mutations since last checkpoint (default 100000)
|
||||
* dirty_pages — dirty (un-checkpointed) pool frames
|
||||
* wal_bytes — bytes appended to the WAL since it was last reclaimed
|
||||
* interval_ms — wall-clock ms since the last checkpoint (checked on writes) */
|
||||
void store_set_checkpoint_policy(EngramPagedStore* s, uint64_t ops,
|
||||
size_t dirty_pages, uint64_t wal_bytes,
|
||||
long long interval_ms);
|
||||
|
||||
/* Introspection: number of pages currently on the free-list. */
|
||||
uint64_t store_free_page_count(const EngramPagedStore* s);
|
||||
|
||||
#endif /* ENGRAM_STORE_H */
|
||||
@@ -0,0 +1,157 @@
|
||||
/* engram_verify.c — the VERIFIER layer. Pure compositions over engram_reason.h +
|
||||
* engram_geometry.h. stdlib + libm only; READ-ONLY over its inputs; touches no
|
||||
* store/index/activation. See engram_verify.h for the design and the frame contract. */
|
||||
#include "engram_verify.h"
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
|
||||
/* ── small float-vector helpers (mirror engram_reason.c) ────────────────────── */
|
||||
static double vdot(const float* a, const float* b, int dim) {
|
||||
double s = 0; for (int i = 0; i < dim; i++) s += (double)a[i] * (double)b[i]; return s;
|
||||
}
|
||||
static double l2(const float* a, const float* b, int dim) {
|
||||
double s = 0; for (int i = 0; i < dim; i++) { double d = (double)a[i] - (double)b[i]; s += d * d; }
|
||||
return sqrt(s);
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════ GROUNDING ══════════ */
|
||||
int engram_verify_grounding(const float* claim, int dim,
|
||||
const GeoDescriptor* const* evidence, int n_evidence,
|
||||
double ext_floor, double ground_threshold,
|
||||
GeoGrounding* out) {
|
||||
if (!claim || dim <= 0 || !evidence || n_evidence < 1 || !out) return -1;
|
||||
if (!(ext_floor > 0)) ext_floor = 1.0;
|
||||
if (!(ground_threshold > 0 && ground_threshold < 1)) ground_threshold = 0.5;
|
||||
memset(out, 0, sizeof *out);
|
||||
out->n_evidence = n_evidence;
|
||||
out->best = -1;
|
||||
out->nearest_centroid_l2 = INFINITY;
|
||||
out->scores = malloc((size_t)n_evidence * sizeof(double));
|
||||
if (!out->scores) return -1;
|
||||
|
||||
double best = -1;
|
||||
for (int i = 0; i < n_evidence; i++) {
|
||||
const GeoDescriptor* e = evidence[i];
|
||||
GeoFit f;
|
||||
if (!e || e->dim != dim || !e->centroid ||
|
||||
engram_reason_point_fit(e, claim, ext_floor, &f) != 0) {
|
||||
out->scores[i] = 0.0;
|
||||
continue;
|
||||
}
|
||||
out->scores[i] = f.score;
|
||||
double cl2 = l2(claim, e->centroid, dim);
|
||||
if (cl2 < out->nearest_centroid_l2) out->nearest_centroid_l2 = cl2;
|
||||
if (out->best < 0 || f.score > best) {
|
||||
best = f.score;
|
||||
out->best = i;
|
||||
out->grounding = f.score;
|
||||
out->best_distance = f.distance;
|
||||
out->best_ortho = f.ortho_residual;
|
||||
}
|
||||
}
|
||||
if (out->best < 0) { out->grounding = 0.0; out->best_distance = INFINITY; }
|
||||
out->grounded = (out->grounding >= ground_threshold) ? 1 : 0;
|
||||
return 0;
|
||||
}
|
||||
void engram_verify_grounding_free(GeoGrounding* out) {
|
||||
if (!out) return;
|
||||
free(out->scores); out->scores = NULL;
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════ CONSISTENCY ════════ */
|
||||
int engram_verify_consistency(const float* claim, int dim,
|
||||
const GeoDescriptor* context,
|
||||
const GeoDescriptor* pole_pos, const GeoDescriptor* pole_neg,
|
||||
const GeoDescriptor* forbidden,
|
||||
double ext_floor, double deadzone_frac,
|
||||
double forbidden_thresh, double max_distance,
|
||||
GeoConsistency* out) {
|
||||
if (!claim || dim <= 0 || !out) return -1;
|
||||
if (!(ext_floor > 0)) ext_floor = 1.0;
|
||||
if (!(deadzone_frac >= 0 && deadzone_frac < 1)) deadzone_frac = 0.10;
|
||||
if (!(forbidden_thresh > 0 && forbidden_thresh < 1)) forbidden_thresh = 0.5;
|
||||
memset(out, 0, sizeof *out);
|
||||
out->verdict = GEO_CONSIST_OK;
|
||||
out->consistency = 1.0;
|
||||
|
||||
int do_polarity = (pole_pos && pole_neg);
|
||||
int do_distance = (max_distance > 0);
|
||||
if ((do_polarity || do_distance) &&
|
||||
(!context || context->dim != dim || !context->centroid)) return -1;
|
||||
if (do_polarity && (pole_pos->dim != dim || pole_neg->dim != dim ||
|
||||
!pole_pos->centroid || !pole_neg->centroid)) return -1;
|
||||
if (forbidden && (forbidden->dim != dim || !forbidden->centroid)) return -1;
|
||||
|
||||
double pol_score = 1.0, geo_score = 1.0;
|
||||
|
||||
/* ── (a) POLARITY / negation inversion ─────────────────────────────────── */
|
||||
if (do_polarity) {
|
||||
/* axis p = (c_pos − c_neg); midpoint o = ½(c_pos + c_neg). */
|
||||
float* p = malloc((size_t)dim * sizeof(float));
|
||||
float* o = malloc((size_t)dim * sizeof(float));
|
||||
if (!p || !o) { free(p); free(o); return -1; }
|
||||
double pn2 = 0;
|
||||
for (int i = 0; i < dim; i++) {
|
||||
double dpos = (double)pole_pos->centroid[i], dneg = (double)pole_neg->centroid[i];
|
||||
p[i] = (float)(dpos - dneg);
|
||||
o[i] = (float)(0.5 * (dpos + dneg));
|
||||
pn2 += (dpos - dneg) * (dpos - dneg);
|
||||
}
|
||||
double pn = sqrt(pn2);
|
||||
out->polarity_separation = 0.5 * pn;
|
||||
if (pn > 1e-12) {
|
||||
/* signed positions along the axis (projection of (x − o) onto unit p). */
|
||||
float* cdo = malloc((size_t)dim * sizeof(float)); /* claim − o */
|
||||
float* rdo = malloc((size_t)dim * sizeof(float)); /* context − o */
|
||||
if (!cdo || !rdo) { free(p); free(o); free(cdo); free(rdo); return -1; }
|
||||
for (int i = 0; i < dim; i++) {
|
||||
cdo[i] = (float)((double)claim[i] - (double)o[i]);
|
||||
rdo[i] = (float)((double)context->centroid[i] - (double)o[i]);
|
||||
}
|
||||
double claim_side = vdot(cdo, p, dim) / pn; /* units: emb-space length */
|
||||
double ref_side = vdot(rdo, p, dim) / pn;
|
||||
out->polarity_claim = claim_side;
|
||||
out->polarity_reference = ref_side;
|
||||
double dz = deadzone_frac * out->polarity_separation; /* neutral band */
|
||||
if (fabs(claim_side) > dz && fabs(ref_side) > dz &&
|
||||
(claim_side > 0) != (ref_side > 0)) {
|
||||
out->inverted = 1;
|
||||
pol_score = 0.0; /* opposite poles ⇒ zero consistency */
|
||||
} else if (fabs(claim_side) <= dz || fabs(ref_side) <= dz) {
|
||||
pol_score = 0.5; /* neutral / undecided */
|
||||
} else {
|
||||
pol_score = 1.0; /* same pole ⇒ consistent */
|
||||
}
|
||||
free(cdo); free(rdo);
|
||||
}
|
||||
free(p); free(o);
|
||||
}
|
||||
|
||||
/* ── (b) GEOMETRIC contradiction ───────────────────────────────────────── */
|
||||
if (forbidden) {
|
||||
GeoFit f;
|
||||
if (engram_reason_point_fit(forbidden, claim, ext_floor, &f) == 0) {
|
||||
out->forbidden_fit = f.score;
|
||||
if (f.score >= forbidden_thresh) {
|
||||
out->geo_violation = 1;
|
||||
double g = 1.0 - f.score; if (g < 0) g = 0;
|
||||
if (g < geo_score) geo_score = g;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (do_distance) {
|
||||
out->context_distance = l2(claim, context->centroid, dim);
|
||||
if (out->context_distance > max_distance) {
|
||||
out->geo_violation = 1;
|
||||
geo_score = 0.0;
|
||||
}
|
||||
}
|
||||
|
||||
/* ── verdict + scalar (polarity is the headline; both flags stay visible) ─ */
|
||||
out->consistency = (pol_score < geo_score) ? pol_score : geo_score;
|
||||
if (out->inverted) out->verdict = GEO_CONSIST_POLARITY;
|
||||
else if (out->geo_violation) out->verdict = GEO_CONSIST_GEOMETRIC;
|
||||
else out->verdict = GEO_CONSIST_OK;
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
/* engram_verify.h — the VERIFIER layer: GROUNDING + CONSISTENCY over the live
|
||||
* geometry (engram_geometry.h) and reasoning (engram_reason.h) operators.
|
||||
*
|
||||
* The geometry PROPOSES (cheap, creative, sometimes wrong); the verifier DISPOSES.
|
||||
* This layer catches the class of failure a grammar check never sees: a fluent,
|
||||
* confident, WRONG output — the "plausible lie". The motivating case: a translation
|
||||
* that DELETED a negation so "you never fought" became "you argued" — reassurance
|
||||
* inverted into accusation, grammatical and invisible, catchable ONLY by the geometry.
|
||||
*
|
||||
* GROUNDING claim → is there ANY real structure that supports it, or is it
|
||||
* floating free of the manifold? (anti-hallucination gate)
|
||||
* CONSISTENCY claim → does it CONTRADICT the established structure? Two catches:
|
||||
* (a) POLARITY: the claim lands on the OPPOSITE side of a negation
|
||||
* axis from the grounded truth (the reassurance→accusation catch),
|
||||
* (b) GEOMETRIC: the claim sits inside a region it must be far from,
|
||||
* or violates a max-distance constraint to its context.
|
||||
*
|
||||
* PURE + READ-ONLY (stdlib + libm only): every function consumes a claim POINT
|
||||
* (float* in R^dim) plus GeoDescriptor(s), and NEVER touches the store, index, or
|
||||
* activation. All geometry is delegated to engram_reason_point_fit / engram_geo_*;
|
||||
* this file only composes and applies thresholds.
|
||||
*
|
||||
* FRAME CONTRACT (inherited): the claim point and every descriptor passed together
|
||||
* MUST share emb `dim` and the same `global_mean` frame — exactly the §5 operator
|
||||
* contract. A function returns <0 on a dim/frame mismatch or bad argument.
|
||||
*/
|
||||
#ifndef ENGRAM_VERIFY_H
|
||||
#define ENGRAM_VERIFY_H
|
||||
|
||||
#include "engram_geometry.h"
|
||||
#include "engram_reason.h"
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
* GROUNDING — anti-hallucination. Score how well a claimed POINT is supported by
|
||||
* the ACTUAL structure: fit the claim against every real evidence neighborhood
|
||||
* (engram_reason_point_fit → in-distribution Mahalanobis + off-model orthogonal
|
||||
* residual) and take the BEST supporter. A claim that sits inside real structure
|
||||
* scores high (grounded); a claim floating far from every neighborhood scores low
|
||||
* on all of them → flagged UNGROUNDED (a hallucination).
|
||||
*
|
||||
* This is an ABSOLUTE-THRESHOLD gate, deliberately distinct from ABDUCTION (which
|
||||
* always RANKS and picks a winner among competing hypotheses): grounding asks the
|
||||
* prior question — "is there any real support at all?" — and is allowed to answer no.
|
||||
* The off-model `ortho_residual` is the sharpest hallucination signal: energy in a
|
||||
* direction the manifold does not even span.
|
||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||
typedef struct {
|
||||
double grounding; /* ∈[0,1]: overall support = best fit score */
|
||||
int grounded; /* 1 iff grounding >= ground_threshold */
|
||||
int best; /* index of best-supporting evidence structure, or −1 */
|
||||
double best_distance; /* full point-to-manifold distance to the best */
|
||||
double best_ortho; /* off-model orthogonal residual of the best fit */
|
||||
double nearest_centroid_l2;/* raw L2 to the nearest evidence centroid (coarse) */
|
||||
int n_evidence;
|
||||
double* scores; /* per-evidence fit score, higher = better (owned)*/
|
||||
} GeoGrounding;
|
||||
/* ext_floor>0 guards zero-extent axes (default 1.0). ground_threshold∈(0,1): the
|
||||
* minimum best-fit score to call the claim grounded (default 0.5). */
|
||||
int engram_verify_grounding(const float* claim, int dim,
|
||||
const GeoDescriptor* const* evidence, int n_evidence,
|
||||
double ext_floor, double ground_threshold,
|
||||
GeoGrounding* out);
|
||||
void engram_verify_grounding_free(GeoGrounding* out);
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
* CONSISTENCY — contradiction detection. Does the claim contradict the established
|
||||
* structure? Two independent sub-checks (either can fire; both flags are reported):
|
||||
*
|
||||
* (a) POLARITY / negation inversion. A polarity axis p is defined by two REAL
|
||||
* poles — pole_pos (asserts X) and pole_neg (asserts ¬X):
|
||||
* p = (c_pos − c_neg)/‖·‖ , midpoint o = ½(c_pos + c_neg).
|
||||
* The claim's side = p·(claim − o); the reference's side = p·(c_context − o).
|
||||
* If the two sides have OPPOSITE sign AND both clear the neutral deadzone, the
|
||||
* claim asserts the polarity opposite to the grounded truth → INVERSION flagged.
|
||||
* This is the "you never fought"→"you argued" catch: the truth ("never fought")
|
||||
* sits on the negate pole, the claim ("argued") on the affirm pole → opposite
|
||||
* sides → flagged, though every word is grammatical.
|
||||
*
|
||||
* (b) GEOMETRIC contradiction. The claim sits INSIDE a `forbidden` region it must
|
||||
* be far from (point_fit score to forbidden ≥ forbidden_thresh), OR it violates
|
||||
* a max-distance constraint to its context centroid (L2 > max_distance).
|
||||
*
|
||||
* pole_pos/pole_neg may both be NULL to skip the polarity check; forbidden may be
|
||||
* NULL and max_distance≤0 to skip the geometric check. `context` (the grounded truth
|
||||
* region) is required whenever polarity or the distance constraint is used.
|
||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||
typedef enum {
|
||||
GEO_CONSIST_OK = 0, /* consistent with context */
|
||||
GEO_CONSIST_POLARITY = 1, /* polarity/negation inversion (asserts ¬X where X) */
|
||||
GEO_CONSIST_GEOMETRIC = 2 /* geometric contradiction (in forbidden / too far) */
|
||||
} GeoConsistencyVerdict;
|
||||
typedef struct {
|
||||
GeoConsistencyVerdict verdict; /* headline (polarity takes precedence) */
|
||||
double consistency; /* ∈[0,1]: min over the checks (1 = fully consistent)*/
|
||||
/* polarity sub-check */
|
||||
int inverted; /* 1 iff a polarity inversion was detected */
|
||||
double polarity_claim; /* p·(claim − o) (signed position on the axis)*/
|
||||
double polarity_reference; /* p·(c_context − o) (the grounded truth's side) */
|
||||
double polarity_separation; /* ½‖c_pos − c_neg‖ (the axis half-length / scale)*/
|
||||
/* geometric sub-check */
|
||||
int geo_violation; /* 1 iff a geometric contradiction was detected */
|
||||
double forbidden_fit; /* claim's point_fit score to the forbidden region*/
|
||||
double context_distance; /* L2(claim, c_context) */
|
||||
} GeoConsistency;
|
||||
/* ext_floor>0 (default 1.0). deadzone_frac∈[0,1): a polarity side within
|
||||
* deadzone_frac·separation of the midpoint is "neutral" and never triggers inversion
|
||||
* (default 0.10). forbidden_thresh∈(0,1): fit-to-forbidden at/above which the claim
|
||||
* counts as inside the forbidden region (default 0.5). max_distance>0 enables the
|
||||
* distance constraint; ≤0 disables it. */
|
||||
int engram_verify_consistency(const float* claim, int dim,
|
||||
const GeoDescriptor* context,
|
||||
const GeoDescriptor* pole_pos, const GeoDescriptor* pole_neg,
|
||||
const GeoDescriptor* forbidden,
|
||||
double ext_floor, double deadzone_frac,
|
||||
double forbidden_thresh, double max_distance,
|
||||
GeoConsistency* out);
|
||||
|
||||
#endif /* ENGRAM_VERIFY_H */
|
||||
@@ -0,0 +1,649 @@
|
||||
/* engram_vindex.c — HNSW ANN index over f32 embedding vectors (design §9 M8).
|
||||
*
|
||||
* Self-contained: plain C11, stdlib + libm (-lm for sqrtf/logf) only. No
|
||||
* dependency on el_runtime; the store is read via its PERMANENT on-disk format
|
||||
* (design §2.4), decoded read-only here so engram_store.{c,h} stay untouched.
|
||||
*
|
||||
* Algorithm: Malkov & Yashunin, "Efficient and robust approximate nearest
|
||||
* neighbor search using Hierarchical Navigable Small World graphs" (2016).
|
||||
* - multi-layer graph; level ~ Exp(1/ln M), assigned by a per-node seeded PRNG
|
||||
* (deterministic: seed = FIXED_SEED ^ node_ordinal) so a rebuild is bit-for-
|
||||
* bit reproducible regardless of wall-clock or global rand() state.
|
||||
* - greedy descent through upper layers to an entry point, then an ef-bounded
|
||||
* best-first search at each layer (Algorithm 2).
|
||||
* - neighbour selection by the diversity heuristic (Algorithm 4), not plain
|
||||
* k-nearest, with keep-pruned backfill for connectivity.
|
||||
* - bidirectional links; a neighbour whose degree exceeds M (2M on layer 0) is
|
||||
* re-pruned with the same heuristic.
|
||||
*
|
||||
* Metric: vectors are L2-normalised on entry, so cosine similarity == dot
|
||||
* product; distance = 1 - dot (in [0,2], smaller == nearer). Deterministic tie-
|
||||
* breaks are by element index so results are stable across identical builds.
|
||||
*/
|
||||
#include "engram_vindex.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
/* Deterministic PRNG seed base (fixed constant — never wall-clock/rand). */
|
||||
#define VINDEX_FIXED_SEED 0x9E3779B97F4A7C15ULL
|
||||
|
||||
/* ── deterministic PRNG (splitmix64) ──────────────────────────────────────── */
|
||||
static inline uint64_t splitmix64(uint64_t* s){
|
||||
uint64_t z = (*s += 0x9E3779B97F4A7C15ULL);
|
||||
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL;
|
||||
z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL;
|
||||
return z ^ (z >> 31);
|
||||
}
|
||||
/* Uniform double in (0,1]. */
|
||||
static inline double sm_uniform(uint64_t* s){
|
||||
/* 53-bit mantissa; +1 keeps it in (0,1] so log() never sees 0. */
|
||||
return ((double)((splitmix64(s) >> 11) + 1)) * (1.0 / 9007199254740993.0);
|
||||
}
|
||||
|
||||
/* ── element + index structures ───────────────────────────────────────────── */
|
||||
typedef struct {
|
||||
int count;
|
||||
int cap;
|
||||
int* ids; /* neighbour element indices */
|
||||
} NeighList;
|
||||
|
||||
typedef struct {
|
||||
uint64_t node_id;
|
||||
int level; /* top layer this element appears on (>=0) */
|
||||
float* vec; /* dim floats, L2-normalised */
|
||||
NeighList* links; /* level+1 lists; links[l] = neighbours at layer l */
|
||||
} Elem;
|
||||
|
||||
struct VIndex {
|
||||
int dim;
|
||||
int M; /* max neighbours per node, upper layers */
|
||||
int M0; /* == 2*M, layer 0 */
|
||||
int ef_construction;
|
||||
double mL; /* level normaliser = 1/ln(M) */
|
||||
|
||||
Elem* elems;
|
||||
size_t n;
|
||||
size_t cap;
|
||||
|
||||
int entry; /* entry-point element index, -1 if empty */
|
||||
int max_level; /* current top layer */
|
||||
|
||||
/* scratch: version-stamped visited set (O(1) reset). */
|
||||
uint32_t* visited;
|
||||
uint32_t visit_epoch;
|
||||
size_t visited_cap;
|
||||
};
|
||||
|
||||
/* ── small helpers ────────────────────────────────────────────────────────── */
|
||||
static float* vec_normalise_copy(const float* v, int dim){
|
||||
float* out = (float*)malloc((size_t)dim * sizeof(float));
|
||||
if (!out) return NULL;
|
||||
double ss = 0.0;
|
||||
for (int i=0;i<dim;i++) ss += (double)v[i]*(double)v[i];
|
||||
if (ss > 0.0){
|
||||
float inv = (float)(1.0 / sqrt(ss));
|
||||
for (int i=0;i<dim;i++) out[i] = v[i]*inv;
|
||||
} else {
|
||||
for (int i=0;i<dim;i++) out[i] = 0.0f; /* zero vector stays zero */
|
||||
}
|
||||
return out;
|
||||
}
|
||||
/* Cosine distance between two normalised vectors: 1 - dot. In [0,2].
|
||||
* Float accumulation in 4 lanes so the compiler auto-vectorises the hot path
|
||||
* (this is the dominant cost of both build and search). */
|
||||
static float vdist(const VIndex* ix, const float* a, const float* b){
|
||||
int dim = ix->dim;
|
||||
float s0=0,s1=0,s2=0,s3=0;
|
||||
int i=0;
|
||||
for (; i+4<=dim; i+=4){
|
||||
s0 += a[i]*b[i]; s1 += a[i+1]*b[i+1];
|
||||
s2 += a[i+2]*b[i+2]; s3 += a[i+3]*b[i+3];
|
||||
}
|
||||
float dot = (s0+s1)+(s2+s3);
|
||||
for (; i<dim; i++) dot += a[i]*b[i];
|
||||
return 1.0f - dot;
|
||||
}
|
||||
|
||||
static int nl_push(NeighList* nl, int id){
|
||||
if (nl->count == nl->cap){
|
||||
int nc = nl->cap ? nl->cap*2 : 4;
|
||||
int* np = (int*)realloc(nl->ids, (size_t)nc*sizeof(int));
|
||||
if (!np) return -1;
|
||||
nl->ids = np; nl->cap = nc;
|
||||
}
|
||||
nl->ids[nl->count++] = id;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── binary heaps over (dist,elem) pairs ──────────────────────────────────── */
|
||||
typedef struct { float d; int e; } Pair;
|
||||
typedef struct { Pair* a; int n, cap; } Heap;
|
||||
|
||||
static int heap_reserve(Heap* h, int need){
|
||||
if (need <= h->cap) return 0;
|
||||
int nc = h->cap ? h->cap*2 : 16;
|
||||
while (nc < need) nc *= 2;
|
||||
Pair* na = (Pair*)realloc(h->a, (size_t)nc*sizeof(Pair));
|
||||
if (!na) return -1;
|
||||
h->a = na; h->cap = nc; return 0;
|
||||
}
|
||||
/* Order predicate: for a MAX-heap on distance, "higher priority" = larger dist;
|
||||
* ties broken by larger element index (deterministic + stable). is_max selects. */
|
||||
static inline int pair_before(Pair x, Pair y, int is_max){
|
||||
if (x.d != y.d) return is_max ? (x.d > y.d) : (x.d < y.d);
|
||||
return is_max ? (x.e > y.e) : (x.e < y.e);
|
||||
}
|
||||
static int heap_push(Heap* h, Pair v, int is_max){
|
||||
if (heap_reserve(h, h->n+1)) return -1;
|
||||
int i = h->n++;
|
||||
h->a[i] = v;
|
||||
while (i > 0){
|
||||
int p = (i-1)/2;
|
||||
if (pair_before(h->a[i], h->a[p], is_max)){
|
||||
Pair t=h->a[i]; h->a[i]=h->a[p]; h->a[p]=t; i=p;
|
||||
} else break;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
static Pair heap_pop(Heap* h, int is_max){
|
||||
Pair top = h->a[0];
|
||||
h->a[0] = h->a[--h->n];
|
||||
int i = 0;
|
||||
for (;;){
|
||||
int l=2*i+1, r=2*i+2, best=i;
|
||||
if (l<h->n && pair_before(h->a[l], h->a[best], is_max)) best=l;
|
||||
if (r<h->n && pair_before(h->a[r], h->a[best], is_max)) best=r;
|
||||
if (best==i) break;
|
||||
Pair t=h->a[i]; h->a[i]=h->a[best]; h->a[best]=t; i=best;
|
||||
}
|
||||
return top;
|
||||
}
|
||||
|
||||
/* ── visited set ──────────────────────────────────────────────────────────── */
|
||||
static int visited_ensure(VIndex* ix){
|
||||
if (ix->visited_cap >= ix->cap && ix->visited) return 0;
|
||||
size_t nc = ix->cap ? ix->cap : 16;
|
||||
uint32_t* nv = (uint32_t*)realloc(ix->visited, nc*sizeof(uint32_t));
|
||||
if (!nv) return -1;
|
||||
if (nc > ix->visited_cap) memset(nv + ix->visited_cap, 0, (nc-ix->visited_cap)*sizeof(uint32_t));
|
||||
ix->visited = nv; ix->visited_cap = nc;
|
||||
return 0;
|
||||
}
|
||||
static inline void visited_reset(VIndex* ix){
|
||||
if (++ix->visit_epoch == 0){ /* wrapped: clear all */
|
||||
memset(ix->visited, 0, ix->visited_cap*sizeof(uint32_t));
|
||||
ix->visit_epoch = 1;
|
||||
}
|
||||
}
|
||||
static inline int is_visited(VIndex* ix, int e){ return ix->visited[e]==ix->visit_epoch; }
|
||||
static inline void mark_visited(VIndex* ix, int e){ ix->visited[e]=ix->visit_epoch; }
|
||||
|
||||
/* ── search one layer (Algorithm 2): best-first, ef-bounded ───────────────── */
|
||||
/* Returns results as an unsorted Heap (max-heap on distance, size<=ef). Caller
|
||||
* owns res->a. `q` is a normalised query. */
|
||||
static int search_layer(VIndex* ix, const float* q, const int* eps, int neps,
|
||||
int ef, int layer, Heap* res /*out, max-heap*/){
|
||||
Heap cand = {0,0,0}; /* min-heap: nearest to expand */
|
||||
res->a=NULL; res->n=0; res->cap=0;
|
||||
visited_reset(ix);
|
||||
for (int i=0;i<neps;i++){
|
||||
int e = eps[i];
|
||||
if (is_visited(ix,e)) continue;
|
||||
mark_visited(ix,e);
|
||||
float d = vdist(ix, q, ix->elems[e].vec);
|
||||
Pair p = { d, e };
|
||||
if (heap_push(&cand,p,0) || heap_push(res,p,1)){ free(cand.a); return -1; }
|
||||
}
|
||||
while (res->n > ef) heap_pop(res,1); /* trim to ef */
|
||||
|
||||
while (cand.n > 0){
|
||||
Pair c = heap_pop(&cand,0);
|
||||
float worst = res->a[0].d; /* farthest kept result */
|
||||
if (res->n >= ef && c.d > worst) break;
|
||||
Elem* ce = &ix->elems[c.e];
|
||||
if (layer <= ce->level){
|
||||
NeighList* nl = &ce->links[layer];
|
||||
for (int i=0;i<nl->count;i++){
|
||||
int e = nl->ids[i];
|
||||
if (is_visited(ix,e)) continue;
|
||||
mark_visited(ix,e);
|
||||
float d = vdist(ix, q, ix->elems[e].vec);
|
||||
if (res->n < ef || d < res->a[0].d){
|
||||
Pair p = { d, e };
|
||||
if (heap_push(&cand,p,0) || heap_push(res,p,1)){ free(cand.a); return -1; }
|
||||
if (res->n > ef) heap_pop(res,1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
free(cand.a);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── neighbour selection heuristic (Algorithm 4) ──────────────────────────── */
|
||||
/* From candidate pairs W (any order), pick up to M diverse neighbours of q.
|
||||
* Keep c only if it is nearer to q than to every already-chosen neighbour;
|
||||
* backfill from the pruned set (nearest first) to reach M for connectivity.
|
||||
* Writes chosen element indices into out[], returns the count. */
|
||||
static int select_neighbors(VIndex* ix, const float* q, Pair* W, int nW, int M, int* out){
|
||||
(void)q; /* q's distances are precomputed in W[].d; kept for call-site clarity */
|
||||
/* sort W ascending by (dist,elem) — deterministic. */
|
||||
for (int i=1;i<nW;i++){ /* insertion sort (nW small) */
|
||||
Pair key=W[i]; int j=i-1;
|
||||
while (j>=0 && !pair_before(W[j],key,0)){ W[j+1]=W[j]; j--; }
|
||||
W[j+1]=key;
|
||||
}
|
||||
int nout = 0;
|
||||
Pair* pruned = (Pair*)malloc((size_t)(nW?nW:1)*sizeof(Pair));
|
||||
int npr = 0;
|
||||
if (!pruned) return -1;
|
||||
for (int i=0;i<nW && nout<M;i++){
|
||||
int good = 1;
|
||||
for (int j=0;j<nout;j++){
|
||||
float d = vdist(ix, ix->elems[W[i].e].vec, ix->elems[out[j]].vec);
|
||||
if (d < W[i].d){ good = 0; break; } /* nearer an existing pick → drop */
|
||||
}
|
||||
if (good) out[nout++] = W[i].e;
|
||||
else pruned[npr++] = W[i];
|
||||
}
|
||||
for (int i=0;i<npr && nout<M;i++) out[nout++] = pruned[i].e; /* keep-pruned backfill */
|
||||
free(pruned);
|
||||
return nout;
|
||||
}
|
||||
|
||||
/* Re-prune a neighbour's over-full adjacency list back to `Mmax`. */
|
||||
static void prune_links(VIndex* ix, int e, int layer, int Mmax){
|
||||
NeighList* nl = &ix->elems[e].links[layer];
|
||||
if (nl->count <= Mmax) return;
|
||||
const float* base = ix->elems[e].vec;
|
||||
Pair* W = (Pair*)malloc((size_t)nl->count*sizeof(Pair));
|
||||
if (!W) return;
|
||||
int nW = nl->count;
|
||||
for (int i=0;i<nW;i++) W[i] = (Pair){ vdist(ix, base, ix->elems[nl->ids[i]].vec), nl->ids[i] };
|
||||
int* keep = (int*)malloc((size_t)nW*sizeof(int));
|
||||
if (!keep){ free(W); return; }
|
||||
int nk = select_neighbors(ix, base, W, nW, Mmax, keep);
|
||||
if (nk >= 0){ nl->count = nk; for (int i=0;i<nk;i++) nl->ids[i]=keep[i]; }
|
||||
free(keep); free(W);
|
||||
}
|
||||
|
||||
/* ── insert ───────────────────────────────────────────────────────────────── */
|
||||
static int elems_reserve(VIndex* ix){
|
||||
if (ix->n < ix->cap) return 0;
|
||||
size_t nc = ix->cap ? ix->cap*2 : 64;
|
||||
Elem* ne = (Elem*)realloc(ix->elems, nc*sizeof(Elem));
|
||||
if (!ne) return -1;
|
||||
ix->elems = ne; ix->cap = nc;
|
||||
return visited_ensure(ix);
|
||||
}
|
||||
|
||||
int vindex_insert(VIndex* ix, uint64_t node_id, const float* vec){
|
||||
if (!ix || !vec) return -1;
|
||||
if (elems_reserve(ix)) return -1;
|
||||
|
||||
int cur = (int)ix->n;
|
||||
/* deterministic level assignment, seeded per-node. */
|
||||
uint64_t seed = VINDEX_FIXED_SEED ^ (node_id + 0x2545F4914F6CDD1DULL*(uint64_t)cur);
|
||||
int level = (int)(-log(sm_uniform(&seed)) * ix->mL);
|
||||
if (level < 0) level = 0;
|
||||
|
||||
Elem* el = &ix->elems[cur];
|
||||
el->node_id = node_id;
|
||||
el->level = level;
|
||||
el->vec = vec_normalise_copy(vec, ix->dim);
|
||||
el->links = (NeighList*)calloc((size_t)level+1, sizeof(NeighList));
|
||||
if (!el->vec || !el->links){ free(el->vec); free(el->links); return -1; }
|
||||
ix->n++;
|
||||
|
||||
if (ix->entry < 0){ /* first element */
|
||||
ix->entry = cur; ix->max_level = level;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ep = ix->entry;
|
||||
int L = ix->max_level;
|
||||
/* greedy descent through layers above `level` to refine the entry point. */
|
||||
for (int lc = L; lc > level; lc--){
|
||||
Heap r = {0,0,0};
|
||||
int eps1[1] = { ep };
|
||||
if (search_layer(ix, el->vec, eps1, 1, 1, lc, &r)){ return -1; }
|
||||
if (r.n){ ep = r.a[0].e; float bd=r.a[0].d;
|
||||
for (int i=1;i<r.n;i++) if (r.a[i].d<bd){bd=r.a[i].d; ep=r.a[i].e;} }
|
||||
free(r.a);
|
||||
}
|
||||
/* from min(L,level) down to 0: connect. Each layer's ef-results seed the next
|
||||
* layer's entry set; `eps` is heap-owned below the top and freed each step. */
|
||||
int start = (L < level) ? L : level;
|
||||
int eps_stack[1] = { ep };
|
||||
int* eps = eps_stack; /* not owned (stack) until reassigned to malloc'd */
|
||||
int* eps_owned = NULL;
|
||||
int neps = 1;
|
||||
int rc = 0;
|
||||
for (int lc = start; lc >= 0; lc--){
|
||||
int Mmax = (lc==0) ? ix->M0 : ix->M;
|
||||
Heap W = {0,0,0};
|
||||
if (search_layer(ix, el->vec, eps, neps, ix->ef_construction, lc, &W)){ rc=-1; break; }
|
||||
int* chosen = (int*)malloc((size_t)(W.n?W.n:1)*sizeof(int));
|
||||
if (!chosen){ free(W.a); rc=-1; break; }
|
||||
int nc = select_neighbors(ix, el->vec, W.a, W.n, Mmax, chosen);
|
||||
if (nc < 0){ free(chosen); free(W.a); rc=-1; break; }
|
||||
/* link cur <-> chosen (bidirectional), prune neighbours if over-full. */
|
||||
for (int i=0;i<nc;i++){
|
||||
int nb = chosen[i];
|
||||
if (nl_push(&el->links[lc], nb) || nl_push(&ix->elems[nb].links[lc], cur)){
|
||||
free(chosen); free(W.a); rc=-1; goto done;
|
||||
}
|
||||
prune_links(ix, nb, lc, Mmax);
|
||||
}
|
||||
free(chosen);
|
||||
/* next layer's entry points = this layer's ef results. */
|
||||
if (lc > 0){
|
||||
int* neweps = (int*)malloc((size_t)(W.n?W.n:1)*sizeof(int));
|
||||
if (!neweps){ free(W.a); rc=-1; break; }
|
||||
for (int i=0;i<W.n;i++) neweps[i]=W.a[i].e;
|
||||
neps = W.n ? W.n : 1;
|
||||
if (!W.n) neweps[0] = eps[0]; /* fall back to prior ep if empty */
|
||||
free(eps_owned);
|
||||
eps = eps_owned = neweps;
|
||||
}
|
||||
free(W.a);
|
||||
}
|
||||
done:
|
||||
free(eps_owned);
|
||||
if (rc) return -1;
|
||||
if (level > ix->max_level){ ix->max_level = level; ix->entry = cur; }
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── search ───────────────────────────────────────────────────────────────── */
|
||||
int vindex_search(VIndex* ix, const float* query, int k, int ef_search,
|
||||
uint64_t* node_id_out, float* dist_out){
|
||||
if (!ix || !query || k <= 0) return -1;
|
||||
if (ix->entry < 0) return 0;
|
||||
if (ef_search <= 0) ef_search = VINDEX_DEFAULT_EF_SEARCH;
|
||||
if (ef_search < k) ef_search = k;
|
||||
|
||||
float* q = vec_normalise_copy(query, ix->dim);
|
||||
if (!q) return -1;
|
||||
|
||||
int ep = ix->entry;
|
||||
for (int lc = ix->max_level; lc > 0; lc--){
|
||||
Heap r = {0,0,0};
|
||||
int eps[1] = { ep };
|
||||
if (search_layer(ix, q, eps, 1, 1, lc, &r)){ free(q); return -1; }
|
||||
if (r.n){ int b=r.a[0].e; float bd=r.a[0].d;
|
||||
for (int i=1;i<r.n;i++) if (r.a[i].d<bd){bd=r.a[i].d; b=r.a[i].e;}
|
||||
ep = b; }
|
||||
free(r.a);
|
||||
}
|
||||
Heap res = {0,0,0};
|
||||
int eps[1] = { ep };
|
||||
if (search_layer(ix, q, eps, 1, ef_search, 0, &res)){ free(res.a); free(q); return -1; }
|
||||
free(q);
|
||||
|
||||
/* res is a max-heap of size<=ef; pop into ascending order, keep nearest k. */
|
||||
int total = res.n;
|
||||
Pair* sorted = (Pair*)malloc((size_t)(total?total:1)*sizeof(Pair));
|
||||
if (!sorted){ free(res.a); return -1; }
|
||||
for (int i=total-1;i>=0;i--) sorted[i] = heap_pop(&res,1); /* farthest first out → fill from end */
|
||||
free(res.a);
|
||||
|
||||
int out_n = (k < total) ? k : total;
|
||||
for (int i=0;i<out_n;i++){
|
||||
if (node_id_out) node_id_out[i] = ix->elems[sorted[i].e].node_id;
|
||||
if (dist_out) dist_out[i] = sorted[i].d;
|
||||
}
|
||||
free(sorted);
|
||||
return out_n;
|
||||
}
|
||||
|
||||
size_t vindex_size(const VIndex* ix){ return ix ? ix->n : 0; }
|
||||
|
||||
VIndex* vindex_create(int dim, int M, int ef_construction){
|
||||
if (dim <= 0) return NULL;
|
||||
if (M <= 0) M = VINDEX_DEFAULT_M;
|
||||
if (ef_construction <= 0) ef_construction = VINDEX_DEFAULT_EF_CONSTRUCTION;
|
||||
VIndex* ix = (VIndex*)calloc(1, sizeof(VIndex));
|
||||
if (!ix) return NULL;
|
||||
ix->dim = dim;
|
||||
ix->M = M;
|
||||
ix->M0 = 2*M;
|
||||
ix->ef_construction = ef_construction;
|
||||
ix->mL = 1.0 / log((double)M > 1.0 ? (double)M : 2.0);
|
||||
ix->entry = -1;
|
||||
ix->max_level = 0;
|
||||
ix->visit_epoch = 0;
|
||||
return ix;
|
||||
}
|
||||
|
||||
void vindex_free(VIndex* ix){
|
||||
if (!ix) return;
|
||||
for (size_t i=0;i<ix->n;i++){
|
||||
Elem* e = &ix->elems[i];
|
||||
if (e->links) for (int l=0;l<=e->level;l++) free(e->links[l].ids);
|
||||
free(e->links);
|
||||
free(e->vec);
|
||||
}
|
||||
free(ix->elems);
|
||||
free(ix->visited);
|
||||
free(ix);
|
||||
}
|
||||
|
||||
/* ── read-only decode of the paged store node format (design §2.4) ─────────── */
|
||||
/* Mirrors engram_store.c constants; the on-disk format is PERMANENT so these are
|
||||
* safe to duplicate for a read-only harvest of emb vectors. */
|
||||
#define VS_PAGE_SIZE 16384u
|
||||
#define VS_HDR 32u
|
||||
#define VS_SLOT_SIZE 6u
|
||||
#define VS_SLOT_LIVE 1u
|
||||
#define VS_REC_HDR 4u
|
||||
#define VS_REC_OVERFLOW 1u
|
||||
#define VS_PT_NODE 1u
|
||||
#define VS_OVF_NEXT 32u
|
||||
#define VS_OVF_LEN 40u
|
||||
#define VS_OVF_DATA 44u
|
||||
#define VS_NT_ID 1u
|
||||
#define VS_NT_EMB 24u
|
||||
#define VS_NT_EMB_DIM 25u
|
||||
|
||||
static uint16_t vg_u16(const uint8_t* p){ return (uint16_t)(p[0] | (p[1]<<8)); }
|
||||
static uint32_t vg_u32(const uint8_t* p){ uint32_t v=0; for(int i=0;i<4;i++) v|=(uint32_t)p[i]<<(8*i); return v; }
|
||||
static uint64_t vg_u64(const uint8_t* p){ uint64_t v=0; for(int i=0;i<8;i++) v|=(uint64_t)p[i]<<(8*i); return v; }
|
||||
|
||||
static int vs_pread(int fd, uint64_t page, uint8_t* buf){
|
||||
off_t off = (off_t)page * VS_PAGE_SIZE;
|
||||
ssize_t r = pread(fd, buf, VS_PAGE_SIZE, off);
|
||||
return (r == (ssize_t)VS_PAGE_SIZE) ? 0 : -1;
|
||||
}
|
||||
/* Read a (possibly overflowed) record body; caller frees *out. */
|
||||
static int vs_read_body(int fd, const uint8_t* page, uint16_t off, uint16_t len,
|
||||
uint8_t** out, size_t* outlen){
|
||||
if (len < VS_REC_HDR) return -1;
|
||||
uint8_t flags = page[off+3];
|
||||
if (flags & VS_REC_OVERFLOW){
|
||||
uint64_t head = vg_u64(page + off + VS_REC_HDR);
|
||||
uint64_t total = vg_u64(page + off + VS_REC_HDR + 8);
|
||||
uint8_t* body = (uint8_t*)malloc(total ? total : 1);
|
||||
if (!body) return -1;
|
||||
size_t got=0; uint64_t id=head;
|
||||
uint8_t ov[VS_PAGE_SIZE];
|
||||
while (id){
|
||||
if (vs_pread(fd, id, ov)){ free(body); return -1; }
|
||||
uint32_t chunk = vg_u32(ov + VS_OVF_LEN);
|
||||
if (got + chunk > total){ free(body); return -1; }
|
||||
memcpy(body+got, ov+VS_OVF_DATA, chunk); got += chunk;
|
||||
id = vg_u64(ov + VS_OVF_NEXT);
|
||||
}
|
||||
if (got != total){ free(body); return -1; }
|
||||
*out = body; *outlen = total;
|
||||
} else {
|
||||
uint16_t reclen = vg_u16(page + off);
|
||||
if (reclen < VS_REC_HDR) return -1;
|
||||
size_t blen = reclen - VS_REC_HDR;
|
||||
uint8_t* body = (uint8_t*)malloc(blen ? blen : 1);
|
||||
if (!body) return -1;
|
||||
memcpy(body, page + off + VS_REC_HDR, blen);
|
||||
*out = body; *outlen = blen;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
/* Extract id (strdup) and emb (malloc'd float[dim]) from a TLV node body. */
|
||||
static void vs_parse_node(const uint8_t* body, size_t len, char** id_out,
|
||||
float** emb_out, int* dim_out){
|
||||
*id_out=NULL; *emb_out=NULL; *dim_out=0;
|
||||
size_t i=0;
|
||||
while (i + 5 <= len){
|
||||
uint8_t tag = body[i];
|
||||
uint32_t flen = vg_u32(body + i + 1);
|
||||
if (i + 5 + (size_t)flen > len) break;
|
||||
const uint8_t* v = body + i + 5;
|
||||
if (tag == VS_NT_ID){
|
||||
char* s = (char*)malloc(flen+1);
|
||||
if (s){ memcpy(s,v,flen); s[flen]=0; free(*id_out); *id_out=s; }
|
||||
} else if (tag == VS_NT_EMB){
|
||||
int dim = (int)(flen/4);
|
||||
float* e = (float*)malloc((size_t)(dim?dim:1)*sizeof(float));
|
||||
if (e){ for (int k=0;k<dim;k++){ uint32_t u=vg_u32(v+k*4); memcpy(&e[k],&u,4);}
|
||||
free(*emb_out); *emb_out=e; if(*dim_out==0) *dim_out=dim; }
|
||||
} else if (tag == VS_NT_EMB_DIM){
|
||||
*dim_out = (int)vg_u32(v);
|
||||
}
|
||||
i += 5 + flen;
|
||||
}
|
||||
}
|
||||
|
||||
/* Tiny open-addressing string set to dedup ids across live records. */
|
||||
typedef struct { char** k; size_t cap, n; } StrSet;
|
||||
static uint64_t vs_fnv(const char* s){ uint64_t h=1469598103934665603ULL; for(;*s;++s){h^=(uint8_t)*s;h*=1099511628211ULL;} return h; }
|
||||
static int strset_add(StrSet* s, const char* key){ /* 1 added, 0 dup, -1 err */
|
||||
if (s->n*2 >= s->cap){
|
||||
size_t nc = s->cap ? s->cap*2 : 1024;
|
||||
char** nk = (char**)calloc(nc, sizeof(char*));
|
||||
if (!nk) return -1;
|
||||
for (size_t i=0;i<s->cap;i++) if (s->k[i]){ size_t j=vs_fnv(s->k[i])&(nc-1); while(nk[j]) j=(j+1)&(nc-1); nk[j]=s->k[i]; }
|
||||
free(s->k); s->k=nk; s->cap=nc;
|
||||
}
|
||||
size_t j = vs_fnv(key)&(s->cap-1);
|
||||
while (s->k[j]){ if (strcmp(s->k[j],key)==0) return 0; j=(j+1)&(s->cap-1); }
|
||||
char* d = strdup(key); if(!d) return -1;
|
||||
s->k[j]=d; s->n++;
|
||||
return 1;
|
||||
}
|
||||
static void strset_free(StrSet* s){ for(size_t i=0;i<s->cap;i++) free(s->k[i]); free(s->k); }
|
||||
|
||||
int vindex_build_from_store(VIndex* ix, const char* store_path,
|
||||
char*** ids_out, int* n_out){
|
||||
if (!ix || !store_path) return -1;
|
||||
int fd = open(store_path, O_RDONLY);
|
||||
if (fd < 0) return -1;
|
||||
struct stat st;
|
||||
if (fstat(fd, &st) != 0){ close(fd); return -1; }
|
||||
uint64_t npages = (uint64_t)st.st_size / VS_PAGE_SIZE;
|
||||
|
||||
char** ids = NULL; size_t ids_n = 0, ids_cap = 0;
|
||||
StrSet seen = {0,0,0};
|
||||
int inserted = 0;
|
||||
uint8_t page[VS_PAGE_SIZE];
|
||||
|
||||
for (uint64_t pg = 2; pg < npages; pg++){ /* pages 0,1 = superblocks */
|
||||
if (vs_pread(fd, pg, page)) continue;
|
||||
if (page[8] != VS_PT_NODE) continue;
|
||||
int slots = vg_u16(page + 10);
|
||||
for (int sidx=0; sidx<slots; sidx++){
|
||||
const uint8_t* sp = page + VS_HDR + (size_t)sidx*VS_SLOT_SIZE;
|
||||
uint16_t off = vg_u16(sp), len = vg_u16(sp+2), fl = vg_u16(sp+4);
|
||||
if (fl != VS_SLOT_LIVE) continue;
|
||||
if ((size_t)off + VS_REC_HDR > VS_PAGE_SIZE) continue;
|
||||
uint8_t* body=NULL; size_t blen=0;
|
||||
if (vs_read_body(fd, page, off, len, &body, &blen)) continue;
|
||||
char* id=NULL; float* emb=NULL; int dim=0;
|
||||
vs_parse_node(body, blen, &id, &emb, &dim);
|
||||
free(body);
|
||||
if (!id || !emb || dim != ix->dim){ free(id); free(emb); continue; }
|
||||
int add = strset_add(&seen, id);
|
||||
if (add <= 0){ free(id); free(emb); continue; } /* dup or err */
|
||||
if (vindex_insert(ix, (uint64_t)inserted, emb) != 0){ free(id); free(emb); break; }
|
||||
free(emb);
|
||||
if (ids_n == ids_cap){
|
||||
size_t nc = ids_cap ? ids_cap*2 : 256;
|
||||
char** ni = (char**)realloc(ids, nc*sizeof(char*));
|
||||
if (!ni){ free(id); break; }
|
||||
ids = ni; ids_cap = nc;
|
||||
}
|
||||
ids[ids_n++] = id; /* transfers ownership */
|
||||
inserted++;
|
||||
}
|
||||
}
|
||||
close(fd);
|
||||
strset_free(&seen);
|
||||
if (ids_out){ *ids_out = ids; if (n_out) *n_out = (int)ids_n; }
|
||||
else { for (size_t i=0;i<ids_n;i++) free(ids[i]); free(ids); if (n_out) *n_out=(int)ids_n; }
|
||||
return inserted;
|
||||
}
|
||||
|
||||
/* ── optional persistence (index is rebuildable; convenience only) ─────────── */
|
||||
#define VINDEX_SAVE_MAGIC "EGVIDX01"
|
||||
int vindex_save(const VIndex* ix, const char* path){
|
||||
if (!ix || !path) return -1;
|
||||
FILE* f = fopen(path, "wb");
|
||||
if (!f) return -1;
|
||||
int ok = 1;
|
||||
#define WR(p,n) do{ if(fwrite((p),1,(n),f)!=(size_t)(n)) ok=0; }while(0)
|
||||
WR(VINDEX_SAVE_MAGIC, 8);
|
||||
int32_t hdr[6] = { ix->dim, ix->M, ix->ef_construction, (int32_t)ix->n, ix->entry, ix->max_level };
|
||||
WR(hdr, sizeof(hdr));
|
||||
for (size_t i=0; ok && i<ix->n; i++){
|
||||
Elem* e = &ix->elems[i];
|
||||
WR(&e->node_id, sizeof(uint64_t));
|
||||
int32_t lvl = e->level; WR(&lvl, sizeof(int32_t));
|
||||
WR(e->vec, (size_t)ix->dim*sizeof(float));
|
||||
for (int l=0; ok && l<=e->level; l++){
|
||||
int32_t c = e->links[l].count; WR(&c, sizeof(int32_t));
|
||||
WR(e->links[l].ids, (size_t)c*sizeof(int));
|
||||
}
|
||||
}
|
||||
#undef WR
|
||||
fclose(f);
|
||||
return ok ? 0 : -1;
|
||||
}
|
||||
VIndex* vindex_load(const char* path){
|
||||
FILE* f = fopen(path, "rb");
|
||||
if (!f) return NULL;
|
||||
char magic[8];
|
||||
if (fread(magic,1,8,f)!=8 || memcmp(magic,VINDEX_SAVE_MAGIC,8)!=0){ fclose(f); return NULL; }
|
||||
int32_t hdr[6];
|
||||
if (fread(hdr,sizeof(hdr),1,f)!=1){ fclose(f); return NULL; }
|
||||
VIndex* ix = vindex_create(hdr[0], hdr[1], hdr[2]);
|
||||
if (!ix){ fclose(f); return NULL; }
|
||||
size_t N = (size_t)hdr[3];
|
||||
int ok = 1;
|
||||
for (size_t i=0; ok && i<N; i++){
|
||||
if (elems_reserve(ix)){ ok=0; break; }
|
||||
Elem* e = &ix->elems[ix->n];
|
||||
int32_t lvl;
|
||||
if (fread(&e->node_id,sizeof(uint64_t),1,f)!=1 || fread(&lvl,sizeof(int32_t),1,f)!=1){ ok=0; break; }
|
||||
e->level = lvl;
|
||||
e->vec = (float*)malloc((size_t)ix->dim*sizeof(float));
|
||||
e->links = (NeighList*)calloc((size_t)lvl+1, sizeof(NeighList));
|
||||
if (!e->vec || !e->links){ free(e->vec); free(e->links); ok=0; break; }
|
||||
if (fread(e->vec,sizeof(float),(size_t)ix->dim,f)!=(size_t)ix->dim){ ok=0; }
|
||||
for (int l=0; ok && l<=lvl; l++){
|
||||
int32_t c; if (fread(&c,sizeof(int32_t),1,f)!=1){ ok=0; break; }
|
||||
e->links[l].ids = (int*)malloc((size_t)(c?c:1)*sizeof(int));
|
||||
e->links[l].cap = c; e->links[l].count = c;
|
||||
if (c && fread(e->links[l].ids,sizeof(int),(size_t)c,f)!=(size_t)c){ ok=0; }
|
||||
}
|
||||
ix->n++;
|
||||
}
|
||||
ix->entry = hdr[4]; ix->max_level = hdr[5];
|
||||
fclose(f);
|
||||
if (!ok){ vindex_free(ix); return NULL; }
|
||||
return ix;
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
/* engram_vindex.h — M8 of the engram query engine: an approximate-nearest-
|
||||
* neighbour (ANN) vector index over the node embedding vectors, for fast
|
||||
* activation-seed selection.
|
||||
*
|
||||
* Replaces the O(n) cosine scan over emb vectors (design §9 M8; backlog #20)
|
||||
* with an HNSW (Hierarchical Navigable Small World) graph that returns
|
||||
* high-recall top-k seeds in ~O(log n).
|
||||
*
|
||||
* Standalone module: plain C11, stdlib + libm only. It does NOT modify the
|
||||
* store format or engram_store.{c,h}; vindex_build_from_store() decodes the
|
||||
* PERMANENT on-disk node format (design §2.4) read-only to harvest emb vectors.
|
||||
*
|
||||
* Similarity metric: cosine. Vectors are L2-normalised on insert/query, so
|
||||
* cosine similarity == dot product. Reported distance = 1 - cosine_similarity
|
||||
* (range [0,2]); smaller == closer. A query equal to an indexed vector scores
|
||||
* distance ~0 against it.
|
||||
*
|
||||
* The index is fully rebuildable from the store, so persistence is optional for
|
||||
* this milestone (see vindex_save/vindex_load below — provided as a convenience;
|
||||
* boot may simply rebuild via vindex_build_from_store()).
|
||||
*/
|
||||
#ifndef ENGRAM_VINDEX_H
|
||||
#define ENGRAM_VINDEX_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
/* Tuned defaults (rationale in engram_vindex.c). Pass 0 to vindex_create for
|
||||
* M / ef_construction to take these; pass ef_search<=0 to vindex_search for
|
||||
* VINDEX_DEFAULT_EF_SEARCH. */
|
||||
#define VINDEX_DEFAULT_M 24
|
||||
#define VINDEX_DEFAULT_EF_CONSTRUCTION 200
|
||||
#define VINDEX_DEFAULT_EF_SEARCH 128
|
||||
|
||||
typedef struct VIndex VIndex;
|
||||
|
||||
/* Create an index over `dim`-dimensional f32 vectors.
|
||||
* M — max neighbours per node on upper layers (2*M on layer 0).
|
||||
* ef_construction — candidate-list width during insert (recall/build cost).
|
||||
* Pass M<=0 or ef_construction<=0 to use the VINDEX_DEFAULT_* above.
|
||||
* Returns NULL on bad args / OOM. */
|
||||
VIndex* vindex_create(int dim, int M, int ef_construction);
|
||||
|
||||
/* Insert one vector under an opaque caller-defined node_id (need not be unique,
|
||||
* but the caller is responsible for meaning). `vec` has `dim` floats; it is
|
||||
* copied and L2-normalised internally. A zero vector is accepted (it simply has
|
||||
* distance ~1 to everything; never produces NaN). Returns 0 on success, <0 on
|
||||
* error (bad args / OOM). */
|
||||
int vindex_insert(VIndex* idx, uint64_t node_id, const float* vec);
|
||||
|
||||
/* Top-k search by cosine similarity. Writes up to k results (fewer if the index
|
||||
* holds fewer than k elements) into node_id_out[] / dist_out[], ordered nearest
|
||||
* first (ascending distance). Either out array may be NULL to skip it.
|
||||
* ef_search — search-time candidate width; larger == higher recall, slower.
|
||||
* Pass <=0 for VINDEX_DEFAULT_EF_SEARCH. Internally clamped to >=k.
|
||||
* Returns the number of results written, or <0 on error. */
|
||||
int vindex_search(VIndex* idx, const float* query, int k, int ef_search,
|
||||
uint64_t* node_id_out, float* dist_out);
|
||||
|
||||
/* Number of vectors currently indexed. */
|
||||
size_t vindex_size(const VIndex* idx);
|
||||
|
||||
void vindex_free(VIndex* idx);
|
||||
|
||||
/* Build an index by scanning every live node record in the paged store at
|
||||
* `store_path` (the on-disk format is decoded read-only; the store need not be
|
||||
* open). Nodes without an emb vector, or whose emb_dim != idx->dim, are skipped.
|
||||
* Each inserted node is assigned node_id = its 0-based insertion ordinal; if
|
||||
* `ids_out`/`n_out` are non-NULL, *ids_out is set to a malloc'd array of that
|
||||
* many strdup'd string ids (ids_out[node_id] == the store id) and *n_out to the
|
||||
* count — the caller frees each string and the array. Returns the number of
|
||||
* vectors inserted, or <0 on error. */
|
||||
int vindex_build_from_store(VIndex* idx, const char* store_path,
|
||||
char*** ids_out, int* n_out);
|
||||
|
||||
/* Optional persistence (index is rebuildable from the store; provided for
|
||||
* convenience). vindex_save writes a self-describing snapshot; vindex_load
|
||||
* reconstructs an index from one. Return 0 / non-NULL on success. */
|
||||
int vindex_save(const VIndex* idx, const char* path);
|
||||
VIndex* vindex_load(const char* path);
|
||||
|
||||
#endif /* ENGRAM_VINDEX_H */
|
||||
@@ -2,7 +2,7 @@
|
||||
//
|
||||
// Thin El wrappers over seed JSON primitives, plus pure-El builders and
|
||||
// helpers. Each function here corresponds to (and replaces) a C function
|
||||
// from el-compiler/runtime/legacy/el_runtime.c (lines 2692–3333).
|
||||
// from runtime/el_runtime.c (lines 2692–3333).
|
||||
//
|
||||
// Seed primitives consumed by this module:
|
||||
// __json_get(json, key) -> String (value as string)
|
||||
|
||||
@@ -25,8 +25,8 @@
|
||||
// runtime/collections.el \
|
||||
// <user-program.el> > combined.el
|
||||
// ./dist/platform/elc combined.el > output.c
|
||||
// cc -std=c11 -I el-compiler/runtime -lcurl -lpthread \
|
||||
// -o output output.c el-compiler/runtime/el_seed.c
|
||||
// cc -std=c11 -I runtime -lcurl -lpthread \
|
||||
// -o output output.c runtime/el_seed.c
|
||||
|
||||
// This file itself is not compiled — it is documentation only.
|
||||
fn runtime_version() -> String {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// runtime/math.el — Float math, integer utilities, and numeric conversions.
|
||||
//
|
||||
// Implements the math/float surface from el-compiler/runtime/legacy/el_runtime.c
|
||||
// Implements the math/float surface from runtime/el_runtime.c
|
||||
// (lines 303–305 for el_abs/max/min, lines 4725–4771 for float/format ops)
|
||||
// in pure El, using seed primitives.
|
||||
//
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// runtime/time.el — Time operations, sleep, and formatting.
|
||||
//
|
||||
// Implements the time surface from el-compiler/runtime/legacy/el_runtime.c
|
||||
// Implements the time surface from runtime/el_runtime.c
|
||||
// (lines 3334–3440, 3471–3656) in pure El, using seed primitives.
|
||||
//
|
||||
// Seed primitives consumed:
|
||||
|
||||
@@ -11,7 +11,7 @@ cd "$(dirname "$0")"
|
||||
|
||||
EL_HOME="${EL_HOME:-$(cd ../.. && pwd)}"
|
||||
ELC="${EL_HOME}/dist/platform/elc"
|
||||
RUNTIME_DIR="${EL_HOME}/el-compiler/runtime"
|
||||
RUNTIME_DIR="${EL_HOME}/runtime"
|
||||
|
||||
if [ ! -x "${ELC}" ]; then
|
||||
echo "elc not found at ${ELC}" >&2
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user