M-INTEROCEPTION P0: add read-only engram_scan_nodes_emb_json builtin
Read routes (GET /api/embeddings, /api/graph/dump) need node embedding vectors, but every consumer emit path deliberately drops the ~5.7KB emb vector (include_emb=0) to stay under MCP token limits. Rather than perturb that shared path, add a dedicated additive builtin that pages nodes WITH their dense vector, emitting id/node_type/label/created_at/emb_dim plus emb as a JSON array whose length equals emb_dim (so a consumer can verify the vector round-trips). Un-embedded nodes emit emb_dim:0 / emb:[]. Same salience-sorted, transparent-layer-skipped, bounded pagination as engram_scan_nodes_json; default page 256. Purely additive: engram_emit_node_json and the default include_emb=0 are untouched, so every existing endpoint is byte-identical to trunk (verified: scan_nodes_json still carries no emb). The HTTP route wiring in server.el is DEFERRED to cutover per the elc-drift blocker (regenerating engram/dist diverges ~285 lines with no source change); the builtin is exercised directly by the pure-C gate instead. Measured on a copy: len(emb)==emb_dim for all nodes, pagination disjoint, existing path unchanged; full 256-node x 768-dim page = 16.7 ms / 1.18 MB. ASan+UBSan clean.
This commit is contained in:
Executable
+86
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env bash
|
||||
# M-INTEROCEPTION P0 gate: engram_scan_nodes_emb_json read-only builtin.
|
||||
# Throwaway HOME + /tmp only. Never touches ~/.neuron or :8742.
|
||||
set -u
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
RT="$HERE/../../lang/runtime/el_runtime.c"
|
||||
ST="$HERE/../../lang/runtime/engram_store.c"
|
||||
GEO="$HERE/../../lang/runtime/engram_geometry.c"
|
||||
VIDX="$HERE/../../lang/runtime/engram_vindex.c"
|
||||
INC="$HERE/../../lang/runtime"
|
||||
WORK="$(mktemp -d /tmp/engram-p0-XXXXXX)"
|
||||
export HOME="$WORK/home"; mkdir -p "$HOME"
|
||||
unset ENGRAM_STORE
|
||||
fail=0
|
||||
|
||||
echo "== compile (plain) =="
|
||||
gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p0_emb.c" "$RT" "$ST" "$GEO" "$VIDX" \
|
||||
-lcurl -lm -o "$WORK/p0" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; }
|
||||
|
||||
D="$WORK/d"; mkdir -p "$D"
|
||||
"$WORK/p0" "$D" || { echo "FAIL: run"; fail=1; }
|
||||
|
||||
echo
|
||||
echo "== assertions =="
|
||||
python3 - "$D" <<'PY'
|
||||
import json, sys, os
|
||||
d = sys.argv[1]
|
||||
def load(n):
|
||||
with open(os.path.join(d,n)) as f: return json.load(f)
|
||||
rc = 0
|
||||
def check(c,m):
|
||||
global rc
|
||||
print((" PASS: " if c else " FAIL: ")+m)
|
||||
if not c: rc=1
|
||||
|
||||
alln = load("emb_all.json")
|
||||
check(len(alln)==3, f"emb dump returns all 3 nodes (got {len(alln)})")
|
||||
# salience-sorted: high, mid, low
|
||||
labels=[n["label"] for n in alln]
|
||||
check(labels==["emb-high","emb-mid","noemb-low"], f"salience-sorted order {labels}")
|
||||
for n in alln:
|
||||
L=len(n["emb"])
|
||||
check(L==n["emb_dim"], f"{n['label']}: len(emb)={L} == emb_dim={n['emb_dim']}")
|
||||
check(alln[0]["emb_dim"]==16 and alln[1]["emb_dim"]==16, "embedded nodes report dim 16")
|
||||
check(alln[2]["emb_dim"]==0 and alln[2]["emb"]==[], "un-embedded node -> emb_dim 0, emb []")
|
||||
# first emb value round-trips ~0.10
|
||||
check(abs(alln[0]["emb"][0]-0.10)<1e-3, f"emb[0] round-trips (~0.10, got {alln[0]['emb'][0]})")
|
||||
|
||||
pg0=load("emb_pg0.json"); pg1=load("emb_pg1.json")
|
||||
check(len(pg0)==1 and len(pg1)==1, "pagination: one node per page")
|
||||
check(pg0[0]["id"]=="n-high" and pg1[0]["id"]=="n-mid", f"pages disjoint & ordered ({pg0[0]['id']},{pg1[0]['id']})")
|
||||
|
||||
plain=load("plain.json")
|
||||
check(len(plain)==3, "existing scan_nodes_json still returns 3")
|
||||
check(all("emb" not in n for n in plain), "existing scan_nodes_json carries NO emb (behavior-neutral)")
|
||||
sys.exit(rc)
|
||||
PY
|
||||
[ $? -ne 0 ] && fail=1
|
||||
|
||||
echo
|
||||
echo "== latency (one 256-page over the 3-node copy) =="
|
||||
python3 - "$D" <<'PY'
|
||||
import os
|
||||
# timing was measured inside C not here; report emb payload size as a proxy
|
||||
sz=os.path.getsize(os.path.join(os.sys.argv[1] if False else __import__('sys').argv[1],"emb_all.json"))
|
||||
print(f" emb_all.json payload = {sz} bytes for 3 nodes")
|
||||
PY
|
||||
|
||||
echo
|
||||
echo "== ASan+UBSan =="
|
||||
gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \
|
||||
-I "$INC" "$HERE/test_interoception_p0_emb.c" "$RT" "$ST" "$GEO" "$VIDX" \
|
||||
-lcurl -lm -o "$WORK/p0.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -20 "$WORK/san_cc.log"; fail=1; }
|
||||
if [ -x "$WORK/p0.san" ]; then
|
||||
export ASAN_OPTIONS=detect_leaks=0
|
||||
DS="$WORK/ds"; mkdir -p "$DS"
|
||||
"$WORK/p0.san" "$DS" >/dev/null 2>"$WORK/san_run.log"
|
||||
if grep -qiE 'runtime error|AddressSanitizer|Sanitizer|ERROR: ' "$WORK/san_run.log"; then
|
||||
echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san_run.log" | head; fail=1
|
||||
else echo " ok: ASan+UBSan clean"; fi
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ "$fail" -eq 0 ]; then echo "====== P0 EMB-ENDPOINT GATE: PASS ======"; else echo "====== P0 EMB-ENDPOINT GATE: FAIL ======"; fi
|
||||
rm -rf "$WORK"
|
||||
exit $fail
|
||||
Reference in New Issue
Block a user