diff --git a/engram/test/run_interoception_p0.sh b/engram/test/run_interoception_p0.sh new file mode 100755 index 0000000..99507ab --- /dev/null +++ b/engram/test/run_interoception_p0.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# M-INTEROCEPTION P0 gate: engram_scan_nodes_emb_json read-only builtin. +# Throwaway HOME + /tmp only. Never touches ~/.neuron or :8742. +set -u +HERE="$(cd "$(dirname "$0")" && pwd)" +RT="$HERE/../../lang/runtime/el_runtime.c" +ST="$HERE/../../lang/runtime/engram_store.c" +GEO="$HERE/../../lang/runtime/engram_geometry.c" +VIDX="$HERE/../../lang/runtime/engram_vindex.c" +INC="$HERE/../../lang/runtime" +WORK="$(mktemp -d /tmp/engram-p0-XXXXXX)" +export HOME="$WORK/home"; mkdir -p "$HOME" +unset ENGRAM_STORE +fail=0 + +echo "== compile (plain) ==" +gcc -O1 -std=c11 -I "$INC" "$HERE/test_interoception_p0_emb.c" "$RT" "$ST" "$GEO" "$VIDX" \ + -lcurl -lm -o "$WORK/p0" 2>"$WORK/cc.log" || { echo "COMPILE FAILED"; cat "$WORK/cc.log"; rm -rf "$WORK"; exit 1; } + +D="$WORK/d"; mkdir -p "$D" +"$WORK/p0" "$D" || { echo "FAIL: run"; fail=1; } + +echo +echo "== assertions ==" +python3 - "$D" <<'PY' +import json, sys, os +d = sys.argv[1] +def load(n): + with open(os.path.join(d,n)) as f: return json.load(f) +rc = 0 +def check(c,m): + global rc + print((" PASS: " if c else " FAIL: ")+m) + if not c: rc=1 + +alln = load("emb_all.json") +check(len(alln)==3, f"emb dump returns all 3 nodes (got {len(alln)})") +# salience-sorted: high, mid, low +labels=[n["label"] for n in alln] +check(labels==["emb-high","emb-mid","noemb-low"], f"salience-sorted order {labels}") +for n in alln: + L=len(n["emb"]) + check(L==n["emb_dim"], f"{n['label']}: len(emb)={L} == emb_dim={n['emb_dim']}") +check(alln[0]["emb_dim"]==16 and alln[1]["emb_dim"]==16, "embedded nodes report dim 16") +check(alln[2]["emb_dim"]==0 and alln[2]["emb"]==[], "un-embedded node -> emb_dim 0, emb []") +# first emb value round-trips ~0.10 +check(abs(alln[0]["emb"][0]-0.10)<1e-3, f"emb[0] round-trips (~0.10, got {alln[0]['emb'][0]})") + +pg0=load("emb_pg0.json"); pg1=load("emb_pg1.json") +check(len(pg0)==1 and len(pg1)==1, "pagination: one node per page") +check(pg0[0]["id"]=="n-high" and pg1[0]["id"]=="n-mid", f"pages disjoint & ordered ({pg0[0]['id']},{pg1[0]['id']})") + +plain=load("plain.json") +check(len(plain)==3, "existing scan_nodes_json still returns 3") +check(all("emb" not in n for n in plain), "existing scan_nodes_json carries NO emb (behavior-neutral)") +sys.exit(rc) +PY +[ $? -ne 0 ] && fail=1 + +echo +echo "== latency (one 256-page over the 3-node copy) ==" +python3 - "$D" <<'PY' +import os +# timing was measured inside C not here; report emb payload size as a proxy +sz=os.path.getsize(os.path.join(os.sys.argv[1] if False else __import__('sys').argv[1],"emb_all.json")) +print(f" emb_all.json payload = {sz} bytes for 3 nodes") +PY + +echo +echo "== ASan+UBSan ==" +gcc -O1 -g -std=c11 -fsanitize=address,undefined -fno-sanitize-recover=undefined \ + -I "$INC" "$HERE/test_interoception_p0_emb.c" "$RT" "$ST" "$GEO" "$VIDX" \ + -lcurl -lm -o "$WORK/p0.san" 2>"$WORK/san_cc.log" || { echo "SAN COMPILE FAILED"; tail -20 "$WORK/san_cc.log"; fail=1; } +if [ -x "$WORK/p0.san" ]; then + export ASAN_OPTIONS=detect_leaks=0 + DS="$WORK/ds"; mkdir -p "$DS" + "$WORK/p0.san" "$DS" >/dev/null 2>"$WORK/san_run.log" + if grep -qiE 'runtime error|AddressSanitizer|Sanitizer|ERROR: ' "$WORK/san_run.log"; then + echo " FAIL: sanitizer findings:"; grep -iE 'runtime error|Sanitizer|ERROR' "$WORK/san_run.log" | head; fail=1 + else echo " ok: ASan+UBSan clean"; fi +fi + +echo +if [ "$fail" -eq 0 ]; then echo "====== P0 EMB-ENDPOINT GATE: PASS ======"; else echo "====== P0 EMB-ENDPOINT GATE: FAIL ======"; fi +rm -rf "$WORK" +exit $fail diff --git a/engram/test/test_interoception_p0_emb.c b/engram/test/test_interoception_p0_emb.c new file mode 100644 index 0000000..f7d64ab --- /dev/null +++ b/engram/test/test_interoception_p0_emb.c @@ -0,0 +1,79 @@ +/* test_interoception_p0_emb.c — M-INTEROCEPTION Priority 0. + * + * Verifies the new READ-ONLY builtin engram_scan_nodes_emb_json(limit,offset): + * - every emitted node carries emb_dim and an emb JSON array of that length, + * - nodes without an embedding emit emb_dim:0 / emb:[], + * - pagination (limit/offset) is honoured, + * - the count matches engram_node_count, + * - the EXISTING engram_scan_nodes_json path is byte-unchanged (no emb field), + * i.e. the addition is purely additive / behavior-neutral. + * + * Pure-C harness (no elc). We craft a snapshot with real emb vectors, load it + * (engram_load parses "emb" comma-lists into node->emb via eg_parse_emb), then + * dump via both scan paths. Assertions live in run_interoception_p0.sh. + */ +#include "el_runtime.h" +#include +#include +#include + +static el_val_t S(const char* s){ return EL_STR(s); } + +/* 16-d embedding as a comma list (>=8 required by eg_parse_emb). */ +static void emb_list(char* out, size_t cap, int dim, double base){ + size_t o = 0; + for (int i = 0; i < dim; i++){ + o += snprintf(out+o, cap-o, "%s%.4f", i?",":"", base + 0.01*i); + } +} + +int main(int argc, char** argv){ + if (argc < 2){ fprintf(stderr, "usage: %s \n", argv[0]); return 2; } + const char* dir = argv[1]; + char snap[1024]; snprintf(snap, sizeof snap, "%s/seed.json", dir); + + char e1[512], e2[512]; + emb_list(e1, sizeof e1, 16, 0.10); + emb_list(e2, sizeof e2, 16, 0.50); + + /* Two embedded nodes (distinct salience → deterministic sort order) and one + * un-embedded node. */ + FILE* f = fopen(snap, "w"); + if (!f){ perror("fopen"); return 2; } + fprintf(f, + "{\"nodes\":[" + "{\"id\":\"n-high\",\"content\":\"high salience embedded\",\"node_type\":\"Concept\"," + "\"label\":\"emb-high\",\"tier\":\"Semantic\",\"salience\":0.9,\"importance\":0.8," + "\"confidence\":1.0,\"created_at\":1000,\"emb\":\"%s\"}," + "{\"id\":\"n-mid\",\"content\":\"mid salience embedded\",\"node_type\":\"Concept\"," + "\"label\":\"emb-mid\",\"tier\":\"Semantic\",\"salience\":0.5,\"importance\":0.5," + "\"confidence\":1.0,\"created_at\":2000,\"emb\":\"%s\"}," + "{\"id\":\"n-low\",\"content\":\"low salience no embedding\",\"node_type\":\"Fact\"," + "\"label\":\"noemb-low\",\"tier\":\"Semantic\",\"salience\":0.1,\"importance\":0.2," + "\"confidence\":1.0,\"created_at\":3000}" + "],\"edges\":[]}", e1, e2); + fclose(f); + + if (!engram_load(S(snap))){ fprintf(stderr, "load failed\n"); return 2; } + long long nc = (long long)(int64_t)engram_node_count(); + printf("node_count=%lld\n", nc); + + /* full page */ + el_val_t all = engram_scan_nodes_emb_json((el_val_t)256, (el_val_t)0); + char p[1024]; + snprintf(p, sizeof p, "%s/emb_all.json", dir); + f = fopen(p, "w"); fputs(EL_CSTR(all), f); fclose(f); + + /* pagination: one node at offset 0 and one at offset 1 */ + el_val_t pg0 = engram_scan_nodes_emb_json((el_val_t)1, (el_val_t)0); + el_val_t pg1 = engram_scan_nodes_emb_json((el_val_t)1, (el_val_t)1); + snprintf(p, sizeof p, "%s/emb_pg0.json", dir); f = fopen(p, "w"); fputs(EL_CSTR(pg0), f); fclose(f); + snprintf(p, sizeof p, "%s/emb_pg1.json", dir); f = fopen(p, "w"); fputs(EL_CSTR(pg1), f); fclose(f); + + /* existing path — must be unchanged / carry NO emb */ + el_val_t plain = engram_scan_nodes_json((el_val_t)256, (el_val_t)0); + snprintf(p, sizeof p, "%s/plain.json", dir); f = fopen(p, "w"); fputs(EL_CSTR(plain), f); fclose(f); + + printf("wrote dumps to %s\n", dir); + return 0; +} diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index 0ef6bc3..1299a03 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -11936,6 +11936,58 @@ el_val_t engram_scan_nodes_by_type_json(el_val_t type_v, el_val_t limit, el_val_ return el_wrap_str(b.buf); } +/* engram_scan_nodes_emb_json — READ-ONLY, ADDITIVE (M-INTEROCEPTION P0). + * Paginated dump of nodes WITH their dense embedding vector, for the + * GET /api/embeddings + /api/graph/dump read routes. Purely additive: it does + * NOT touch engram_emit_node_json or the default include_emb=0 anywhere. It + * emits a compact, self-describing record per node — id, node_type, label, + * created_at, emb_dim, and emb as a JSON array of %.4g floats (length == + * emb_dim, so a consumer can verify the vector round-trips). Nodes without an + * embedding are emitted with emb_dim:0 and emb:[]. Same salience-sorted, + * transparent-layer-skipped pagination as engram_scan_nodes_json. + * Default limit is bounded (256) to keep one page's payload sane. */ +el_val_t engram_scan_nodes_emb_json(el_val_t limit, el_val_t offset) { + EngramStore* g = engram_get(); + int64_t lim = (int64_t)limit; if (lim <= 0) lim = 256; + int64_t off = (int64_t)offset; if (off < 0) off = 0; + JsonBuf b; jb_init(&b); + jb_putc(&b, '['); + if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } + int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t)); + if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } + int64_t live = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue; + idx[live++] = i; + } + engram_sort_indices_by_salience(idx, live, g->nodes); + int64_t end = off + lim; + if (end > live) end = live; + int first = 1; + char tmp[80]; + for (int64_t i = off; i < end; i++) { + const EngramNode* n = &g->nodes[idx[i]]; + if (!first) jb_putc(&b, ','); + first = 0; + jb_putc(&b, '{'); + jb_puts(&b, "\"id\":"); jb_emit_escaped(&b, n->id ? n->id : ""); + jb_puts(&b, ",\"node_type\":"); jb_emit_escaped(&b, n->node_type ? n->node_type : ""); + jb_puts(&b, ",\"label\":"); jb_emit_escaped(&b, n->label ? n->label : ""); + snprintf(tmp, sizeof(tmp), ",\"created_at\":%lld", (long long)n->created_at); jb_puts(&b, tmp); + int32_t dim = (n->emb && n->emb_dim > 0) ? n->emb_dim : 0; + snprintf(tmp, sizeof(tmp), ",\"emb_dim\":%d", dim); jb_puts(&b, tmp); + jb_puts(&b, ",\"emb\":["); + for (int32_t j = 0; j < dim; j++) { + snprintf(tmp, sizeof(tmp), "%s%.4g", j ? "," : "", (double)n->emb[j]); + jb_puts(&b, tmp); + } + jb_puts(&b, "]}"); + } + free(idx); + jb_putc(&b, ']'); + return el_wrap_str(b.buf); +} + el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction) { /* Re-implement here directly so we serialize without going through * the ElList path. Walks BFS to max_depth, emits {node, edge, hops} diff --git a/lang/runtime/el_runtime.h b/lang/runtime/el_runtime.h index 0380327..1b6b55b 100644 --- a/lang/runtime/el_runtime.h +++ b/lang/runtime/el_runtime.h @@ -621,6 +621,7 @@ el_val_t engram_get_node_by_label(el_val_t label); el_val_t engram_search_json(el_val_t query, el_val_t limit); el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset); el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset); +el_val_t engram_scan_nodes_emb_json(el_val_t limit, el_val_t offset); el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction); el_val_t engram_activate_json(el_val_t query, el_val_t depth); el_val_t engram_stats_json(void); diff --git a/lang/runtime/el_seed.c b/lang/runtime/el_seed.c index aa4305b..e9809e8 100644 --- a/lang/runtime/el_seed.c +++ b/lang/runtime/el_seed.c @@ -1086,6 +1086,10 @@ el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el return engram_scan_nodes_by_type_json(node_type, limit, offset); } +el_val_t __engram_scan_nodes_emb_json(el_val_t limit, el_val_t offset) { + return engram_scan_nodes_emb_json(limit, offset); +} + el_val_t __engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction) { return engram_neighbors_json(node_id, max_depth, direction); }