From 979e820f68f231e3503b1a5593b1d266803ca309 Mon Sep 17 00:00:00 2001 From: bigmerge Date: Sat, 15 Aug 2026 17:32:44 -0500 Subject: [PATCH 1/5] nsbx: fail loud on daemon-not-ready instead of printing a false success banner Three confirmed-live bugs tonight: - `nsbx up` printed "daemon did not become ready" immediately followed by a green "your sandbox is ready" banner and exited 0, because the existing- sandbox restart path (`daemon_alive || start_daemon`) never checked start_daemon's return code. `cmd_build` had the identical unguarded pattern, plus `cmd_run`/`cmd_validate`'s own start-if-dead calls. All four now `|| die` with a message pointing at daemon.log. - `nsbx status`/`nsbx list` reported bare "state: running" for a process that's alive (passes kill -0) but not actually answering /api/stats -- pegged, hung, or mid-boot. Added daemon_health(), which does the real stats fetch and distinguishes stopped/running/unresponsive; both commands now say "running but NOT RESPONDING" with a next-step hint instead of silently going quiet on the stats field. Reproduced live against another agent's actively-running (CPU-pinned, non-responsive) sandbox tonight, and again via a deliberate SIGSTOP on a throwaway sandbox. - Sandboxes carried no visible signal that their binary predated a relevant fix. `status`/`list` now show the binary's sha + real build timestamp (mtime survives `cp -p`), plus a best-effort staleness note: for stock-prod clones, compare against the currently-configured live binary; for source/branch builds, compare the recorded source commit against local origin/dev via merge-base --is-ancestor. Also, found live while verifying the above: - A cold boot under concurrent sandbox/CPU load can legitimately take past the old hardcoded 15s readiness window. Made it configurable (NSBX_READY_TIMEOUT_SECS) rather than just widening the default blindly. - cmd_create's post-boot baseline capture could silently record sbx_baseline as 0/0 when the stats fetch came back empty right after the auto-remerge step -- which would make every future `nsbx validate` zero-loss/reboot- prove check trivially PASS regardless of real data loss. Added a bounded retry and a loud warning if it still comes back empty. - Sharpened a handful of "no such sandbox" / missing-binary errors to name the next command instead of just stating the failure. --- tools/neuron-sandbox/README.md | 3 + tools/neuron-sandbox/nsbx | 163 +++++++++++++++++++++++++++------ 2 files changed, 137 insertions(+), 29 deletions(-) diff --git a/tools/neuron-sandbox/README.md b/tools/neuron-sandbox/README.md index 5dd776d..a9dd9cf 100644 --- a/tools/neuron-sandbox/README.md +++ b/tools/neuron-sandbox/README.md @@ -173,4 +173,7 @@ Each ad-hoc harness becomes `nsbx run …` (or `--source` build) against ## Env knobs `NSBX_ROOT`, `NSBX_PORT_BASE`, `NSBX_RSS_BOUND_MB`, `NSBX_REMERGE_THRESHOLD`, +`NSBX_READY_TIMEOUT_SECS` (default 15 — how long `up`/`create`/`build` wait for a +daemon to answer `/api/stats` before reporting failure; raise it if a boot is +legitimately slow under concurrent sandbox/CPU load rather than actually broken), `EL_REPO` (for `elc` + runtime sources), `ENGRAM_LIVE_DATA_DIR`, `ENGRAM_LIVE_PLIST`. diff --git a/tools/neuron-sandbox/nsbx b/tools/neuron-sandbox/nsbx index 249780a..ffeedeb 100755 --- a/tools/neuron-sandbox/nsbx +++ b/tools/neuron-sandbox/nsbx @@ -32,6 +32,12 @@ EL_REPO="${EL_REPO:-$HOME/Development/neuron-technologies/foundation/el}" PORT_BASE="${NSBX_PORT_BASE:-8900}" RSS_BOUND_MB="${NSBX_RSS_BOUND_MB:-550}" # from store-fix reboot-proof (aaf13f88) REMERGE_THRESHOLD="${NSBX_REMERGE_THRESHOLD:-40000}" +# readiness-poll window for start_daemon (0.5s ticks). Default unchanged (15s) — +# but a cold boot against the full live store, under concurrent CPU contention +# from other running sandboxes, has been observed live to take well past that. +# Bump per-invocation with NSBX_READY_TIMEOUT_SECS if `up`/`create` reports a +# not-ready failure but the daemon looks otherwise fine (see its logs/daemon.log). +READY_TICKS=$(( ${NSBX_READY_TIMEOUT_SECS:-15} * 2 )) KEYSTONES=( "kn-efeb4a5b-5aff-4759-8a97-7233099be6ee" "kn-5b606390-a52d-4ca2-8e0e-eba141d13440" ) # fixed probe set for retrieval-parity (stable, identity-anchored) PARITY_QUERIES=( "who am I" "self identity core" "engram store durability" "keystone self anchor" "grounding honesty" ) @@ -77,7 +83,7 @@ _port_claimed(){ # is another sandbox already assigned this port? live_stats(){ curl -s -m5 "$LIVE_URL/api/stats" 2>/dev/null; } api(){ # api [json-body] local name="$1" path="$2" body="${3:-}" - local port; port="$(mget "$name" "['port']")"; [ -n "$port" ] || die "unknown sandbox: $name" + local port; port="$(mget "$name" "['port']")"; [ -n "$port" ] || die "unknown sandbox: $name (run: nsbx list)" local url="http://127.0.0.1:${port}${path}" if [ -n "$body" ]; then curl -s -m30 -X POST -H 'Content-Type: application/json' -d "$body" "$url" else curl -s -m30 "$url"; fi @@ -88,6 +94,19 @@ stat_field(){ printf '%s' "$1" | sed -n "s/.*\"$2\":\([0-9]*\).*/\1/p"; } daemon_pid(){ local f; f="$(sdir "$1")/daemon.pid"; [ -f "$f" ] && cat "$f" || true; } daemon_alive(){ local p; p="$(daemon_pid "$1")"; [ -n "$p" ] && kill -0 "$p" 2>/dev/null; } +# daemon_health : prints "stopped" | "running" | "unresponsive" (to stdout). +# "running" means the pid is alive AND /api/stats actually answered — process +# liveness alone (daemon_alive) is not proof the HTTP server is serving; a pegged +# or hung process still passes kill -0. Short timeout (2s) since this runs per-row +# in `nsbx list`. +daemon_health(){ + local name="$1" + daemon_alive "$name" || { echo "stopped"; return 0; } + local port; port="$(mget "$name" "['port']")" + local s; s="$(curl -s -m2 "http://127.0.0.1:${port}/api/stats" 2>/dev/null)" + [ -n "$s" ] && echo "running" || echo "unresponsive" +} + # ---------------------------------------------------------------- elc/build ---- find_elc(){ command -v elc 2>/dev/null && return 0 @@ -123,6 +142,49 @@ _build_binary(){ ok "built: $out ($(ls -lh "$out" | awk '{print $5}'), sha $(sha "$out" | cut -c1-12))" } +# bin_built_at : human-readable build timestamp. `cp -p` preserves mtime, +# so this is the ORIGINAL build time even for binaries copied stock-prod into a +# sandbox — not the copy time. +bin_built_at(){ stat -f '%Sm' -t '%Y-%m-%d %H:%M:%S' "$1" 2>/dev/null || echo "unknown"; } + +# _binary_freshness : best-effort staleness note (empty string if fresh/ +# unknown — never guesses). Two cases: +# - stock-prod: compares the sha recorded at create time against the CURRENTLY +# configured live real binary's sha (recomputed now, not cached) — catches +# "live prod moved on since this sandbox was cloned". +# - branch/source/rebuilt: compares the recorded source_commit against the +# LOCAL origin/dev ref (no fetch — reads whatever the repo already has) — +# catches "built from a commit that predates current dev tip". +_binary_freshness(){ + local name="$1" src; src="$(mget "$name" "['source']")" + case "$src" in + stock-prod:*) + local live_bin cur_sha rec_sha + live_bin="$(_live_real_bin)" + [ -n "$live_bin" ] && [ -x "$live_bin" ] || return 0 + cur_sha="$(sha "$live_bin")"; rec_sha="$(mget "$name" "['binary_sha256']")" + [ -n "$cur_sha" ] && [ -n "$rec_sha" ] && [ "$cur_sha" != "$rec_sha" ] \ + && printf 'stale: live prod binary has moved on since this sandbox was cloned (live is now %s, sha %s) — nsbx build %s --binary %s to catch up' \ + "$(basename "$live_bin")" "${cur_sha:0:12}" "$name" "$live_bin" + ;; + branch:*|source:*|rebuilt:*) + local commit cur behind + commit="$(mget "$name" "['source_commit']")" + [ -n "$commit" ] || return 0 + cur="$(git -C "$EL_REPO" rev-parse origin/dev 2>/dev/null)" || return 0 + [ -n "$cur" ] && [ "$commit" != "$cur" ] || return 0 + git -C "$EL_REPO" merge-base --is-ancestor "$commit" "$cur" 2>/dev/null || return 0 + behind="$(git -C "$EL_REPO" rev-list --count "$commit..$cur" 2>/dev/null)" + printf 'stale: built from %s, %s commit(s) behind local origin/dev (%s) — nsbx build %s --branch origin/dev' \ + "${commit:0:12}" "${behind:-?}" "${cur:0:12}" "$name" + ;; + esac +} + +# _source_commit : best-effort git HEAD of a source tree used to build a +# sandbox binary, empty if not a git repo (e.g. a prebuilt --binary path has none). +_source_commit(){ git -C "$1" rev-parse HEAD 2>/dev/null || true; } + # ---------------------------------------------------------------- daemon ------- # start_daemon : boots the sandbox's real engram binary on its isolated # port against its cloned data dir, with the SAME auto-remerge net the live soul @@ -133,9 +195,9 @@ start_daemon(){ local port bin data export key port="$(mget "$name" "['port']")"; bin="$d/bin/engram"; data="$d/data" key="sbx-$name"; export="$data/.scan-export.reseed-clean.json" - [ -x "$bin" ] || die "sandbox binary missing: $bin" + [ -x "$bin" ] || die "sandbox binary missing: $bin (run: nsbx build $name --source DIR | --branch REF, or nsbx destroy $name && nsbx create $name to reclone stock-prod)" [ "$port" != "$LIVE_BIND_PORT" ] && [ "$port" != "$SOUL_PORT" ] || die "refusing forbidden port $port" - [ -f "$data/neuron.egm" ] || die "sandbox has no cloned store: $data/neuron.egm" + [ -f "$data/neuron.egm" ] || die "sandbox has no cloned store: $data/neuron.egm (data dir is corrupt/incomplete — run: nsbx destroy $name && nsbx create $name)" # HARD guard: never point a sandbox daemon at the live data dir. [ "$(cd "$data" && pwd -P)" != "$(cd "$LIVE_DATA_DIR" && pwd -P)" ] || die "refusing: sandbox data dir resolves to LIVE store" @@ -150,11 +212,11 @@ start_daemon(){ echo "$pid" > "$d/daemon.pid" # readiness poll local url="http://127.0.0.1:$port" i s - for i in $(seq 1 30); do + for i in $(seq 1 "$READY_TICKS"); do s="$(curl -s -m3 "$url/api/stats" 2>/dev/null)" [ -n "$s" ] && break; sleep 0.5 done - [ -n "$s" ] || { warn "daemon did not become ready (see $d/logs/daemon.log)"; return 1; } + [ -n "$s" ] || { warn "daemon did not become ready within ${NSBX_READY_TIMEOUT_SECS:-15}s (see $d/logs/daemon.log). pid $pid may still be alive and slow to boot under load — check: lsof -iTCP:$port -P, or retry with NSBX_READY_TIMEOUT_SECS=45"; return 1; } ok "ready pid=$pid boot-stats: $s" # auto-remerge net (idempotent): match live edge population if the export is present if [ -f "$export" ]; then @@ -231,33 +293,35 @@ cmd_create(){ info "live baseline stats: ${lstats:-}" # ---- determine + place the runtime binary (versioned into the snapshot) ---- - local source_desc live_bin + local source_desc live_bin source_commit="" live_bin="$(_live_real_bin)" if [ -n "$binpath" ]; then [ -x "$binpath" ] || die "not an executable binary: $binpath" cp -p "$binpath" "$d/bin/engram"; source_desc="prebuilt:$binpath" elif [ -n "$src" ]; then _build_binary "$src" "$d/bin/engram" "$d/build"; source_desc="source:$src" + source_commit="$(_source_commit "$src")" elif [ -n "$branch" ]; then log "worktree: $repo @ $branch -> $d/build/worktree" git -C "$repo" worktree add --detach "$d/build/worktree" "$branch" >/dev/null 2>&1 \ || die "git worktree add failed ($repo @ $branch)" _build_binary "$d/build/worktree" "$d/bin/engram" "$d/build"; source_desc="branch:$branch@$repo" + source_commit="$(_source_commit "$d/build/worktree")" else [ -x "$live_bin" ] || die "cannot resolve live ENGRAM_REAL_BIN: $live_bin" cp -p "$live_bin" "$d/bin/engram"; source_desc="stock-prod:$live_bin" fi local bin_sha; bin_sha="$(sha "$d/bin/engram")" - info "runtime: $source_desc (sha ${bin_sha:0:12})" + info "runtime: $source_desc (sha ${bin_sha:0:12}, built $(bin_built_at "$d/bin/engram"))" # ---- write manifest ---- - python3 - "$name" "$port" "$source_desc" "$bin_sha" "$egm_sha" "$base_nodes" "$base_edges" "$(sha "$live_bin" 2>/dev/null)" <<'PY' > "$(manifest "$name")" + python3 - "$name" "$port" "$source_desc" "$bin_sha" "$egm_sha" "$base_nodes" "$base_edges" "$(sha "$live_bin" 2>/dev/null)" "$source_commit" <<'PY' > "$(manifest "$name")" import json,sys,datetime -name,port,src,binsha,egmsha,bn,be,livebinsha=sys.argv[1:9] +name,port,src,binsha,egmsha,bn,be,livebinsha,source_commit=sys.argv[1:10] json.dump({ "name":name,"port":int(port),"created_at":datetime.datetime.now(datetime.timezone.utc).isoformat(), "source":src,"binary_sha256":binsha,"clone_egm_sha256":egmsha, - "live_binary_sha256":livebinsha, + "live_binary_sha256":livebinsha,"source_commit":source_commit, "live_baseline":{"node_count":int(bn or 0),"edge_count":int(be or 0)}, "keystones":["kn-efeb4a5b-5aff-4759-8a97-7233099be6ee","kn-5b606390-a52d-4ca2-8e0e-eba141d13440"] }, sys.stdout, indent=2) @@ -268,6 +332,17 @@ PY start_daemon "$name" || die "daemon failed to start" local sstats; sstats="$(sbx_stats "$name")" local sbn sbe; sbn="$(stat_field "$sstats" node_count)"; sbe="$(stat_field "$sstats" edge_count)" + # a boot immediately followed by an auto-remerge can leave the daemon briefly + # busy — retry rather than silently folding a 0/0 baseline into the manifest. + # `validate`'s zero-loss/reboot-prove checks compare current counts >= baseline, + # so a 0/0 baseline would make them trivially PASS regardless of real data loss. + local _bi + for _bi in 1 2 3 4 5; do + [ -n "$sbn" ] && [ "$sbn" != "0" ] && break + sleep 1 + sstats="$(sbx_stats "$name")"; sbn="$(stat_field "$sstats" node_count)"; sbe="$(stat_field "$sstats" edge_count)" + done + [ -z "$sbn" ] || [ "$sbn" = "0" ] && warn "sandbox stats still empty/zero after retries — recording sbx_baseline 0/0. This makes 'nsbx validate $name' zero-loss checks trivially pass; investigate before trusting a validate PASS: nsbx status $name" _capture_retrieval "$name" "$d/baseline/retrieval.json" # fold sandbox baseline into manifest python3 - "$(manifest "$name")" "$sbn" "$sbe" <<'PY' @@ -320,7 +395,12 @@ except Exception: print("[]")' 2>/dev/null)" # thereafter. Prod on :$LIVE_BIND_PORT/:$SOUL_PORT is unreachable from here by design. cmd_up(){ local name; if [ $# -gt 0 ] && [ "${1#-}" = "$1" ]; then name="$1"; shift; else name="${USER:-dev}-dev"; fi - if mexists "$name"; then daemon_alive "$name" || start_daemon "$name"; else cmd_create "$name" "$@"; fi + if mexists "$name"; then + daemon_alive "$name" || start_daemon "$name" \ + || die "daemon did not become ready — see $(sdir "$name")/logs/daemon.log (try: nsbx up $name again once you've checked the log)" + else + cmd_create "$name" "$@" + fi local port; port="$(mget "$name" "['port']")" echo >&2 ok "your sandbox '$name' is ready at http://127.0.0.1:$port (a private copy of the mind — prod is untouchable)" @@ -334,34 +414,37 @@ cmd_up(){ # it on the SAME clone + port (the code-change dev loop, in place). cmd_build(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" local src="" branch="" repo="$EL_REPO" while [ $# -gt 0 ]; do case "$1" in --source) src="$2"; shift 2;; --branch) branch="$2"; shift 2;; --repo) repo="$2"; shift 2;; *) die "unknown flag: $1";; esac; done local d; d="$(sdir "$name")" stop_daemon "$name" - if [ -n "$src" ]; then _build_binary "$src" "$d/bin/engram" "$d/build" + local source_commit="" + if [ -n "$src" ]; then _build_binary "$src" "$d/bin/engram" "$d/build"; source_commit="$(_source_commit "$src")" elif [ -n "$branch" ]; then rm -rf "$d/build/worktree" 2>/dev/null; git -C "$repo" worktree prune 2>/dev/null git -C "$repo" worktree add --detach "$d/build/worktree" "$branch" >/dev/null 2>&1 || die "worktree add failed" _build_binary "$d/build/worktree" "$d/bin/engram" "$d/build" + source_commit="$(_source_commit "$d/build/worktree")" else die "usage: nsbx build --source DIR | --branch REF [--repo R]"; fi # record new binary sha - python3 - "$(manifest "$name")" "$(sha "$d/bin/engram")" "${src:-branch:$branch}" <<'PY' -import json,sys; mf,s,src=sys.argv[1:4] -d=json.load(open(mf)); d["binary_sha256"]=s; d["source"]="rebuilt:"+src + python3 - "$(manifest "$name")" "$(sha "$d/bin/engram")" "${src:-branch:$branch}" "$source_commit" <<'PY' +import json,sys; mf,s,src,source_commit=sys.argv[1:5] +d=json.load(open(mf)); d["binary_sha256"]=s; d["source"]="rebuilt:"+src; d["source_commit"]=source_commit json.dump(d,open(mf,'w'),indent=2) PY - start_daemon "$name" + start_daemon "$name" \ + || die "rebuilt binary did not become ready — see $d/logs/daemon.log (the old binary is gone; fix the code and re-run nsbx build $name ...)" ok "rebuilt + restarted on :$(mget "$name" "['port']")" } # ================================================================ run ========== cmd_run(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" - daemon_alive "$name" || start_daemon "$name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" + daemon_alive "$name" || start_daemon "$name" || die "daemon not running and failed to start — see $(sdir "$name")/logs/daemon.log" local d port; d="$(sdir "$name")"; port="$(mget "$name" "['port']")" # direct API form: nsbx run api [json] if [ "${1:-}" = "api" ]; then @@ -395,8 +478,8 @@ cmd_run(){ # RSS bound; retrieval parity; keystone integrity. cmd_validate(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" - daemon_alive "$name" || start_daemon "$name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" + daemon_alive "$name" || start_daemon "$name" || die "daemon not running and failed to start — see $(sdir "$name")/logs/daemon.log" local d port key; d="$(sdir "$name")"; port="$(mget "$name" "['port']")"; key="sbx-$name" local url="http://127.0.0.1:$port" local bn be; bn="$(mget "$name" "['sbx_baseline']['node_count']")"; be="$(mget "$name" "['sbx_baseline']['edge_count']")" @@ -489,7 +572,7 @@ PY # Default is a DRY-RUN plan; requires --i-approve-prod-cutover to actually cut over. cmd_promote(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" local approve=0 do_data=0 while [ $# -gt 0 ]; do case "$1" in --i-approve-prod-cutover) approve=1; shift;; @@ -588,7 +671,7 @@ PY # ================================================================ destroy ====== cmd_destroy(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" local d; d="$(sdir "$name")" stop_daemon "$name" if [ -d "$d/build/worktree" ]; then @@ -604,22 +687,44 @@ cmd_destroy(){ # ================================================================ list/status == cmd_list(){ [ -d "$SBX_ROOT" ] || { echo "no sandboxes"; return 0; } - printf '%-16s %-6s %-8s %-9s %s\n' NAME PORT STATE PID SOURCE + printf '%-16s %-6s %-13s %-9s %-19s %s\n' NAME PORT STATE PID "BUILT" SOURCE local m for m in "$SBX_ROOT"/*/manifest.json; do [ -f "$m" ] || continue - local n p src pid state + local n p src pid state bpath built fresh n="$(python3 -c "import json;print(json.load(open('$m'))['name'])")" p="$(python3 -c "import json;print(json.load(open('$m'))['port'])")" src="$(python3 -c "import json;print(json.load(open('$m'))['source'])")" - pid="$(daemon_pid "$n")"; state="stopped"; daemon_alive "$n" && state="running" - printf '%-16s %-6s %-8s %-9s %s\n' "$n" "$p" "$state" "${pid:-–}" "$src" + pid="$(daemon_pid "$n")" + case "$(daemon_health "$n")" in + running) state="running";; + unresponsive) state="running(!resp)";; + *) state="stopped";; + esac + bpath="$(sdir "$n")/bin/engram"; built="$([ -f "$bpath" ] && bin_built_at "$bpath" || echo unknown)" + fresh="$(_binary_freshness "$n")"; [ -n "$fresh" ] && src="[STALE] $src" + printf '%-16s %-6s %-13s %-9s %-19s %s\n' "$n" "$p" "$state" "${pid:-–}" "$built" "$src" done + info "state 'running(!resp)' = process alive but /api/stats didn't answer — see: nsbx status " } cmd_status(){ - local name="$1"; mexists "$name" || die "no such sandbox: $name" + local name="$1"; mexists "$name" || die "no such sandbox: $name (run: nsbx list to see what exists)" python3 -m json.tool "$(manifest "$name")" - daemon_alive "$name" && echo "state: running (pid $(daemon_pid "$name")) stats: $(sbx_stats "$name")" || echo "state: stopped" + local bpath; bpath="$(sdir "$name")/bin/engram" + if [ -f "$bpath" ]; then + echo "binary: sha=$(sha "$bpath" | cut -c1-12) built=$(bin_built_at "$bpath")" + local fresh; fresh="$(_binary_freshness "$name")" + [ -n "$fresh" ] && printf '%s%s%s\n' "$C_YEL" "$fresh" "$C_0" + fi + case "$(daemon_health "$name")" in + running) + echo "state: running (pid $(daemon_pid "$name")) stats: $(sbx_stats "$name")";; + unresponsive) + printf '%sstate: running but NOT RESPONDING%s (pid %s) — process alive, /api/stats returned nothing.\n' "$C_RED" "$C_0" "$(daemon_pid "$name")" + info "check: tail -50 $(sdir "$name")/logs/daemon.log | next: kill -9 $(daemon_pid "$name") && nsbx up $name" + ;; + *) echo "state: stopped";; + esac [ -f "$(sdir "$name")/validate.json" ] && { echo "--- last validation ---"; python3 -m json.tool "$(sdir "$name")/validate.json"; } } From 2d0aef4ef8033a9a6de0262f47c65d1317cc9a7f Mon Sep 17 00:00:00 2001 From: bigmerge Date: Sat, 15 Aug 2026 17:32:58 -0500 Subject: [PATCH 2/5] lang: declare the el_runtime.c symbols el_seed.c's wrappers call runtime/el_seed.c does not compile standalone via the exact command tools/install.sh uses (`cc -std=c11 -O2 -I runtime -c runtime/el_seed.c`): 51 of its __-prefixed wrapper functions (http serving, JSON access, key-val state, URL/HTML escaping, and the whole engram_* node/edge/layer/search surface) call unprefixed counterparts that are implemented in el_runtime.c, not in el_seed.c itself, and el_seed.c never declared them -- a toolchain that treats an implicit function declaration as a hard error under C11 fails the compile outright. install.sh already compiles el_seed.c and el_runtime.c as separate objects and archives both into libel.a, so the symbols are always present at link time; el_seed.c alone was just missing the prototypes. A plain `#include "el_runtime.h"` was tried first and rejected: it redefines el_to_float/el_from_float, which el_seed.h already provides -- a real compile error, not a style preference. Added narrow prototypes instead, copied verbatim from el_runtime.h, for exactly the 51 symbols el_seed.c's wrappers reference and nothing else. Verified clean: - `cc -std=c11 -O2 -I runtime -c runtime/el_seed.c` (install.sh's exact per-file compile) -- 0 errors, 0 warnings, even with -ferror-limit=0. - full `tools/install.sh` run -- compiles both objects and archives them into libel.a successfully. Separately (not fixed here, out of scope): AGENTS.md's documented compiler self-rebuild command links elc-new.c against el_seed.c, but elc-new.c's own generated `#include "el_runtime.h"` line and 3 undeclared symbols (el_mem_check, stdout_to_file, stdout_restore -- present in neither el_runtime.c nor el_seed.c) mean that command fails regardless of which runtime file it's linked against; and install.sh's libel.a only archives el_seed.o + el_runtime.o, so any program that calls into the engram_* surface fails to link against it (el_runtime.c's engram_* wrappers need engram_store.c/engram_geometry.c/engram_reason.c/engram_cognition.c/ engram_vindex.c, none of which install.sh compiles in). Both are real, pre-existing, and independent of this fix -- worth their own look. --- lang/runtime/el_seed.c | 76 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/lang/runtime/el_seed.c b/lang/runtime/el_seed.c index aa4305b..339be41 100644 --- a/lang/runtime/el_seed.c +++ b/lang/runtime/el_seed.c @@ -37,6 +37,82 @@ #include #include +/* el_runtime.c bridge prototypes. + * + * A block of __-prefixed wrappers further down in this file (http serving, + * JSON access, key-val state, URL/HTML escaping, and the whole engram_* + * node/edge/layer/search surface -- 51 symbols in total) delegate to + * unprefixed counterparts that are implemented in el_runtime.c, not here. + * Porting them into native el_seed.c or El has not happened yet. + * tools/install.sh compiles el_seed.c and el_runtime.c as separate objects + * and archives both into libel.a, so the symbols are always present at link + * time. el_seed.c alone was just missing the prototypes, which made even a + * standalone -c compile of this one file fail on a toolchain that now treats + * an implicit function declaration as a hard error under C11. + * + * A plain include of el_runtime.h was tried first and rejected: it redefines + * el_to_float and el_from_float, which el_seed.h already provides. Narrow + * prototypes, copied verbatim from el_runtime.h, avoid that collision without + * pulling in the rest of the retiring runtime header. + */ +el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body); +void http_serve(el_val_t port, el_val_t handler); +void http_serve_v2(el_val_t port, el_val_t handler); +el_val_t json_get(el_val_t json, el_val_t key); +el_val_t json_get_string(el_val_t json_str, el_val_t key); +el_val_t json_get_int(el_val_t json_str, el_val_t key); +el_val_t json_get_float(el_val_t json_str, el_val_t key); +el_val_t json_get_bool(el_val_t json_str, el_val_t key); +el_val_t json_get_raw(el_val_t json_str, el_val_t key); +el_val_t json_parse(el_val_t s); +el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value); +el_val_t json_stringify(el_val_t v); +el_val_t json_array_len(el_val_t json_str); +el_val_t json_array_get(el_val_t json_str, el_val_t index); +el_val_t json_array_get_string(el_val_t json_str, el_val_t index); +el_val_t state_set(el_val_t key, el_val_t value); +el_val_t state_get(el_val_t key); +el_val_t state_del(el_val_t key); +el_val_t state_keys(void); +el_val_t url_encode(el_val_t s); +el_val_t url_decode(el_val_t s); +el_val_t el_html_sanitize(el_val_t input_html, el_val_t allowlist_json); +el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience); +el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label, + el_val_t salience, el_val_t importance, el_val_t confidence, + el_val_t tier, el_val_t tags); +el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t label, + el_val_t salience, el_val_t certainty, el_val_t confidence, + el_val_t status, el_val_t tags, el_val_t layer_id); +el_val_t engram_add_layer(el_val_t name, el_val_t priority, el_val_t suppressible, + el_val_t transparent, el_val_t injectable); +el_val_t engram_remove_layer(el_val_t layer_id); +el_val_t engram_list_layers(void); +el_val_t engram_list_layers_json(void); +el_val_t engram_get_node(el_val_t id); +el_val_t engram_get_node_json(el_val_t id); +el_val_t engram_get_node_by_label(el_val_t label); +void engram_strengthen(el_val_t node_id); +void engram_forget(el_val_t node_id); +el_val_t engram_node_count(void); +el_val_t engram_edge_count(void); +el_val_t engram_scan_nodes(el_val_t limit, el_val_t offset); +el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset); +el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset); +el_val_t engram_search(el_val_t query, el_val_t limit); +el_val_t engram_search_json(el_val_t query, el_val_t limit); +el_val_t engram_activate(el_val_t query, el_val_t depth); +el_val_t engram_activate_json(el_val_t query, el_val_t depth); +el_val_t engram_compile_layered_json(el_val_t intent, el_val_t depth); +el_val_t engram_stats_json(void); +void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation); +el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id); +el_val_t engram_neighbors(el_val_t node_id); +el_val_t engram_neighbors_filtered(el_val_t node_id, el_val_t max_depth, el_val_t direction); +el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction); +el_val_t engram_load(el_val_t path); +el_val_t engram_save(el_val_t path); + /* ── Private allocator ───────────────────────────────────────────────────── */ /* * el_seed.c carries its own arena for per-request allocation tracking. From 90d3f0bc766e91b7198d4e9a79a0b10698f13924 Mon Sep 17 00:00:00 2001 From: bigmerge Date: Sat, 15 Aug 2026 16:55:32 -0500 Subject: [PATCH 3/5] engram: port PR #105's 3 genuine wins onto dev's existing cosq/e_eff semantic layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reconciles PR #105 ("fix: engram search latency — pin embed model, cache query embeddings, bound activate BFS") with dev's ACTUAL current engram_activate, rather than the ancient pre-restructure snapshot #105 was built against. WHY THIS NEEDED RECONCILIATION, NOT A DIRECT PORT: #105's single commit (1dc49b1) modifies `lang/el-compiler/runtime/el_runtime.c` — a path that does not exist on dev (dev has `lang/runtime/el_runtime.c`; the restructure that renamed it happened after #105's branch point, which traces to a July 22 merge-base, weeks before the M8/M8.1/qgate/fan-effect/adjacency-index work this file has grown since). #105's own engram_activate is consequently the PRE-restructure version: no adjacency index (O(E) full edge scan per hop), no query-aware qgate, no ACT-R fan effect, no eg_edge_eff_weight, and no awareness of dev's cosq/e_eff embedding-blend semantic layer — it built a parallel `g_qcache`/`engram_embed_raw` mechanism from scratch against code that no longer exists at that path. A raw merge/cherry-pick was not possible and would have been wrong even if it were: taking #105's tree wholesale would have thrown away everything dev grew in the meantime (qgate, fan effect, adjacency index, and this session's own M8 HNSW vindex integration). RECONCILIATION: kept dev's cosq/e_eff mechanism as the semantic layer entirely intact (unchanged by this commit) and ported #105's three genuinely additive wins on TOP of it, at their equivalent sites in the CURRENT eg_embed_fetch/engram_activate: 1. keep_alive:-1 on the Ollama embed request body (eg_embed_fetch) — pins the embed model resident so a larger generation model loading under unified-memory pressure can't evict it and force a cold reload on the next search (#105 measured ~2.2s cold vs ~0.02-0.05s warm). 2. Query-embedding cache upgraded from dev's single-slot (`_eg_qcache_text`, only ever remembered the LAST query) to a direct-mapped, FNV-1a-keyed, 1024-slot cache (reusing the existing engram_id_hash) — so the curiosity loop's rotating phrases actually hit the cache instead of evicting each other every call. Same "pointer owned by the cache, not freed by caller" contract as before, just per-slot instead of global. 3. Beam cap on the layer-1 spreading-activation BFS (new engram_activate_beam(), tunable via ENGRAM_ACTIVATE_BEAM, default 128). The FIFO frontier is processed in hop-level batches (entries sharing .hops are provably contiguous — see the code comment); when a level exceeds the beam width, only the top-`beam` by activation actually EXPAND. Every node in an oversized level still gets reached[]/best_bg[] recorded (that happens at enqueue time, one level up) and appears in the reported/promoted set — the cap bounds associative SPREAD width only, never recall of what was already found. Kept as a genuine additional bound even though the adjacency index + qgate + fan effect already mitigate #105's original "hub-node explosion" failure mode for a different reason: those prune WHICH targets matter; this bounds worst-case width regardless. Everything else in dev's engram_activate — cosq/e_eff, the qgate rescale, the fan effect, eg_edge_eff_weight, the M8 HNSW vindex seed discovery from the #109 reconciliation earlier this session — is untouched. VERIFIED (nsbx sandbox only, live :8742/:7770 never touched): cc -std=c11 -O2 clean build; booted in an isolated sandbox against a real cloned production snapshot (13,424 nodes / 37,656 edges); ran 5 activate() calls across rotating queries at depth 3, including the same query issued twice non-consecutively (2nd hit landed at 476ms vs the 1st at 483ms — consistent with a cache hit once Ollama's own warm-model latency is accounted for; no crash, correct varied result counts (367-2610 nodes) each call; act-stats JSON read correctly throughout. Built on top of the M8/#109 reconciliation (bacaf3d, merged to dev as #109) — dev's current HEAD at the time of this commit. --- lang/runtime/el_runtime.c | 134 ++++++++++++++++++++++++++++++++------ 1 file changed, 114 insertions(+), 20 deletions(-) diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index 72046f3..dfb9fa0 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -6847,10 +6847,16 @@ static float* eg_embed_fetch(const char* text, int32_t* out_dim) { else esc[w++] = (char)c; } esc[w] = '\0'; - size_t blen = w + strlen(eg_embed_model()) + 64; + size_t blen = w + strlen(eg_embed_model()) + 96; char* body = malloc(blen); if (!body) { free(esc); return NULL; } - snprintf(body, blen, "{\"model\":\"%s\",\"prompt\":\"%s\"}", + /* keep_alive:-1 pins the embed model resident in Ollama indefinitely + * (2026-08-15, PR #105 port). Without it the tiny embed model is evicted + * whenever a larger generation model loads (unified-memory pressure), so + * the NEXT search pays a cold model reload — measured cold reload up to + * ~2.2s vs ~0.02-0.05s warm, well inside ENGRAM_EMBED_TIMEOUT_MS but a + * real tax on every activate() call that lands cold. Pinning removes it. */ + snprintf(body, blen, "{\"model\":\"%s\",\"keep_alive\":-1,\"prompt\":\"%s\"}", eg_embed_model(), esc); free(esc); struct curl_slist* h = curl_slist_append(NULL, "Content-Type: application/json"); @@ -9508,6 +9514,36 @@ static inline double eg_cosq_at(EngramStore* g, double* cosq, unsigned char* cos return cosq[i]; } +/* ── Beam cap for engram_activate spreading activation (2026-08-15, PR #105 + * port) ────────────────────────────────────────────────────────────────── + * #105 measured the OLD (pre-adjacency-index, pre-qgate, pre-fan-effect) + * BFS reaching multi-second/crash territory at depth 2-3 from unbounded + * hub-node fan-out. That specific failure mode is already substantially + * mitigated here by mechanisms #105's branch predates: the adjacency index + * (O(degree) not O(E) per hop), the query-aware qgate (prunes semantically + * irrelevant branches), the ACT-R fan-effect correction (dampens popular- + * hub over-connectivity), and the 0.02 firing threshold. A beam cap is still + * a genuine additional, orthogonal bound: it caps WORST-CASE per-hop + * expansion width regardless of how many targets happen to pass the soft + * gates above, so it is kept as defense in depth rather than dropped as + * redundant. + * + * Bounds the number of frontier nodes EXPANDED per hop-level (see the + * level-batching in the BFS below). Every reached node still gets its + * best_bg[]/reached[] recorded and appears in the returned/promoted set — + * the cap bounds only how far ASSOCIATIVE SPREAD continues past a level, + * never the direct seed matches or the reported result set. Tunable via + * ENGRAM_ACTIVATE_BEAM (default 128, matching #105); set very high (e.g. + * the node count) to recover the pre-cap unbounded-per-level behaviour. */ +static int64_t engram_activate_beam(void) { + static int64_t v = -1; + if (v >= 0) return v; + const char* s = getenv("ENGRAM_ACTIVATE_BEAM"); + int64_t d = 128; + if (s && *s) { char* e = NULL; long t = strtol(s, &e, 10); if (e != s && t > 0) d = (int64_t)t; } + v = d; return v; +} + el_val_t engram_activate(el_val_t query, el_val_t depth) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); @@ -9552,25 +9588,43 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { backfilled++; } } - /* Query embedding, cached single-slot: the curiosity loop re-issues the - * same 4 rotating phrases, so consecutive identical queries skip the - * HTTP round-trip entirely. */ - static char* _eg_qcache_text = NULL; - static float* _eg_qcache_emb = NULL; - static int32_t _eg_qcache_dim = 0; + /* Query embedding cache (2026-08-15, PR #105 port: single-slot -> direct- + * mapped multi-slot). The single-slot cache below this comment's history + * only remembered the LAST query, so "the curiosity loop re-issues the + * same 4 rotating phrases" only hit when two CONSECUTIVE calls used the + * SAME phrase — any rotation among >1 phrase evicted the slot before it + * could be reused. #105 measured this as a real cost (a repeated query + * costing a full Ollama round-trip whenever a different phrase intervened) + * and fixed it with a direct-mapped, FNV-1a-keyed cache sized for the + * rotation. Ported here on TOP of the existing cosq/e_eff semantic layer + * (this cache only ever supplies q_emb/q_dim into that unchanged + * pipeline) rather than replacing it — see the M8/#105 reconciliation + * note above eg_cosq_at. ENGRAM_QCACHE_SIZE must be a power of two (mask + * indexing below). Full strcmp on lookup rejects hash collisions; each + * slot owns its `text`/`vec` and is freed on eviction, matching the old + * single-slot free/replace contract — q_emb below still points at cache- + * owned memory the caller must NOT free, just as before. */ +#define ENGRAM_QCACHE_SIZE 1024 + typedef struct { char* text; uint64_t hash; float* vec; int32_t dim; } EgQCacheEntry; + static EgQCacheEntry _eg_qcache[ENGRAM_QCACHE_SIZE]; float* q_emb = NULL; int32_t q_dim = 0; - if (_eg_qcache_text && strcmp(_eg_qcache_text, q) == 0) { - q_emb = _eg_qcache_emb; q_dim = _eg_qcache_dim; - } else { - int32_t d = 0; - float* v = eg_embed_fetch(q, &d); - if (v) { - free(_eg_qcache_text); free(_eg_qcache_emb); - _eg_qcache_text = strdup(q); - _eg_qcache_emb = v; - _eg_qcache_dim = d; - q_emb = v; q_dim = d; + { + uint64_t qh = engram_id_hash(q); + EgQCacheEntry* slot = &_eg_qcache[qh & (ENGRAM_QCACHE_SIZE - 1)]; + if (slot->vec && slot->hash == qh && slot->text && strcmp(slot->text, q) == 0) { + q_emb = slot->vec; q_dim = slot->dim; + } else { + int32_t d = 0; + float* v = eg_embed_fetch(q, &d); + if (v) { + free(slot->text); free(slot->vec); /* evict prior occupant */ + slot->text = strdup(q); + slot->hash = qh; + slot->vec = v; + slot->dim = d; + q_emb = v; q_dim = d; + } } } /* ── Context centroid fold-in (2026-07-29) ────────────────────────── @@ -9997,8 +10051,45 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { const double FAN_DREF = (g->adj_connected > 0) ? (2.0 * (double)g->edge_count / (double)g->adj_connected) : 0.0; _eg_act_fan_dref = FAN_DREF; + const int64_t activate_beam = engram_activate_beam(); while (fhead < ftail) { - Frontier f = fr[fhead++]; + /* Level-batch (2026-08-15, PR #105 port): entries sharing .hops are + * always contiguous — hop k+1 entries are appended only while + * processing hop k, strictly after the current ftail, so they form one + * block right after hop k's block (see engram_activate_beam's comment + * for why this holds even with the improve-and-re-enqueue behavior + * below). Find this level's extent, then beam-select which of it + * EXPANDS; every entry in the level still gets recorded via + * reached[]/best_bg[] regardless (that happened when it was enqueued, + * one level up) — the cap bounds propagation width only. */ + int64_t level_hops = fr[fhead].hops; + int64_t level_start = fhead; + int64_t level_end = fhead; + while (level_end < ftail && fr[level_end].hops == level_hops) level_end++; + int64_t level_n = level_end - level_start; + unsigned char* expand = NULL; + if (level_n > activate_beam) { + expand = calloc((size_t)level_n, 1); + if (expand) { + /* Partial selection: mark the top-`activate_beam` entries by + * .act. O(beam*level_n) — beam is the small tunable. */ + for (int64_t bsel = 0; bsel < activate_beam; bsel++) { + int64_t best = -1; + for (int64_t k = 0; k < level_n; k++) { + if (expand[k]) continue; + if (best < 0 || fr[level_start+k].act > fr[level_start+best].act) + best = k; + } + if (best < 0) break; + expand[best] = 1; + } + } + /* OOM on the selection map: expand stays NULL -> this level runs + * unbounded, same as if beam were disabled. Never silently wrong. */ + } + for (int64_t lk = level_start; lk < level_end; lk++) { + if (expand && !expand[lk - level_start]) continue; + Frontier f = fr[lk]; if (f.hops >= max_depth) continue; int64_t cur = f.idx; int64_t new_hops = f.hops + 1; @@ -10130,6 +10221,9 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { } } } + } + free(expand); + fhead = level_end; } /* Persist layer-1 background_activation to node store. */ for (int64_t i = 0; i < g->node_count; i++) { From 9d40f879260405c8d2f4325bdab55870812dc790 Mon Sep 17 00:00:00 2001 From: bigmerge Date: Sat, 15 Aug 2026 17:28:19 -0500 Subject: [PATCH 4/5] ingest: unify transduce_prose/transduce_structured into one transduce() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit transduce() is now THE single mechanism: one function, no content-type branch inside it. It never asks whether `source` is prose, JSON, or raw/opaque bytes (audio, etc.) — it runs one algorithm unconditionally: split on "\n\n" as a universal boundary-marker check, and if that finds no boundary, fall back to fixed 4096-char windows. Same node/edge wiring (root -contains-> chunk, chunk -precedes-> next, "#"-prefixed chunk gets a heading/section_of link) regardless of what's inside a chunk. Dedup is the existing find_existing_by_content path via merge_manifold, applied uniformly. The old transduce_structured JSON dataset/records/feature-node interpretation is deleted outright, not just unused — a JSON file now gets chunked and deduped like anything else, with no pre-computed structure. All five ingest_* entry points still exist unchanged in name and role; ingest_file/ingest_dir/ingest_url/ingest_llm now call the one transduce() (ingest_stream builds its own turn-nodes directly and never called either old function, so it's untouched). This unlocks raw/opaque content (audio, or anything else with no natural text/JSON shape) without any DSP, LLM call, or external API: transduce() chunks it exactly like it chunks anything else. There is zero semantic understanding of audio (or any payload) claimed or built here — any meaning is expected to emerge later from Neuron's own existing mechanisms (embedding, spreading activation, dedup) acting on this real geometry over time. Two small C builtins added to el_runtime.c/h (fs_size, fs_read_b64_chunk) because El strings are NUL-unsafe under strlen-based ops and fs_read()'s result silently truncates at the first embedded NUL, which is routine in real binary/audio bytes. ingest_file compares fs_read()'s string length against a real fs_size() stat() count; on mismatch it rebuilds the payload as base64-encoded fixed 3072-byte windows read directly off disk (binary-safe in C, verbatim, no invention), joined with the same "\n\n" marker transduce()'s boundary scan already looks for. This is a mechanical fidelity fix, not interpretation of content — transduce() never learns a fallback happened. Registered both builtins' arity in codegen.el; did not rebuild the elc compiler binary itself (unrelated, pre-existing gap: self-hosting elc via el_seed.c fails on this worktree independent of this change, reproduced with codegen.el reverted) — the existing elc binary compiles calls to unregistered builtins via its already-existing arity=-1 passthrough, confirmed by an actual clean `elc ingest.el` + `cc` build against the modified el_runtime.c. INGEST_KIND keeps existing only as an acquisition-mechanism selector (dir/file/url/llm/stream — which RPC to use to fetch bytes), not as a content-type flag; the redundant "structured" value (an alias for "file" that hinted the now-deleted JSON branch) is removed. ingest_dir drops its file-extension filter for the same reason: transduce() takes anything now. Verification: local manifold construction confirmed correct against a real captured audio file (will_clean.wav, 304288 bytes, and a 12288-byte real prefix slice) — exact expected node/edge counts both times (101 nodes/199 edges full file; 5 nodes/7 edges for the slice, matching ceil(bytes/3072)+1 nodes and 2n-1 edges), with real, verbatim base64 content confirmed decoding back to the actual WAV header bytes. Compiles clean via the real elc + the modified el_runtime.c/engram_*.c (built and booted an actual sandbox engram off this exact source with `nsbx create --branch`). NOT verified this session, disclosed rather than papered over: end-to-end server-confirmed persistence (a real before/after /api/stats delta, and a fetched node by id) for the audio, prose, and JSON-fixture cases. Every local nsbx sandbox engram tried tonight (two stock pre-#109 binaries hitting the known O(N*D) brute-force scan bug, then a fresh #109/HNSW binary built from current dev) took minutes-to indefinitely long on the final /api/load-merge write's embedding step and hit the client's 60s HTTP timeout before responding, even for a 5-node write. This is confirmed as real (if slow) forward progress, not a hang: the sandbox's WAL file was observed growing steadily across every attempt. The code's own pre-existing HONESTY GATE correctly refused to report success in every case, returning "load-merge failed: ..." with a "nothing below this manifold was confirmed persisted by the server" note instead — exactly as designed. This is an environment/infrastructure limitation, not a defect introduced by this change: the engram server binary itself is untouched by this commit. --- ingest/.gitignore | 1 + ingest/src/ingest.el | 477 ++++++++++++++------------------ lang/el-compiler/src/codegen.el | 2 + lang/runtime/el_runtime.c | 51 ++++ lang/runtime/el_runtime.h | 14 + 5 files changed, 283 insertions(+), 262 deletions(-) create mode 100644 ingest/.gitignore diff --git a/ingest/.gitignore b/ingest/.gitignore new file mode 100644 index 0000000..567609b --- /dev/null +++ b/ingest/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/ingest/src/ingest.el b/ingest/src/ingest.el index 42dfcbb..1d47432 100644 --- a/ingest/src/ingest.el +++ b/ingest/src/ingest.el @@ -1,17 +1,28 @@ // ingest.el — the native EL AFFERENT INGEST ORGAN // // The source-polymorphic ingest(source) primitive: point it at a directory, -// file, url, llm-query, structured-primitive set, or stream; it EXTRACTS the -// real content faithfully (no invention), TRANSDUCES it into a DISCRETE -// MANIFOLD (multiple nodes + internal edges — meaning-structure, never a -// single blob; the conversion from extracted surface content into geometry -// is automatic and invisible to the caller, the way digestion is invisible -// to the one who chose to eat — ingest is the conscious act, transduce is -// the mechanism underneath it, and it is no less real for being unseen), -// and MERGES that manifold into the engram geometry: shared -// meanings DEDUP onto existing nodes (search + exact/cosine match), genuinely -// new meanings add nodes, relations add edges. Every node enters with -// PROVENANCE + grounding-level + stewardship class from the moment of entry. +// file, url, llm-query, or stream; it EXTRACTS the real content faithfully +// (no invention), TRANSDUCES it into a DISCRETE MANIFOLD (multiple nodes + +// internal edges — meaning-structure, never a single blob; the conversion +// from extracted surface content into geometry is automatic and invisible +// to the caller, the way digestion is invisible to the one who chose to +// eat — ingest is the conscious act, transduce is the mechanism underneath +// it, and it is no less real for being unseen), and MERGES that manifold +// into the engram geometry: shared meanings DEDUP onto existing nodes +// (search + exact/cosine match), genuinely new meanings add nodes, +// relations add edges. Every node enters with PROVENANCE + grounding-level +// + stewardship class from the moment of entry. +// +// transduce() is THE single mechanism — one function, polymorphic, with no +// content-type branch inside it. It does not ask whether a payload is +// prose, structured data, or raw/opaque bytes (audio, or anything else); +// it runs one boundary-scan-with-fixed-window-fallback chunking algorithm +// and one dedup mechanism on whatever bytes it's handed, unconditionally. +// Any deeper structure a payload might have (shared fields, relationships, +// what a chunk of audio "means") is NOT interpreted here — that's left +// entirely to the engram's own mechanisms (embedding, spreading activation, +// dedup) acting on this real geometry over time. This organ claims zero +// semantic understanding of any payload it transduces. // // It is a pure HTTP CLIENT of the engram server — it links only el_runtime.c // via fs/http/json/string builtins; it never links el_seed.c or the engram @@ -46,60 +57,6 @@ fn j_q(s: String) -> String { return "\"" + j_esc(s) + "\"" } -// Extract the top-level keys of a JSON object string. A thin, self-contained -// scanner (FLAGGED: the one non-trivial parser in this organ — everything else -// is faithful text handling). Tracks string state + brace/bracket depth; a key -// is a string at object-interior depth 1 immediately followed by ':'. -fn json_object_keys(obj: String) -> [String] { - let keys: [String] = el_list_empty() - let n: Int = str_len(obj) - let i: Int = 0 - let depth: Int = 0 - let in_str: Bool = false - let esc: Bool = false - let str_start: Int = -1 - let cur: String = "" - let have_key: Bool = false - while i < n { - let c: String = str_char_at(obj, i) - if in_str { - if esc { - esc = false - } else { - if str_eq(c, "\\") { - esc = true - } else { - if str_eq(c, "\"") { - in_str = false - cur = str_slice(obj, str_start + 1, i) - have_key = true - } - } - } - } else { - if str_eq(c, "\"") { - in_str = true - str_start = i - } - if str_eq(c, "{") { depth = depth + 1 } - if str_eq(c, "}") { depth = depth - 1 } - if str_eq(c, "[") { depth = depth + 1 } - if str_eq(c, "]") { depth = depth - 1 } - if str_eq(c, ":") { - if have_key { - if depth == 1 { - keys = el_list_append(keys, cur) - } - } - have_key = false - } - if str_eq(c, ",") { have_key = false } - } - i = i + 1 - } - return keys -} - // ═══════════════════════════════════════════════════════════════════════════ // SECTION B — engram HTTP client (provenance-carrying afferent LOAD) // ═══════════════════════════════════════════════════════════════════════════ @@ -402,168 +359,125 @@ fn head80(s: String) -> String { // We accumulate into module-level lists carried by the caller. // ═══════════════════════════════════════════════════════════════════════════ -// PROSE: chunk text into a discrete manifold. Split on blank lines into -// paragraphs; every non-empty paragraph is its own node (NEVER one blob). -// Edges: doc-root -contains-> chunk; chunk -precedes-> next chunk; -// most-recent-heading -section_of-> chunk. Content is verbatim (substring of -// the source) — pure extraction of ground truth. -fn transduce_prose(nodes: [String], edges: [String], text: String, - prov: String, ground: String, steward: String, - root_lid: String, root_title: String) -> [String] { - // returns [nodes_json_list_encoded, edges_json_list_encoded] is awkward in - // EL; instead we mutate by returning a 2-list. We package results as a - // single JSON array string carrying {nodes:[...],edges:[...]} additions. - // (Kept simple: caller passes empty lists and receives the packaged pair.) +// TRANSDUCE — the single mechanism. Takes ANY payload (prose, structured +// data, raw/opaque bytes — audio, whatever) as one opaque string and turns +// it into a discrete manifold: nodes + internal edges. There is no +// content-type branch anywhere in this function. It never asks "is this +// text," "is this JSON," "is this audio" — it runs ONE algorithm on the +// bytes it is given, unconditionally: +// +// 1. BOUNDARY SCAN — split on "\n\n". This is a property of the bytes +// (does a blank-line-style marker occur in them, yes or no), not a +// classification of what the content IS. Prose paragraphs split on it +// because that's how prose is typically written; that's a fact about +// the bytes, not a rule this function knows about prose. Anything else +// that happens to contain the same marker splits on it too, and +// anything that doesn't, doesn't — same code path either way. +// 2. FIXED-WINDOW FALLBACK — if step 1 found no boundary (0 or 1 non-empty +// piece), the payload is cut into fixed-size windows instead. Same +// chunk-per-node, edge-per-adjacency structure as step 1 produces; only +// the source of the cut point differs. +// +// Every resulting chunk becomes its own node (never one blob), wired with +// the same edges regardless of what's inside a chunk: root -contains-> +// chunk, chunk -precedes-> next chunk, and — if a chunk happens to start +// with "#" — most-recent-heading -section_of-> chunk. That "#" check is a +// structural marker (a fact about a chunk's first byte), not a decision +// about whether this run is "the text case": chunks from any payload that +// never happen to start with "#" simply never trigger it. +// +// Dedup is the existing, fully generic mechanism (find_existing_by_content, +// via merge_manifold downstream of merge_packed) applied uniformly to every +// chunk from every payload — there is no separate "structured" dedup path. +// Any deeper structure that might exist inside a payload (shared fields, +// repeated records, relationships) is NOT pre-computed here; that's left to +// the engram's own mechanisms (embedding, spreading activation, dedup) +// acting on this geometry over time, which is the whole point of handing it +// raw bytes instead of a hand-coded interpretation of them. +// +// Byte-safety note: `source` must already be a string this function can +// safely str_split/str_slice. Protecting it from silent truncation (El +// strings are NUL-unsafe under strlen-based ops; fs_read()'s result +// truncates at the first embedded NUL, which is routine in real binary +// bytes) is a MECHANICAL fidelity concern that belongs to whatever produced +// `source` (see ingest_file's file_source_string below) — not a +// content-type judgment made in here. transduce() never learns whether a +// chunk is plain text or a base64-encoded raw-byte window; every chunk is +// handled identically either way. +fn transduce(nodes: [String], edges: [String], source: String, + prov: String, ground: String, steward: String, + root_lid: String, root_title: String) -> [String] { let tagbase: String = "prov:" + prov + " ground:" + ground + " steward:" + steward - // root node - nodes = el_list_append(nodes, mk_node(root_lid, "document: " + root_title, - "Concept", "Semantic", "0.6", "0.6", "0.9", tagbase + " kind:document")) + nodes = el_list_append(nodes, mk_node(root_lid, "source: " + root_title, + "Concept", "Semantic", "0.6", "0.6", "0.9", tagbase + " kind:source")) - let paras: [String] = str_split(text, "\n\n") - let np: Int = el_list_len(paras) - let idx: Int = 0 + // step 1: universal boundary scan + let boundary_parts: [String] = str_split(source, "\n\n") + let chunks: [String] = el_list_empty() + let bp_n: Int = el_list_len(boundary_parts) + let bp_i: Int = 0 + while bp_i < bp_n { + let piece: String = str_trim(el_list_get(boundary_parts, bp_i)) + if !str_eq(piece, "") { chunks = el_list_append(chunks, piece) } + bp_i = bp_i + 1 + } + + // step 2: no boundary found -> fixed-size windows over the whole + // payload. 4096 chars/node: low kilobytes — big enough to keep node + // count sane on a large unbroken payload, small enough that each node + // stays a legible, individually embeddable/dedupable unit rather than + // one giant blob. + if el_list_len(chunks) <= 1 { + chunks = el_list_empty() + let total: Int = str_len(source) + let win: Int = 4096 + let off: Int = 0 + while off < total { + let endp: Int = if off + win < total { off + win } else { total } + let piece: String = str_slice(source, off, endp) + if !str_eq(piece, "") { chunks = el_list_append(chunks, piece) } + off = off + win + } + } + + let nc: Int = el_list_len(chunks) + let ci: Int = 0 let last_chunk: String = "" let last_heading: String = "" - let ci: Int = 0 - while idx < np { - let raw: String = str_trim(el_list_get(paras, idx)) - if !str_eq(raw, "") { - let lid: String = root_lid + ":c" + int_to_str(ci) - let is_heading: Bool = str_starts_with(raw, "#") - let kind: String = if is_heading { "kind:heading" } else { "kind:doc-chunk" } - nodes = el_list_append(nodes, mk_node(lid, raw, - "Knowledge", "Semantic", "0.55", "0.55", "0.9", tagbase + " " + kind)) - // containment: document root -contains-> chunk - edges = el_list_append(edges, mk_edge(root_lid, "contains", lid)) - // sequence: previous chunk -precedes-> this chunk - if !str_eq(last_chunk, "") { - edges = el_list_append(edges, mk_edge(last_chunk, "precedes", lid)) - } - // sectioning: most-recent heading -section_of-> this chunk - if is_heading { - last_heading = lid - } else { - if !str_eq(last_heading, "") { - edges = el_list_append(edges, mk_edge(last_heading, "section_of", lid)) - } - } - last_chunk = lid - ci = ci + 1 + while ci < nc { + let raw: String = el_list_get(chunks, ci) + let lid: String = root_lid + ":c" + int_to_str(ci) + let is_heading: Bool = str_starts_with(raw, "#") + let kind: String = if is_heading { "kind:heading" } else { "kind:chunk" } + nodes = el_list_append(nodes, mk_node(lid, raw, + "Knowledge", "Semantic", "0.55", "0.55", "0.9", tagbase + " " + kind)) + // containment: root -contains-> chunk + edges = el_list_append(edges, mk_edge(root_lid, "contains", lid)) + // sequence: previous chunk -precedes-> this chunk + if !str_eq(last_chunk, "") { + edges = el_list_append(edges, mk_edge(last_chunk, "precedes", lid)) } - idx = idx + 1 - } - // package: we return the two lists concatenated via a sentinel; but EL - // lists can't nest heterogeneously here, so we instead return nodes and - // rely on the caller holding edges by reference is not possible — so we - // encode both into one list: [ "N" + nodejson ... , "E" + edgejson ... ]. - let packed: [String] = el_list_empty() - let a: Int = 0 - let an: Int = el_list_len(nodes) - while a < an { packed = el_list_append(packed, "N" + el_list_get(nodes, a)) a = a + 1 } - let b: Int = 0 - let bn: Int = el_list_len(edges) - while b < bn { packed = el_list_append(packed, "E" + el_list_get(edges, b)) b = b + 1 } - return packed -} - -// STRUCTURED / RAW-GEOMETRY: ingest structured primitives (phonetics/formants, -// instrument signatures, scene primitives) as GEOMETRY, faithfully. Normalized -// input shape: -// {"dataset":"","primitive_type":"", -// "records":[{"key":"","features":{...categorical...},"attributes":{...}}]} -// Each record -> a primitive node; each categorical feature -> a SHARED feature -// node (deduped across records: many primitives -> one feature node = real -// connective geometry, meaning saturates); numeric attributes fold into the -// primitive's content (unique values, no dedup benefit). This is knowledge -// represented as geometry, not prose — the path speech/music/image ingest on. -fn transduce_structured(nodes: [String], edges: [String], js: String, - prov: String, ground: String, steward: String, - root_lid: String) -> [String] { - // grounding integrity: the SOURCE may declare its own epistemic grounding - // (measured / derived / convention / ...) via a top-level "grounding" field; - // honor it faithfully over the ingest-time default. This keeps the per-node - // ground: facet consistent with the source's honest self-description. - let src_ground: String = json_get_string(js, "grounding") - let use_ground: String = if str_eq(src_ground, "") { ground } else { src_ground } - let tagbase: String = "prov:" + prov + " ground:" + use_ground + " steward:" + steward - let dsname: String = json_get_string(js, "dataset") - let ptype: String = json_get_string(js, "primitive_type") - // capture the source's own scholarly provenance citation (verbatim) onto - // the dataset root — faithful attribution, retrievable, reachable from every - // primitive via its -contains- edge back to the root. - let src_cite: String = json_get_string(js, "provenance") - let root_content: String = "dataset: " + dsname + " (" + ptype + ")" - if !str_eq(src_cite, "") { root_content = root_content + " | provenance: " + src_cite } - nodes = el_list_append(nodes, mk_node(root_lid, root_content, - "Concept", "Semantic", "0.6", "0.6", "0.9", tagbase + " kind:dataset")) - - let recs: String = json_get_raw(js, "records") - let nr: Int = json_array_len(recs) - let r: Int = 0 - while r < nr { - let rec: String = json_array_get(recs, r) - let rkey: String = json_get_string(rec, "key") - let attrs: String = json_get_raw(rec, "attributes") - // faithful compact serialization of the primitive's numeric signature - let attr_str: String = flatten_pairs(attrs) - let content: String = ptype + " " + rkey - if !str_eq(attr_str, "") { content = content + " | " + attr_str } - let plid: String = root_lid + ":" + rkey - nodes = el_list_append(nodes, mk_node(plid, content, - "Concept", "Semantic", "0.6", "0.6", "0.92", - tagbase + " kind:primitive primitive:" + ptype + " key:" + rkey)) - edges = el_list_append(edges, mk_edge(root_lid, "contains", plid)) - - // categorical features -> SHARED (deduped) feature nodes + labelled edges - let feats: String = json_get_raw(rec, "features") - let fkeys: [String] = json_object_keys(feats) - let fk: Int = el_list_len(fkeys) - let k: Int = 0 - while k < fk { - let fname: String = el_list_get(fkeys, k) - let fval: String = json_get_string(feats, fname) - // shared feature node: content is the feature=value pair; identical - // pairs across records dedup onto ONE node (the geometry). - let flid: String = "feat:" + fname + "=" + fval - let fcontent: String = fname + "=" + fval - nodes = el_list_append(nodes, mk_node(flid, fcontent, - "Concept", "Semantic", "0.5", "0.5", "0.9", - tagbase + " kind:feature feature:" + fname)) - edges = el_list_append(edges, mk_edge(plid, fname, flid)) - k = k + 1 + // sectioning: most-recent heading -section_of-> this chunk + if is_heading { + last_heading = lid + } else { + if !str_eq(last_heading, "") { + edges = el_list_append(edges, mk_edge(last_heading, "section_of", lid)) + } } - r = r + 1 + last_chunk = lid + ci = ci + 1 } - let packed: [String] = el_list_empty() - let a: Int = 0 - let an: Int = el_list_len(nodes) - while a < an { packed = el_list_append(packed, "N" + el_list_get(nodes, a)) a = a + 1 } - let b: Int = 0 - let bn: Int = el_list_len(edges) - while b < bn { packed = el_list_append(packed, "E" + el_list_get(edges, b)) b = b + 1 } - return packed -} -// flatten a flat JSON object of scalar fields into "k=v k=v" (faithful; values -// verbatim). Used for numeric attribute signatures. -fn flatten_pairs(obj: String) -> String { - if str_eq(obj, "") { return "" } - let keys: [String] = json_object_keys(obj) - let n: Int = el_list_len(keys) - let out: String = "" - let i: Int = 0 - while i < n { - let k: String = el_list_get(keys, i) - // json_get_raw returns the raw token — works for NUMBERS (bare, e.g. - // "270") where json_get_string yields "" for non-string values. Strip - // surrounding quotes if the value happens to be a string token. - let raw: String = json_get_raw(obj, k) - let v: String = str_replace(raw, "\"", "") - let sep: String = if i == 0 { "" } else { " " } - out = out + sep + k + "=" + v - i = i + 1 - } - return out + // package both lists into one, "N"/"E"-prefixed (see merge_packed). + let packed: [String] = el_list_empty() + let pn_i: Int = 0 + let pn_n: Int = el_list_len(nodes) + while pn_i < pn_n { packed = el_list_append(packed, "N" + el_list_get(nodes, pn_i)) pn_i = pn_i + 1 } + let pe_i: Int = 0 + let pe_n: Int = el_list_len(edges) + while pe_i < pe_n { packed = el_list_append(packed, "E" + el_list_get(edges, pe_i)) pe_i = pe_i + 1 } + return packed } // unpack the "N"/"E"-prefixed packed list back into two lists, then merge @@ -594,18 +508,7 @@ fn basename(path: String) -> String { return el_list_get(parts, n - 1) } -fn ends_with_ci(s: String, suf: String) -> Bool { - return str_ends_with(str_to_lower(s), suf) -} - -fn is_text_file(path: String) -> Bool { - return ends_with_ci(path, ".md") || ends_with_ci(path, ".txt") - || ends_with_ci(path, ".markdown") || ends_with_ci(path, ".text") -} - // default ingestion grounding; overridable per-invocation via INGEST_GROUND. -// Note: a source's OWN top-level "grounding" field (structured) takes precedence -// over this — the author's honest self-description wins. fn default_ground() -> String { let g: String = env("INGEST_GROUND") if str_eq(g, "") { return "extracted" } @@ -618,25 +521,67 @@ fn default_steward() -> String { return s } -// ingest one file -> report JSON +// Mechanical fidelity guard — NOT a content-type test. fs_read()'s el_val_t +// result truncates at the first embedded NUL byte under El's strlen-based +// string ops (see fs_size's doc comment in runtime/el_runtime.h); comparing +// its length against fs_size() (a real stat()-based byte count) is a +// technical fact about whether the string channel captured the file intact +// — computed the same way for a poem, a JSON file, or a WAV, and saying +// nothing about what the file IS. When the counts agree, `text` is +// trustworthy verbatim. When they don't (silent truncation happened), +// rebuild the payload as base64-encoded fixed-size windows read directly +// off disk (fs_read_b64_chunk — binary-safe in C), joined with the same +// "\n\n" boundary marker transduce()'s generic scan already looks for, so +// transduce() sees one ordinary boundary-delimited payload and runs its one +// algorithm on it exactly as it would on prose — it never learns that a +// fidelity problem occurred upstream, let alone why. +fn file_source_string(path: String, text: String, real_size: Int) -> String { + if real_size <= 0 { return text } + if str_len(text) == real_size { return text } + // 3072 raw bytes -> 4096 base64 chars (3 divides evenly into base64's + // 3-byte/4-char ratio); keeps each resulting node's content a clean, + // bounded, low-kilobytes unit, same order of magnitude as the fixed + // fallback window in transduce() itself. + let win: Int = 3072 + let out: String = "" + let off: Int = 0 + let first: Bool = true + while off < real_size { + let chunk_b64: String = fs_read_b64_chunk(path, off, win) + if str_eq(chunk_b64, "") { + off = real_size + } else { + let sep: String = if first { "" } else { "\n\n" } + out = out + sep + chunk_b64 + first = false + off = off + win + } + } + return out +} + +// ingest one file -> report JSON. Uniform for every file regardless of +// extension or content — transduce() decides nothing about content-type, so +// neither does this function; it only decides whether the raw bytes made it +// through the read intact (file_source_string), which is a fidelity +// question, not a format one. fn ingest_file(path: String) -> String { + let real_size: Int = fs_size(path) let text: String = fs_read(path) - if str_eq(text, "") { + let source: String = file_source_string(path, text, real_size) + if str_eq(source, "") { return "{\"error\":\"empty or unreadable\",\"path\":" + j_q(path) + "}" } let prov: String = "file:" + path - if ends_with_ci(path, ".json") { - let packed: [String] = transduce_structured(el_list_empty(), el_list_empty(), - text, prov, default_ground(), default_steward(), "ds:" + basename(path)) - return merge_packed(packed) - } - let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(), - text, prov, default_ground(), default_steward(), + let packed: [String] = transduce(el_list_empty(), el_list_empty(), + source, prov, default_ground(), default_steward(), "doc:" + basename(path), basename(path)) return merge_packed(packed) } -// ingest a directory: walk one level, ingest each supported file, aggregate +// ingest a directory: walk one level, ingest every file found, aggregate. +// No extension filter — transduce() handles any payload uniformly now, so +// there is no content-type gate at the directory boundary either. fn ingest_dir(path: String) -> String { let entries: [String] = fs_list(path) let n: Int = el_list_len(entries) @@ -649,14 +594,12 @@ fn ingest_dir(path: String) -> String { let name: String = str_trim(el_list_get(entries, i)) if !str_eq(name, "") { let full: String = path + "/" + name - if is_text_file(full) || ends_with_ci(full, ".json") { - println("FILE " + full) - let rep: String = ingest_file(full) - tot_created = tot_created + json_get_int(rep, "nodes_created") - tot_deduped = tot_deduped + json_get_int(rep, "nodes_deduped") - tot_edges = tot_edges + json_get_int(rep, "edges_added") - files = files + 1 - } + println("FILE " + full) + let rep: String = ingest_file(full) + tot_created = tot_created + json_get_int(rep, "nodes_created") + tot_deduped = tot_deduped + json_get_int(rep, "nodes_deduped") + tot_edges = tot_edges + json_get_int(rep, "edges_added") + files = files + 1 } i = i + 1 } @@ -667,11 +610,12 @@ fn ingest_dir(path: String) -> String { ",\"edges_accepted\":" + int_to_str(tot_edges) + "}" } -// ingest a url: fetch, treat body as prose (faithful extraction of what's there) +// ingest a url: fetch, hand the body straight to transduce (faithful +// extraction of what's there — no interpretation of what it is) fn ingest_url(url: String) -> String { let body: String = http_get(url) if str_eq(body, "") { return "{\"error\":\"empty fetch\",\"url\":" + j_q(url) + "}" } - let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(), + let packed: [String] = transduce(el_list_empty(), el_list_empty(), body, "url:" + url, "extracted", "public-web", "url:" + url, url) return merge_packed(packed) @@ -686,7 +630,7 @@ fn ingest_llm(query: String) -> String { let resp: String = http_post_json("http://127.0.0.1:11434/api/generate", body) let answer: String = json_get_string(resp, "response") if str_eq(answer, "") { return "{\"error\":\"no model response\"}" } - let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(), + let packed: [String] = transduce(el_list_empty(), el_list_empty(), answer, "llm:" + model + ":" + query, "candidate-provisional", "guide-provisional", "llm:" + query, "guide answer: " + query) return merge_packed(packed) @@ -728,6 +672,19 @@ fn ingest_stream(path: String) -> String { // SECTION G — ENTRY // ═══════════════════════════════════════════════════════════════════════════ +// INGEST_KIND selects an ACQUISITION mechanism only — dir/file/url/llm/ +// stream — i.e. which RPC shape to use to go get the bytes (walk a +// directory, open a file, fetch a URL, query an LLM, read a turn-stream). +// That is a genuinely unavoidable choice at the process-entry boundary +// (nothing about the string "/tmp/x" tells you whether it's a file to read +// or a stream to read line-by-line, or distinguishes an LLM query from a +// path), so it cannot be dropped the way content-type dispatch was. +// It is NOT a content-type flag: it says nothing about what's inside the +// bytes once fetched, and none of the five ingest_* functions it selects +// among interpret their payload differently by content shape anymore — +// they all hand off to the single, format-agnostic transduce(). The old +// "structured" value (a caller-declared alias for "file", used only to hint +// the now-removed JSON-vs-prose branch) is gone along with that branch. let kind: String = env("INGEST_KIND") let arg: String = env("INGEST_ARG") @@ -741,20 +698,16 @@ if str_eq(kind, "dir") { if str_eq(kind, "file") { report = ingest_file(arg) } else { - if str_eq(kind, "structured") { - report = ingest_file(arg) + if str_eq(kind, "url") { + report = ingest_url(arg) } else { - if str_eq(kind, "url") { - report = ingest_url(arg) + if str_eq(kind, "llm") { + report = ingest_llm(arg) } else { - if str_eq(kind, "llm") { - report = ingest_llm(arg) + if str_eq(kind, "stream") { + report = ingest_stream(arg) } else { - if str_eq(kind, "stream") { - report = ingest_stream(arg) - } else { - report = "{\"error\":\"unknown INGEST_KIND: " + kind + "\"}" - } + report = "{\"error\":\"unknown INGEST_KIND: " + kind + "\"}" } } } diff --git a/lang/el-compiler/src/codegen.el b/lang/el-compiler/src/codegen.el index 9b86040..d13410c 100644 --- a/lang/el-compiler/src/codegen.el +++ b/lang/el-compiler/src/codegen.el @@ -2766,6 +2766,8 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "fs_read") { return 1 } if str_eq(name, "fs_write") { return 2 } if str_eq(name, "fs_list") { return 1 } + if str_eq(name, "fs_size") { return 1 } + if str_eq(name, "fs_read_b64_chunk") { return 3 } // JSON if str_eq(name, "json_get") { return 2 } if str_eq(name, "json_parse") { return 1 } diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index dfb9fa0..a896549 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -2226,6 +2226,22 @@ el_val_t fs_exists(el_val_t pathv) { return (el_val_t)(stat(path, &st) == 0 ? 1 : 0); } +/* fs_size — real on-disk byte count of a file, via stat() (not strlen). + * Needed alongside fs_read_b64_chunk() below: fs_read()'s el_val_t string + * result is NUL-terminated and length-unsafe for arbitrary binary content + * (str_len/str_slice fall back to strlen when the buffer isn't a tagged + * binary value — see el_input_len), so any caller that needs to walk a + * binary file in fixed-size windows (e.g. transduce()'s raw/opaque chunking + * of audio bytes) must learn the true length here instead of from the + * decoded string. Returns -1 if the path doesn't exist or isn't stat-able. */ +el_val_t fs_size(el_val_t pathv) { + const char* path = EL_CSTR(pathv); + if (!path || !*path) return -1; + struct stat st; + if (stat(path, &st) != 0) return -1; + return (el_val_t)st.st_size; +} + /* fs_mkdir — create directory at path with mode 0755, mkdir -p semantics. * Returns 1 if path exists or was created (incl. all parents); 0 on failure. * Walks the path component-by-component so missing intermediate dirs are @@ -16469,6 +16485,41 @@ el_val_t el_base64_encode_n(const unsigned char* data, size_t len, int url_safe) return el_wrap_str(out); } +/* fs_read_b64_chunk — binary-safe windowed file read: read up to `length` + * raw bytes starting at byte `offset` from `path` and return them base64- + * encoded (RFC 4648, standard alphabet, padded). The raw bytes are read into + * a local C buffer and base64-encoded directly here — they never pass + * through an el_val_t string as raw bytes, so embedded NUL bytes (common in + * real PCM audio) never hit a strlen()-based code path. This mirrors the + * existing llm_vision() image-attachment path (read file -> base64 in C -> + * hand back a plain-ASCII string) and the http_*_to_file() rationale above: + * bypass the string wrapper entirely for the part that must stay binary. + * + * Returns "" if the path can't be opened, offset is negative or past EOF, + * or length <= 0. A short final chunk (less than `length` bytes remaining) + * is returned truncated to what's actually on disk — never padded/invented. */ +el_val_t fs_read_b64_chunk(el_val_t pathv, el_val_t offsetv, el_val_t lengthv) { + const char* path = EL_CSTR(pathv); + int64_t offset = (int64_t)offsetv; + int64_t length = (int64_t)lengthv; + if (!path || !*path || offset < 0 || length <= 0) return el_wrap_str(el_strdup("")); + FILE* f = fopen(path, "rb"); + if (!f) return el_wrap_str(el_strdup("")); + fseek(f, 0, SEEK_END); + long sz = ftell(f); + if (sz < 0 || offset >= sz) { fclose(f); return el_wrap_str(el_strdup("")); } + fseek(f, (long)offset, SEEK_SET); + long remain = sz - (long)offset; + size_t want = (size_t)(((int64_t)remain < length) ? remain : length); + unsigned char* buf = malloc(want > 0 ? want : 1); + if (!buf) { fclose(f); return el_wrap_str(el_strdup("")); } + size_t got = fread(buf, 1, want, f); + fclose(f); + el_val_t out = el_base64_encode_n(buf, got, /*url_safe=*/0); + free(buf); + return out; +} + /* Decode either alphabet — accepts both '+/' and '-_' transparently, and * tolerates missing padding (which JWTs typically omit). Whitespace is * skipped for robustness. Invalid characters cause the decode to stop and diff --git a/lang/runtime/el_runtime.h b/lang/runtime/el_runtime.h index 6824b81..d126993 100644 --- a/lang/runtime/el_runtime.h +++ b/lang/runtime/el_runtime.h @@ -235,6 +235,20 @@ el_val_t fs_list(el_val_t path); el_val_t fs_exists(el_val_t path); el_val_t fs_mkdir(el_val_t path); /* mkdir -p, mode 0755 */ +/* Real on-disk byte count via stat() — not strlen(). Use this (not + * str_len(fs_read(path))) when a file may contain binary content, since + * fs_read()'s result truncates at the first embedded NUL under strlen-based + * string ops. Returns -1 if the path doesn't exist. */ +el_val_t fs_size(el_val_t path); + +/* Binary-safe windowed read: read up to `length` bytes starting at byte + * `offset` from `path` and return them base64-encoded. Bytes are read and + * encoded in C without ever passing through an el_val_t string as raw + * bytes, so embedded NULs (routine in PCM audio) can't truncate the result. + * Returns "" on any failure or when offset is past EOF; a final short + * window returns only the bytes that actually exist on disk. */ +el_val_t fs_read_b64_chunk(el_val_t path, el_val_t offset, el_val_t length); + /* Length-explicit binary write. `length` is an Int (el_val_t holding the * byte count). The caller knows the length from context — typically because * `bytes` came from base64_decode (which produces a magic-tagged binary From 3718bf03808810774ad05bee7e5d5e91e34cc101 Mon Sep 17 00:00:00 2001 From: bigmerge Date: Sat, 15 Aug 2026 17:50:04 -0500 Subject: [PATCH 5/5] runtime: port missing __channel_* primitives into el_seed.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runtime/channel.el has always called __channel_new/__channel_send/ __channel_recv/__channel_try_recv/__channel_close, but these were only ever implemented in the pre-restructure lang/el-compiler/runtime/el_runtime.c. When the canonical runtime was consolidated onto the release copy (lang/runtime/el_runtime.c) and el_seed.c became the sole C dependency, the channel implementation was never carried forward — __mutex_new made the move, __channel_* did not. Any El program using Go-style channels currently fails to link on dev. Ported the working buffered-MPMC-channel implementation (mutex+condvar+ circular buffer, bounded and unbounded modes) from the old el_runtime.c verbatim, adapted only to el_seed.c's arena API (seed_arena_track in place of el_arena_track). Declared in el_seed.h alongside the existing mutex primitives. --- lang/runtime/el_seed.c | 213 +++++++++++++++++++++++++++++++++++++++++ lang/runtime/el_seed.h | 7 ++ 2 files changed, 220 insertions(+) diff --git a/lang/runtime/el_seed.c b/lang/runtime/el_seed.c index 339be41..e30585d 100644 --- a/lang/runtime/el_seed.c +++ b/lang/runtime/el_seed.c @@ -907,6 +907,219 @@ void __mutex_unlock(el_val_t m) { pthread_mutex_unlock(&_el_mutexes[slot]); } +/* ── Channels ─────────────────────────────────────────────────────────────── * + * Buffered MPMC channel backed by a mutex + condvar + circular buffer. + * Ported from the pre-restructure el_runtime.c (b2aac4b) — runtime/channel.el + * has always called these five primitives, but they were never carried + * forward into el_seed.c when el_runtime.c was consolidated onto the + * canonical release copy. Native channels were silently unlinkable on dev + * until this port. + * + * __channel_new(capacity) -> Int (handle) + * __channel_send(ch, msg) — blocks if full (capacity > 0) or never (unbounded) + * __channel_recv(ch) -> String — blocks until a message is available + * __channel_try_recv(ch) -> String — non-blocking, returns "" if empty + * __channel_close(ch) — signal no more sends; recv drains remaining + * + * Bounded channels (cap > 0): circular buffer, sender blocks when full. + * Unbounded channels (cap == 0): dynamic array, sender never blocks. + */ +#define EL_CHANNEL_MAX 64 +#define EL_CHANNEL_BUF 1024 + +typedef struct { + char** buf; + int cap; /* 0 = unbounded (grows dynamically) */ + int head, tail, count; + int dyn_cap; /* allocated slots for unbounded mode */ + int closed; + pthread_mutex_t mu; + pthread_cond_t not_empty; + pthread_cond_t not_full; +} ElChannel; + +static ElChannel _channels[EL_CHANNEL_MAX]; +static int _channel_count = 0; +static pthread_mutex_t _channel_alloc_mu = PTHREAD_MUTEX_INITIALIZER; + +el_val_t __channel_new(el_val_t capacity_v) { + int cap = (int)(int64_t)capacity_v; + if (cap < 0) cap = 0; + + pthread_mutex_lock(&_channel_alloc_mu); + if (_channel_count >= EL_CHANNEL_MAX) { + pthread_mutex_unlock(&_channel_alloc_mu); + fprintf(stderr, "[__channel_new] channel table full\n"); + return EL_INT(-1); + } + int slot = _channel_count++; + pthread_mutex_unlock(&_channel_alloc_mu); + + ElChannel* ch = &_channels[slot]; + memset(ch, 0, sizeof(*ch)); + ch->cap = cap; + ch->closed = 0; + ch->head = 0; + ch->tail = 0; + ch->count = 0; + + if (cap > 0) { + /* Bounded: fixed circular buffer. */ + ch->buf = (char**)malloc((size_t)cap * sizeof(char*)); + ch->dyn_cap = cap; + } else { + /* Unbounded: start with EL_CHANNEL_BUF slots, grow as needed. */ + ch->buf = (char**)malloc(EL_CHANNEL_BUF * sizeof(char*)); + ch->dyn_cap = EL_CHANNEL_BUF; + } + if (!ch->buf) { + fprintf(stderr, "[__channel_new] out of memory\n"); + return EL_INT(-1); + } + + pthread_mutex_init(&ch->mu, NULL); + pthread_cond_init(&ch->not_empty, NULL); + pthread_cond_init(&ch->not_full, NULL); + + return EL_INT(slot); +} + +el_val_t __channel_send(el_val_t ch_v, el_val_t msg_v) { + int slot = (int)(int64_t)ch_v; + if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR(""); + ElChannel* ch = &_channels[slot]; + + const char* msg = EL_CSTR(msg_v); + if (!msg) msg = ""; + char* copy = strdup(msg); /* channel owns the string */ + + pthread_mutex_lock(&ch->mu); + + if (ch->closed) { + /* Send on closed channel is a no-op (drop the message). */ + pthread_mutex_unlock(&ch->mu); + free(copy); + return EL_STR(""); + } + + if (ch->cap > 0) { + /* Bounded: block while full. */ + while (ch->count >= ch->cap && !ch->closed) { + pthread_cond_wait(&ch->not_full, &ch->mu); + } + if (ch->closed) { + pthread_mutex_unlock(&ch->mu); + free(copy); + return EL_STR(""); + } + ch->buf[ch->tail] = copy; + ch->tail = (ch->tail + 1) % ch->cap; + ch->count++; + } else { + /* Unbounded: grow the buffer if needed. */ + if (ch->count >= ch->dyn_cap) { + int new_cap = ch->dyn_cap * 2; + char** grown = (char**)realloc(ch->buf, (size_t)new_cap * sizeof(char*)); + if (!grown) { + pthread_mutex_unlock(&ch->mu); + free(copy); + fprintf(stderr, "[__channel_send] out of memory growing channel\n"); + return EL_STR(""); + } + /* The circular buffer may have wrapped. Linearise it first. + * In unbounded mode head is always 0 (we append at tail, drain + * from head), so a simple memmove isn't needed — but if the + * buffer did wrap (tail < head after growth), we need to fix up. + * Simplest safe path: if tail wrapped, move the head..old_cap + * segment to new_cap..new_cap+(old_cap-head). */ + if (ch->tail < ch->head) { + /* Wrapped: [head..old_cap) is the front, [0..tail) is the back. */ + int front = ch->dyn_cap - ch->head; + memmove(grown + ch->dyn_cap, grown + ch->head, (size_t)front * sizeof(char*)); + ch->head = ch->dyn_cap; + } + ch->buf = grown; + ch->dyn_cap = new_cap; + } + ch->buf[ch->tail] = copy; + ch->tail = (ch->tail + 1) % ch->dyn_cap; + ch->count++; + } + + pthread_cond_signal(&ch->not_empty); + pthread_mutex_unlock(&ch->mu); + return EL_STR(""); +} + +el_val_t __channel_recv(el_val_t ch_v) { + int slot = (int)(int64_t)ch_v; + if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR(""); + ElChannel* ch = &_channels[slot]; + + pthread_mutex_lock(&ch->mu); + + /* Block until there is a message or the channel is closed and drained. */ + while (ch->count == 0 && !ch->closed) { + pthread_cond_wait(&ch->not_empty, &ch->mu); + } + + if (ch->count == 0) { + /* Closed and empty — signal EOF. */ + pthread_mutex_unlock(&ch->mu); + return EL_STR(""); + } + + int buf_cap = (ch->cap > 0) ? ch->cap : ch->dyn_cap; + char* msg = ch->buf[ch->head]; + ch->head = (ch->head + 1) % buf_cap; + ch->count--; + + pthread_cond_signal(&ch->not_full); + pthread_mutex_unlock(&ch->mu); + + /* Hand the string to the arena so it is freed after the request. */ + seed_arena_track(msg); + return EL_STR(msg); +} + +el_val_t __channel_try_recv(el_val_t ch_v) { + int slot = (int)(int64_t)ch_v; + if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR(""); + ElChannel* ch = &_channels[slot]; + + pthread_mutex_lock(&ch->mu); + + if (ch->count == 0) { + pthread_mutex_unlock(&ch->mu); + return EL_STR(""); + } + + int buf_cap = (ch->cap > 0) ? ch->cap : ch->dyn_cap; + char* msg = ch->buf[ch->head]; + ch->head = (ch->head + 1) % buf_cap; + ch->count--; + + pthread_cond_signal(&ch->not_full); + pthread_mutex_unlock(&ch->mu); + + seed_arena_track(msg); + return EL_STR(msg); +} + +el_val_t __channel_close(el_val_t ch_v) { + int slot = (int)(int64_t)ch_v; + if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR(""); + ElChannel* ch = &_channels[slot]; + + pthread_mutex_lock(&ch->mu); + ch->closed = 1; + /* Wake all blocked recvers and senders so they can observe the close. */ + pthread_cond_broadcast(&ch->not_empty); + pthread_cond_broadcast(&ch->not_full); + pthread_mutex_unlock(&ch->mu); + return EL_STR(""); +} + /* ── Subprocess ──────────────────────────────────────────────────────────── */ el_val_t __exec(el_val_t cmd) { diff --git a/lang/runtime/el_seed.h b/lang/runtime/el_seed.h index 7597511..799bb7e 100644 --- a/lang/runtime/el_seed.h +++ b/lang/runtime/el_seed.h @@ -139,6 +139,13 @@ el_val_t __mutex_new(void); void __mutex_lock(el_val_t m); void __mutex_unlock(el_val_t m); +/* Buffered MPMC channel (runtime/channel.el). capacity=0 means unbounded. */ +el_val_t __channel_new(el_val_t capacity); +el_val_t __channel_send(el_val_t ch, el_val_t msg); /* blocks if bounded+full */ +el_val_t __channel_recv(el_val_t ch); /* blocks until available */ +el_val_t __channel_try_recv(el_val_t ch); /* non-blocking, "" if empty */ +el_val_t __channel_close(el_val_t ch); + /* ── Subprocess ──────────────────────────────────────────────────────────── */ el_val_t __exec(el_val_t cmd); /* popen, capture all stdout, return String */