diff --git a/tools/neuron-sandbox/README.md b/tools/neuron-sandbox/README.md index 5dd776d..a9dd9cf 100644 --- a/tools/neuron-sandbox/README.md +++ b/tools/neuron-sandbox/README.md @@ -173,4 +173,7 @@ Each ad-hoc harness becomes `nsbx run …` (or `--source` build) against ## Env knobs `NSBX_ROOT`, `NSBX_PORT_BASE`, `NSBX_RSS_BOUND_MB`, `NSBX_REMERGE_THRESHOLD`, +`NSBX_READY_TIMEOUT_SECS` (default 15 — how long `up`/`create`/`build` wait for a +daemon to answer `/api/stats` before reporting failure; raise it if a boot is +legitimately slow under concurrent sandbox/CPU load rather than actually broken), `EL_REPO` (for `elc` + runtime sources), `ENGRAM_LIVE_DATA_DIR`, `ENGRAM_LIVE_PLIST`. diff --git a/tools/neuron-sandbox/nsbx b/tools/neuron-sandbox/nsbx index 249780a..ffeedeb 100755 --- a/tools/neuron-sandbox/nsbx +++ b/tools/neuron-sandbox/nsbx @@ -32,6 +32,12 @@ EL_REPO="${EL_REPO:-$HOME/Development/neuron-technologies/foundation/el}" PORT_BASE="${NSBX_PORT_BASE:-8900}" RSS_BOUND_MB="${NSBX_RSS_BOUND_MB:-550}" # from store-fix reboot-proof (aaf13f88) REMERGE_THRESHOLD="${NSBX_REMERGE_THRESHOLD:-40000}" +# readiness-poll window for start_daemon (0.5s ticks). Default unchanged (15s) — +# but a cold boot against the full live store, under concurrent CPU contention +# from other running sandboxes, has been observed live to take well past that. +# Bump per-invocation with NSBX_READY_TIMEOUT_SECS if `up`/`create` reports a +# not-ready failure but the daemon looks otherwise fine (see its logs/daemon.log). +READY_TICKS=$(( ${NSBX_READY_TIMEOUT_SECS:-15} * 2 )) KEYSTONES=( "kn-efeb4a5b-5aff-4759-8a97-7233099be6ee" "kn-5b606390-a52d-4ca2-8e0e-eba141d13440" ) # fixed probe set for retrieval-parity (stable, identity-anchored) PARITY_QUERIES=( "who am I" "self identity core" "engram store durability" "keystone self anchor" "grounding honesty" ) @@ -77,7 +83,7 @@ _port_claimed(){ # is another sandbox already assigned this port? live_stats(){ curl -s -m5 "$LIVE_URL/api/stats" 2>/dev/null; } api(){ # api [json-body] local name="$1" path="$2" body="${3:-}" - local port; port="$(mget "$name" "['port']")"; [ -n "$port" ] || die "unknown sandbox: $name" + local port; port="$(mget "$name" "['port']")"; [ -n "$port" ] || die "unknown sandbox: $name (run: nsbx list)" local url="http://127.0.0.1:${port}${path}" if [ -n "$body" ]; then curl -s -m30 -X POST -H 'Content-Type: application/json' -d "$body" "$url" else curl -s -m30 "$url"; fi @@ -88,6 +94,19 @@ stat_field(){ printf '%s' "$1" | sed -n "s/.*\"$2\":\([0-9]*\).*/\1/p"; } daemon_pid(){ local f; f="$(sdir "$1")/daemon.pid"; [ -f "$f" ] && cat "$f" || true; } daemon_alive(){ local p; p="$(daemon_pid "$1")"; [ -n "$p" ] && kill -0 "$p" 2>/dev/null; } +# daemon_health : prints "stopped" | "running" | "unresponsive" (to stdout). +# "running" means the pid is alive AND /api/stats actually answered — process +# liveness alone (daemon_alive) is not proof the HTTP server is serving; a pegged +# or hung process still passes kill -0. Short timeout (2s) since this runs per-row +# in `nsbx list`. +daemon_health(){ + local name="$1" + daemon_alive "$name" || { echo "stopped"; return 0; } + local port; port="$(mget "$name" "['port']")" + local s; s="$(curl -s -m2 "http://127.0.0.1:${port}/api/stats" 2>/dev/null)" + [ -n "$s" ] && echo "running" || echo "unresponsive" +} + # ---------------------------------------------------------------- elc/build ---- find_elc(){ command -v elc 2>/dev/null && return 0 @@ -123,6 +142,49 @@ _build_binary(){ ok "built: $out ($(ls -lh "$out" | awk '{print $5}'), sha $(sha "$out" | cut -c1-12))" } +# bin_built_at : human-readable build timestamp. `cp -p` preserves mtime, +# so this is the ORIGINAL build time even for binaries copied stock-prod into a +# sandbox — not the copy time. +bin_built_at(){ stat -f '%Sm' -t '%Y-%m-%d %H:%M:%S' "$1" 2>/dev/null || echo "unknown"; } + +# _binary_freshness : best-effort staleness note (empty string if fresh/ +# unknown — never guesses). Two cases: +# - stock-prod: compares the sha recorded at create time against the CURRENTLY +# configured live real binary's sha (recomputed now, not cached) — catches +# "live prod moved on since this sandbox was cloned". +# - branch/source/rebuilt: compares the recorded source_commit against the +# LOCAL origin/dev ref (no fetch — reads whatever the repo already has) — +# catches "built from a commit that predates current dev tip". +_binary_freshness(){ + local name="$1" src; src="$(mget "$name" "['source']")" + case "$src" in + stock-prod:*) + local live_bin cur_sha rec_sha + live_bin="$(_live_real_bin)" + [ -n "$live_bin" ] && [ -x "$live_bin" ] || return 0 + cur_sha="$(sha "$live_bin")"; rec_sha="$(mget "$name" "['binary_sha256']")" + [ -n "$cur_sha" ] && [ -n "$rec_sha" ] && [ "$cur_sha" != "$rec_sha" ] \ + && printf 'stale: live prod binary has moved on since this sandbox was cloned (live is now %s, sha %s) — nsbx build %s --binary %s to catch up' \ + "$(basename "$live_bin")" "${cur_sha:0:12}" "$name" "$live_bin" + ;; + branch:*|source:*|rebuilt:*) + local commit cur behind + commit="$(mget "$name" "['source_commit']")" + [ -n "$commit" ] || return 0 + cur="$(git -C "$EL_REPO" rev-parse origin/dev 2>/dev/null)" || return 0 + [ -n "$cur" ] && [ "$commit" != "$cur" ] || return 0 + git -C "$EL_REPO" merge-base --is-ancestor "$commit" "$cur" 2>/dev/null || return 0 + behind="$(git -C "$EL_REPO" rev-list --count "$commit..$cur" 2>/dev/null)" + printf 'stale: built from %s, %s commit(s) behind local origin/dev (%s) — nsbx build %s --branch origin/dev' \ + "${commit:0:12}" "${behind:-?}" "${cur:0:12}" "$name" + ;; + esac +} + +# _source_commit : best-effort git HEAD of a source tree used to build a +# sandbox binary, empty if not a git repo (e.g. a prebuilt --binary path has none). +_source_commit(){ git -C "$1" rev-parse HEAD 2>/dev/null || true; } + # ---------------------------------------------------------------- daemon ------- # start_daemon : boots the sandbox's real engram binary on its isolated # port against its cloned data dir, with the SAME auto-remerge net the live soul @@ -133,9 +195,9 @@ start_daemon(){ local port bin data export key port="$(mget "$name" "['port']")"; bin="$d/bin/engram"; data="$d/data" key="sbx-$name"; export="$data/.scan-export.reseed-clean.json" - [ -x "$bin" ] || die "sandbox binary missing: $bin" + [ -x "$bin" ] || die "sandbox binary missing: $bin (run: nsbx build $name --source DIR | --branch REF, or nsbx destroy $name && nsbx create $name to reclone stock-prod)" [ "$port" != "$LIVE_BIND_PORT" ] && [ "$port" != "$SOUL_PORT" ] || die "refusing forbidden port $port" - [ -f "$data/neuron.egm" ] || die "sandbox has no cloned store: $data/neuron.egm" + [ -f "$data/neuron.egm" ] || die "sandbox has no cloned store: $data/neuron.egm (data dir is corrupt/incomplete — run: nsbx destroy $name && nsbx create $name)" # HARD guard: never point a sandbox daemon at the live data dir. [ "$(cd "$data" && pwd -P)" != "$(cd "$LIVE_DATA_DIR" && pwd -P)" ] || die "refusing: sandbox data dir resolves to LIVE store" @@ -150,11 +212,11 @@ start_daemon(){ echo "$pid" > "$d/daemon.pid" # readiness poll local url="http://127.0.0.1:$port" i s - for i in $(seq 1 30); do + for i in $(seq 1 "$READY_TICKS"); do s="$(curl -s -m3 "$url/api/stats" 2>/dev/null)" [ -n "$s" ] && break; sleep 0.5 done - [ -n "$s" ] || { warn "daemon did not become ready (see $d/logs/daemon.log)"; return 1; } + [ -n "$s" ] || { warn "daemon did not become ready within ${NSBX_READY_TIMEOUT_SECS:-15}s (see $d/logs/daemon.log). pid $pid may still be alive and slow to boot under load — check: lsof -iTCP:$port -P, or retry with NSBX_READY_TIMEOUT_SECS=45"; return 1; } ok "ready pid=$pid boot-stats: $s" # auto-remerge net (idempotent): match live edge population if the export is present if [ -f "$export" ]; then @@ -231,33 +293,35 @@ cmd_create(){ info "live baseline stats: ${lstats:-}" # ---- determine + place the runtime binary (versioned into the snapshot) ---- - local source_desc live_bin + local source_desc live_bin source_commit="" live_bin="$(_live_real_bin)" if [ -n "$binpath" ]; then [ -x "$binpath" ] || die "not an executable binary: $binpath" cp -p "$binpath" "$d/bin/engram"; source_desc="prebuilt:$binpath" elif [ -n "$src" ]; then _build_binary "$src" "$d/bin/engram" "$d/build"; source_desc="source:$src" + source_commit="$(_source_commit "$src")" elif [ -n "$branch" ]; then log "worktree: $repo @ $branch -> $d/build/worktree" git -C "$repo" worktree add --detach "$d/build/worktree" "$branch" >/dev/null 2>&1 \ || die "git worktree add failed ($repo @ $branch)" _build_binary "$d/build/worktree" "$d/bin/engram" "$d/build"; source_desc="branch:$branch@$repo" + source_commit="$(_source_commit "$d/build/worktree")" else [ -x "$live_bin" ] || die "cannot resolve live ENGRAM_REAL_BIN: $live_bin" cp -p "$live_bin" "$d/bin/engram"; source_desc="stock-prod:$live_bin" fi local bin_sha; bin_sha="$(sha "$d/bin/engram")" - info "runtime: $source_desc (sha ${bin_sha:0:12})" + info "runtime: $source_desc (sha ${bin_sha:0:12}, built $(bin_built_at "$d/bin/engram"))" # ---- write manifest ---- - python3 - "$name" "$port" "$source_desc" "$bin_sha" "$egm_sha" "$base_nodes" "$base_edges" "$(sha "$live_bin" 2>/dev/null)" <<'PY' > "$(manifest "$name")" + python3 - "$name" "$port" "$source_desc" "$bin_sha" "$egm_sha" "$base_nodes" "$base_edges" "$(sha "$live_bin" 2>/dev/null)" "$source_commit" <<'PY' > "$(manifest "$name")" import json,sys,datetime -name,port,src,binsha,egmsha,bn,be,livebinsha=sys.argv[1:9] +name,port,src,binsha,egmsha,bn,be,livebinsha,source_commit=sys.argv[1:10] json.dump({ "name":name,"port":int(port),"created_at":datetime.datetime.now(datetime.timezone.utc).isoformat(), "source":src,"binary_sha256":binsha,"clone_egm_sha256":egmsha, - "live_binary_sha256":livebinsha, + "live_binary_sha256":livebinsha,"source_commit":source_commit, "live_baseline":{"node_count":int(bn or 0),"edge_count":int(be or 0)}, "keystones":["kn-efeb4a5b-5aff-4759-8a97-7233099be6ee","kn-5b606390-a52d-4ca2-8e0e-eba141d13440"] }, sys.stdout, indent=2) @@ -268,6 +332,17 @@ PY start_daemon "$name" || die "daemon failed to start" local sstats; sstats="$(sbx_stats "$name")" local sbn sbe; sbn="$(stat_field "$sstats" node_count)"; sbe="$(stat_field "$sstats" edge_count)" + # a boot immediately followed by an auto-remerge can leave the daemon briefly + # busy — retry rather than silently folding a 0/0 baseline into the manifest. + # `validate`'s zero-loss/reboot-prove checks compare current counts >= baseline, + # so a 0/0 baseline would make them trivially PASS regardless of real data loss. + local _bi + for _bi in 1 2 3 4 5; do + [ -n "$sbn" ] && [ "$sbn" != "0" ] && break + sleep 1 + sstats="$(sbx_stats "$name")"; sbn="$(stat_field "$sstats" node_count)"; sbe="$(stat_field "$sstats" edge_count)" + done + [ -z "$sbn" ] || [ "$sbn" = "0" ] && warn "sandbox stats still empty/zero after retries — recording sbx_baseline 0/0. This makes 'nsbx validate $name' zero-loss checks trivially pass; investigate before trusting a validate PASS: nsbx status $name" _capture_retrieval "$name" "$d/baseline/retrieval.json" # fold sandbox baseline into manifest python3 - "$(manifest "$name")" "$sbn" "$sbe" <<'PY' @@ -320,7 +395,12 @@ except Exception: print("[]")' 2>/dev/null)" # thereafter. Prod on :$LIVE_BIND_PORT/:$SOUL_PORT is unreachable from here by design. cmd_up(){ local name; if [ $# -gt 0 ] && [ "${1#-}" = "$1" ]; then name="$1"; shift; else name="${USER:-dev}-dev"; fi - if mexists "$name"; then daemon_alive "$name" || start_daemon "$name"; else cmd_create "$name" "$@"; fi + if mexists "$name"; then + daemon_alive "$name" || start_daemon "$name" \ + || die "daemon did not become ready — see $(sdir "$name")/logs/daemon.log (try: nsbx up $name again once you've checked the log)" + else + cmd_create "$name" "$@" + fi local port; port="$(mget "$name" "['port']")" echo >&2 ok "your sandbox '$name' is ready at http://127.0.0.1:$port (a private copy of the mind — prod is untouchable)" @@ -334,34 +414,37 @@ cmd_up(){ # it on the SAME clone + port (the code-change dev loop, in place). cmd_build(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" local src="" branch="" repo="$EL_REPO" while [ $# -gt 0 ]; do case "$1" in --source) src="$2"; shift 2;; --branch) branch="$2"; shift 2;; --repo) repo="$2"; shift 2;; *) die "unknown flag: $1";; esac; done local d; d="$(sdir "$name")" stop_daemon "$name" - if [ -n "$src" ]; then _build_binary "$src" "$d/bin/engram" "$d/build" + local source_commit="" + if [ -n "$src" ]; then _build_binary "$src" "$d/bin/engram" "$d/build"; source_commit="$(_source_commit "$src")" elif [ -n "$branch" ]; then rm -rf "$d/build/worktree" 2>/dev/null; git -C "$repo" worktree prune 2>/dev/null git -C "$repo" worktree add --detach "$d/build/worktree" "$branch" >/dev/null 2>&1 || die "worktree add failed" _build_binary "$d/build/worktree" "$d/bin/engram" "$d/build" + source_commit="$(_source_commit "$d/build/worktree")" else die "usage: nsbx build --source DIR | --branch REF [--repo R]"; fi # record new binary sha - python3 - "$(manifest "$name")" "$(sha "$d/bin/engram")" "${src:-branch:$branch}" <<'PY' -import json,sys; mf,s,src=sys.argv[1:4] -d=json.load(open(mf)); d["binary_sha256"]=s; d["source"]="rebuilt:"+src + python3 - "$(manifest "$name")" "$(sha "$d/bin/engram")" "${src:-branch:$branch}" "$source_commit" <<'PY' +import json,sys; mf,s,src,source_commit=sys.argv[1:5] +d=json.load(open(mf)); d["binary_sha256"]=s; d["source"]="rebuilt:"+src; d["source_commit"]=source_commit json.dump(d,open(mf,'w'),indent=2) PY - start_daemon "$name" + start_daemon "$name" \ + || die "rebuilt binary did not become ready — see $d/logs/daemon.log (the old binary is gone; fix the code and re-run nsbx build $name ...)" ok "rebuilt + restarted on :$(mget "$name" "['port']")" } # ================================================================ run ========== cmd_run(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" - daemon_alive "$name" || start_daemon "$name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" + daemon_alive "$name" || start_daemon "$name" || die "daemon not running and failed to start — see $(sdir "$name")/logs/daemon.log" local d port; d="$(sdir "$name")"; port="$(mget "$name" "['port']")" # direct API form: nsbx run api [json] if [ "${1:-}" = "api" ]; then @@ -395,8 +478,8 @@ cmd_run(){ # RSS bound; retrieval parity; keystone integrity. cmd_validate(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" - daemon_alive "$name" || start_daemon "$name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" + daemon_alive "$name" || start_daemon "$name" || die "daemon not running and failed to start — see $(sdir "$name")/logs/daemon.log" local d port key; d="$(sdir "$name")"; port="$(mget "$name" "['port']")"; key="sbx-$name" local url="http://127.0.0.1:$port" local bn be; bn="$(mget "$name" "['sbx_baseline']['node_count']")"; be="$(mget "$name" "['sbx_baseline']['edge_count']")" @@ -489,7 +572,7 @@ PY # Default is a DRY-RUN plan; requires --i-approve-prod-cutover to actually cut over. cmd_promote(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" local approve=0 do_data=0 while [ $# -gt 0 ]; do case "$1" in --i-approve-prod-cutover) approve=1; shift;; @@ -588,7 +671,7 @@ PY # ================================================================ destroy ====== cmd_destroy(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" local d; d="$(sdir "$name")" stop_daemon "$name" if [ -d "$d/build/worktree" ]; then @@ -604,22 +687,44 @@ cmd_destroy(){ # ================================================================ list/status == cmd_list(){ [ -d "$SBX_ROOT" ] || { echo "no sandboxes"; return 0; } - printf '%-16s %-6s %-8s %-9s %s\n' NAME PORT STATE PID SOURCE + printf '%-16s %-6s %-13s %-9s %-19s %s\n' NAME PORT STATE PID "BUILT" SOURCE local m for m in "$SBX_ROOT"/*/manifest.json; do [ -f "$m" ] || continue - local n p src pid state + local n p src pid state bpath built fresh n="$(python3 -c "import json;print(json.load(open('$m'))['name'])")" p="$(python3 -c "import json;print(json.load(open('$m'))['port'])")" src="$(python3 -c "import json;print(json.load(open('$m'))['source'])")" - pid="$(daemon_pid "$n")"; state="stopped"; daemon_alive "$n" && state="running" - printf '%-16s %-6s %-8s %-9s %s\n' "$n" "$p" "$state" "${pid:-–}" "$src" + pid="$(daemon_pid "$n")" + case "$(daemon_health "$n")" in + running) state="running";; + unresponsive) state="running(!resp)";; + *) state="stopped";; + esac + bpath="$(sdir "$n")/bin/engram"; built="$([ -f "$bpath" ] && bin_built_at "$bpath" || echo unknown)" + fresh="$(_binary_freshness "$n")"; [ -n "$fresh" ] && src="[STALE] $src" + printf '%-16s %-6s %-13s %-9s %-19s %s\n' "$n" "$p" "$state" "${pid:-–}" "$built" "$src" done + info "state 'running(!resp)' = process alive but /api/stats didn't answer — see: nsbx status " } cmd_status(){ - local name="$1"; mexists "$name" || die "no such sandbox: $name" + local name="$1"; mexists "$name" || die "no such sandbox: $name (run: nsbx list to see what exists)" python3 -m json.tool "$(manifest "$name")" - daemon_alive "$name" && echo "state: running (pid $(daemon_pid "$name")) stats: $(sbx_stats "$name")" || echo "state: stopped" + local bpath; bpath="$(sdir "$name")/bin/engram" + if [ -f "$bpath" ]; then + echo "binary: sha=$(sha "$bpath" | cut -c1-12) built=$(bin_built_at "$bpath")" + local fresh; fresh="$(_binary_freshness "$name")" + [ -n "$fresh" ] && printf '%s%s%s\n' "$C_YEL" "$fresh" "$C_0" + fi + case "$(daemon_health "$name")" in + running) + echo "state: running (pid $(daemon_pid "$name")) stats: $(sbx_stats "$name")";; + unresponsive) + printf '%sstate: running but NOT RESPONDING%s (pid %s) — process alive, /api/stats returned nothing.\n' "$C_RED" "$C_0" "$(daemon_pid "$name")" + info "check: tail -50 $(sdir "$name")/logs/daemon.log | next: kill -9 $(daemon_pid "$name") && nsbx up $name" + ;; + *) echo "state: stopped";; + esac [ -f "$(sdir "$name")/validate.json" ] && { echo "--- last validation ---"; python3 -m json.tool "$(sdir "$name")/validate.json"; } }