bigmerge
112bb2540f
Generalise the ad-hoc cog-arch (worktree+build+store-clone+C-tests) and store-fix (secondary soul + launchctl rails cutover) proto-sandboxes into one reproducible primitive: run experiments and code changes against the REAL engram runtime on an isolated snapshot of the live mind, with a gated promote-to-prod path. Dev environment as a primitive — any team member gets a private, isolated copy of the mind (separate port/store/process); prod on :8742/:7770 is untouchable from a sandbox. Wraps the real binary; never reimplements engram logic. Lifecycle: create/up (consistent store+WAL+config snapshot; place OR build the runtime from --source/--branch/--binary; boot on an isolated port) · build · run · validate (rails as checks: zero-loss under load+reboot, reboot-prove, RSS bound, retrieval parity, keystone integrity) · promote (gated rails cutover: snapshot-first, additive binary swap, bootout→settle-poll→bootstrap, verify, auto-rollback; never pkill/kickstart -k; dry-run unless approved) · destroy. Dogfooded: reproduced retrieval-parity 25/25 vs baseline and the cog-arch correspondence-loop known result (Brier 0.028648->0.000586, reboot-proven) and real-store reboot-prove at 10994-node scale, all inside a sandbox; prod untouched.
664 lines
31 KiB
Bash
Executable File
664 lines
31 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# nsbx — the Neuron Sandbox: a reproducible primitive for running experiments and
|
||
# code changes against the REAL engram runtime on an isolated snapshot of the live
|
||
# mind, with a gated promote-to-prod path built on the proven rails.
|
||
#
|
||
# It WRAPS the real engram binary — it never reimplements any engram logic. The only
|
||
# prod-touching op is `promote`, which is explicit, gated, and per-use approved.
|
||
#
|
||
# Generalises two proven proto-sandboxes:
|
||
# - the cog-arch build (isolated git worktree + build + clone of live .egm + real C tests)
|
||
# - the store-fix cutover (secondary soul + launchctl bootout->settle->bootstrap rails)
|
||
#
|
||
# Lifecycle: create -> [build] -> run -> validate -> promote(gated) -> destroy
|
||
#
|
||
# Rails (always): built offline; NEVER auto-promotes; never touches live :8742/:7770
|
||
# except READ for the snapshot and the gated promote; snapshot-first; honest measured
|
||
# reporting. Cutover is launchctl bootout -> settle-poll -> bootstrap ONLY —
|
||
# never pkill, never kickstart -k.
|
||
set -uo pipefail
|
||
|
||
# ---------------------------------------------------------------- constants ----
|
||
LIVE_DATA_DIR="${ENGRAM_LIVE_DATA_DIR:-$HOME/.neuron/engram}"
|
||
LIVE_PLIST="${ENGRAM_LIVE_PLIST:-$HOME/Library/LaunchAgents/ai.neuron.engram.plist}"
|
||
LIVE_LABEL="ai.neuron.engram"
|
||
LIVE_BIND_PORT=8742 # engram — FORBIDDEN for sandboxes
|
||
SOUL_PORT=7770 # soul — FORBIDDEN for sandboxes
|
||
LIVE_KEY="${ENGRAM_API_KEY:-ntn-user-2026}"
|
||
LIVE_URL="http://127.0.0.1:${LIVE_BIND_PORT}"
|
||
SBX_ROOT="${NSBX_ROOT:-$HOME/.neuron/sandboxes}"
|
||
BACKUP_ROOT="$HOME/.neuron/backups"
|
||
EL_REPO="${EL_REPO:-$HOME/Development/neuron-technologies/foundation/el}"
|
||
PORT_BASE="${NSBX_PORT_BASE:-8900}"
|
||
RSS_BOUND_MB="${NSBX_RSS_BOUND_MB:-550}" # from store-fix reboot-proof (aaf13f88)
|
||
REMERGE_THRESHOLD="${NSBX_REMERGE_THRESHOLD:-40000}"
|
||
KEYSTONES=( "kn-efeb4a5b-5aff-4759-8a97-7233099be6ee" "kn-5b606390-a52d-4ca2-8e0e-eba141d13440" )
|
||
# fixed probe set for retrieval-parity (stable, identity-anchored)
|
||
PARITY_QUERIES=( "who am I" "self identity core" "engram store durability" "keystone self anchor" "grounding honesty" )
|
||
|
||
C_RED=$'\033[31m'; C_GRN=$'\033[32m'; C_YEL=$'\033[33m'; C_DIM=$'\033[2m'; C_BLD=$'\033[1m'; C_0=$'\033[0m'
|
||
|
||
# ---------------------------------------------------------------- helpers ------
|
||
die(){ printf '%serror:%s %s\n' "$C_RED" "$C_0" "$*" >&2; exit 1; }
|
||
log(){ printf '%s==>%s %s\n' "$C_BLD" "$C_0" "$*" >&2; }
|
||
info(){ printf ' %s\n' "$*" >&2; }
|
||
ok(){ printf ' %s%s%s\n' "$C_GRN" "$*" "$C_0" >&2; }
|
||
warn(){ printf ' %s%s%s\n' "$C_YEL" "$*" "$C_0" >&2; }
|
||
need(){ command -v "$1" >/dev/null 2>&1 || die "missing dependency: $1"; }
|
||
now(){ date -u +%Y%m%dT%H%M%SZ; }
|
||
sha(){ shasum -a 256 "$1" 2>/dev/null | awk '{print $1}'; }
|
||
epoch(){ python3 -c 'import time;print(time.time())'; }
|
||
|
||
sdir(){ printf '%s/%s' "$SBX_ROOT" "$1"; }
|
||
manifest(){ printf '%s/manifest.json' "$(sdir "$1")"; }
|
||
mexists(){ [ -f "$(manifest "$1")" ]; }
|
||
mget(){ # mget <name> <jsonpath>
|
||
python3 -c "import json,sys; d=json.load(open('$(manifest "$1")')); print(d$2)" 2>/dev/null
|
||
}
|
||
|
||
port_free(){ ! (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; }
|
||
alloc_port(){
|
||
local p="$PORT_BASE"
|
||
while :; do
|
||
if [ "$p" = "$LIVE_BIND_PORT" ] || [ "$p" = "$SOUL_PORT" ]; then p=$((p+1)); continue; fi
|
||
if port_free "$p" && ! _port_claimed "$p"; then echo "$p"; return 0; fi
|
||
p=$((p+1)); [ "$p" -gt 9100 ] && die "no free sandbox port in range"
|
||
done
|
||
}
|
||
_port_claimed(){ # is another sandbox already assigned this port?
|
||
local p="$1" d
|
||
for d in "$SBX_ROOT"/*/manifest.json; do
|
||
[ -f "$d" ] || continue
|
||
[ "$(python3 -c "import json;print(json.load(open('$d'))['port'])" 2>/dev/null)" = "$p" ] && return 0
|
||
done
|
||
return 1
|
||
}
|
||
|
||
live_stats(){ curl -s -m5 "$LIVE_URL/api/stats" 2>/dev/null; }
|
||
api(){ # api <name> <path> [json-body]
|
||
local name="$1" path="$2" body="${3:-}"
|
||
local port; port="$(mget "$name" "['port']")"; [ -n "$port" ] || die "unknown sandbox: $name"
|
||
local url="http://127.0.0.1:${port}${path}"
|
||
if [ -n "$body" ]; then curl -s -m30 -X POST -H 'Content-Type: application/json' -d "$body" "$url"
|
||
else curl -s -m30 "$url"; fi
|
||
}
|
||
sbx_stats(){ api "$1" "/api/stats"; }
|
||
stat_field(){ printf '%s' "$1" | sed -n "s/.*\"$2\":\([0-9]*\).*/\1/p"; }
|
||
|
||
daemon_pid(){ local f; f="$(sdir "$1")/daemon.pid"; [ -f "$f" ] && cat "$f" || true; }
|
||
daemon_alive(){ local p; p="$(daemon_pid "$1")"; [ -n "$p" ] && kill -0 "$p" 2>/dev/null; }
|
||
|
||
# ---------------------------------------------------------------- elc/build ----
|
||
find_elc(){
|
||
command -v elc 2>/dev/null && return 0
|
||
local arch; arch="$(uname -m)"
|
||
case "$arch" in
|
||
arm64) echo "$EL_REPO/lang/dist/platform/elc-darwin-arm64";;
|
||
x86_64) echo "$EL_REPO/lang/dist/platform/elc-linux-amd64";;
|
||
*) echo "$EL_REPO/lang/dist/platform/elc";;
|
||
esac
|
||
}
|
||
|
||
# _build_binary <src_tree> <out_bin> <build_log_dir>
|
||
# Replicates the proven engram release recipe:
|
||
# elc engram/src/server.el > engram.c
|
||
# cc -std=c11 -O2 -I lang/runtime engram.c el_runtime.c engram_*.c -lcurl -lpthread
|
||
_build_binary(){
|
||
local src="$1" out="$2" blog="$3"
|
||
local elc server rt
|
||
elc="$(find_elc)"; [ -x "$elc" ] || die "elc not found/executable: $elc (set EL_REPO)"
|
||
server="$src/engram/src/server.el"; rt="$src/lang/runtime"
|
||
[ -f "$server" ] || die "no engram/src/server.el under source tree: $src"
|
||
[ -f "$rt/el_runtime.c" ] || die "no lang/runtime/el_runtime.c under source tree: $src (this branch may keep it generated/untracked)"
|
||
ls "$rt"/engram_*.c >/dev/null 2>&1 || die "no lang/runtime/engram_*.c engine sources under: $src"
|
||
mkdir -p "$blog"
|
||
log "build: elc transpile server.el -> engram.c"
|
||
"$elc" "$server" > "$blog/engram.c" 2>"$blog/elc.err" || { cat "$blog/elc.err" >&2; die "elc transpile failed"; }
|
||
info "engram.c: $(wc -c <"$blog/engram.c" | tr -d ' ') bytes"
|
||
log "build: cc link (el_runtime + engram_* engine)"
|
||
cc -std=c11 -O2 -w -I "$rt" -o "$out" \
|
||
"$blog/engram.c" "$rt/el_runtime.c" "$rt"/engram_*.c \
|
||
-lcurl -lpthread 2>"$blog/cc.err" \
|
||
|| { grep -i 'error:' "$blog/cc.err" | sort -u | head >&2; die "cc link failed (see $blog/cc.err)"; }
|
||
ok "built: $out ($(ls -lh "$out" | awk '{print $5}'), sha $(sha "$out" | cut -c1-12))"
|
||
}
|
||
|
||
# ---------------------------------------------------------------- daemon -------
|
||
# start_daemon <name> : boots the sandbox's real engram binary on its isolated
|
||
# port against its cloned data dir, with the SAME auto-remerge net the live soul
|
||
# uses (so the sandbox faithfully reaches the live edge population on boot).
|
||
start_daemon(){
|
||
local name="$1" d; d="$(sdir "$name")"
|
||
daemon_alive "$name" && { info "already running (pid $(daemon_pid "$name"))"; return 0; }
|
||
local port bin data export key
|
||
port="$(mget "$name" "['port']")"; bin="$d/bin/engram"; data="$d/data"
|
||
key="sbx-$name"; export="$data/.scan-export.reseed-clean.json"
|
||
[ -x "$bin" ] || die "sandbox binary missing: $bin"
|
||
[ "$port" != "$LIVE_BIND_PORT" ] && [ "$port" != "$SOUL_PORT" ] || die "refusing forbidden port $port"
|
||
[ -f "$data/neuron.egm" ] || die "sandbox has no cloned store: $data/neuron.egm"
|
||
# HARD guard: never point a sandbox daemon at the live data dir.
|
||
[ "$(cd "$data" && pwd -P)" != "$(cd "$LIVE_DATA_DIR" && pwd -P)" ] || die "refusing: sandbox data dir resolves to LIVE store"
|
||
|
||
log "boot engram on isolated :$port (data=$data)"
|
||
(
|
||
ENGRAM_DATA_DIR="$data" ENGRAM_BIND=":$port" ENGRAM_API_KEY="$key" \
|
||
ENGRAM_STORE=1 ENGRAM_CHRONOCEPTION=1 ENGRAM_SELF_REIFY=1 ENGRAM_GC=1 \
|
||
ENGRAM_POOL_FRAMES=16384 ENGRAM_WRITE_BARRIER=1 \
|
||
exec "$bin"
|
||
) >"$d/logs/daemon.log" 2>&1 &
|
||
local pid=$!
|
||
echo "$pid" > "$d/daemon.pid"
|
||
# readiness poll
|
||
local url="http://127.0.0.1:$port" i s
|
||
for i in $(seq 1 30); do
|
||
s="$(curl -s -m3 "$url/api/stats" 2>/dev/null)"
|
||
[ -n "$s" ] && break; sleep 0.5
|
||
done
|
||
[ -n "$s" ] || { warn "daemon did not become ready (see $d/logs/daemon.log)"; return 1; }
|
||
ok "ready pid=$pid boot-stats: $s"
|
||
# auto-remerge net (idempotent): match live edge population if the export is present
|
||
if [ -f "$export" ]; then
|
||
local edges; edges="$(stat_field "$s" edge_count)"
|
||
if [ -n "$edges" ] && [ "$edges" -lt "$REMERGE_THRESHOLD" ]; then
|
||
log "auto-remerge: booted with $edges edges (< $REMERGE_THRESHOLD) — merging full edge export"
|
||
local r; r="$(curl -s -m300 -X POST -H 'Content-Type: application/json' \
|
||
-d "{\"_auth\":\"$key\",\"path\":\"$export\"}" "$url/api/load-merge" 2>/dev/null)"
|
||
info "remerge resp: ${r:0:120}"
|
||
ok "post-remerge stats: $(curl -s -m5 "$url/api/stats")"
|
||
fi
|
||
fi
|
||
return 0
|
||
}
|
||
|
||
# stop_daemon <name> : graceful TERM + settle-poll until the port is free.
|
||
# (Sandbox daemons are plain supervised bg processes — not launchd — so teardown
|
||
# is a signal + poll, never pkill of anything else.)
|
||
stop_daemon(){
|
||
local name="$1" pid port
|
||
pid="$(daemon_pid "$name")"; port="$(mget "$name" "['port']")"
|
||
[ -n "$pid" ] || { info "not running"; return 0; }
|
||
log "stop daemon pid=$pid, settle-poll until :$port frees"
|
||
kill "$pid" 2>/dev/null || true
|
||
local i
|
||
for i in $(seq 1 40); do
|
||
kill -0 "$pid" 2>/dev/null || { port_free "$port" && { ok "stopped, port $port free"; : >"$(sdir "$name")/daemon.pid"; return 0; }; }
|
||
printf '.' >&2; sleep 0.5
|
||
done
|
||
printf '\n' >&2
|
||
kill -9 "$pid" 2>/dev/null || true; sleep 1
|
||
: >"$(sdir "$name")/daemon.pid"
|
||
port_free "$port" && ok "stopped (after SIGKILL), port $port free" || warn "port $port still busy"
|
||
}
|
||
|
||
# ================================================================ create =======
|
||
cmd_create(){
|
||
local name="" port="" src="" branch="" repo="$EL_REPO" binpath=""
|
||
# first positional arg is the name unless it's a flag; default to "<user>-dev"
|
||
if [ $# -gt 0 ] && [ "${1#-}" = "$1" ]; then name="$1"; shift; else name="${USER:-dev}-dev"; fi
|
||
while [ $# -gt 0 ]; do case "$1" in
|
||
--port) port="$2"; shift 2;;
|
||
--source) src="$2"; shift 2;;
|
||
--branch) branch="$2"; shift 2;;
|
||
--repo) repo="$2"; shift 2;;
|
||
--binary) binpath="$2"; shift 2;;
|
||
*) die "unknown flag: $1";;
|
||
esac; done
|
||
mexists "$name" && die "sandbox '$name' already exists (destroy it first)"
|
||
need curl; need python3; need shasum
|
||
[ -f "$LIVE_DATA_DIR/neuron.egm" ] || die "live store not found: $LIVE_DATA_DIR/neuron.egm"
|
||
if [ -n "$port" ]; then
|
||
{ [ "$port" = "$LIVE_BIND_PORT" ] || [ "$port" = "$SOUL_PORT" ]; } && die "refusing forbidden port $port (live)"
|
||
port_free "$port" || die "port $port already in use"
|
||
else port="$(alloc_port)"; fi
|
||
|
||
local d; d="$(sdir "$name")"
|
||
mkdir -p "$d/data" "$d/bin" "$d/logs" "$d/build" "$d/baseline"
|
||
log "sandbox '$name' at $d (isolated port $port)"
|
||
|
||
# ---- CONSISTENT snapshot of the live mind (file-copy: same set the rails backup
|
||
# uses; WAL replay on sandbox boot reconciles the tail -> crash-consistent) ----
|
||
log "snapshot live store -> clone (store + WAL + config)"
|
||
local f
|
||
for f in neuron.egm neuron.wal conf meta.json self_anchor .scan-export.reseed-clean.json; do
|
||
if [ -e "$LIVE_DATA_DIR/$f" ]; then cp -p "$LIVE_DATA_DIR/$f" "$d/data/$f"; info "cloned $f ($(du -h "$d/data/$f" | awk '{print $1}'))"; fi
|
||
done
|
||
local egm_sha; egm_sha="$(sha "$d/data/neuron.egm")"
|
||
|
||
# ---- capture live baseline (READ only) ----
|
||
local lstats; lstats="$(live_stats)"
|
||
local base_nodes base_edges
|
||
base_nodes="$(stat_field "$lstats" node_count)"; base_edges="$(stat_field "$lstats" edge_count)"
|
||
info "live baseline stats: ${lstats:-<unavailable>}"
|
||
|
||
# ---- determine + place the runtime binary (versioned into the snapshot) ----
|
||
local source_desc live_bin
|
||
live_bin="$(_live_real_bin)"
|
||
if [ -n "$binpath" ]; then
|
||
[ -x "$binpath" ] || die "not an executable binary: $binpath"
|
||
cp -p "$binpath" "$d/bin/engram"; source_desc="prebuilt:$binpath"
|
||
elif [ -n "$src" ]; then
|
||
_build_binary "$src" "$d/bin/engram" "$d/build"; source_desc="source:$src"
|
||
elif [ -n "$branch" ]; then
|
||
log "worktree: $repo @ $branch -> $d/build/worktree"
|
||
git -C "$repo" worktree add --detach "$d/build/worktree" "$branch" >/dev/null 2>&1 \
|
||
|| die "git worktree add failed ($repo @ $branch)"
|
||
_build_binary "$d/build/worktree" "$d/bin/engram" "$d/build"; source_desc="branch:$branch@$repo"
|
||
else
|
||
[ -x "$live_bin" ] || die "cannot resolve live ENGRAM_REAL_BIN: $live_bin"
|
||
cp -p "$live_bin" "$d/bin/engram"; source_desc="stock-prod:$live_bin"
|
||
fi
|
||
local bin_sha; bin_sha="$(sha "$d/bin/engram")"
|
||
info "runtime: $source_desc (sha ${bin_sha:0:12})"
|
||
|
||
# ---- write manifest ----
|
||
python3 - "$name" "$port" "$source_desc" "$bin_sha" "$egm_sha" "$base_nodes" "$base_edges" "$(sha "$live_bin" 2>/dev/null)" <<'PY' > "$(manifest "$name")"
|
||
import json,sys,datetime
|
||
name,port,src,binsha,egmsha,bn,be,livebinsha=sys.argv[1:9]
|
||
json.dump({
|
||
"name":name,"port":int(port),"created_at":datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||
"source":src,"binary_sha256":binsha,"clone_egm_sha256":egmsha,
|
||
"live_binary_sha256":livebinsha,
|
||
"live_baseline":{"node_count":int(bn or 0),"edge_count":int(be or 0)},
|
||
"keystones":["kn-efeb4a5b-5aff-4759-8a97-7233099be6ee","kn-5b606390-a52d-4ca2-8e0e-eba141d13440"]
|
||
}, sys.stdout, indent=2)
|
||
PY
|
||
ok "manifest written"
|
||
|
||
# ---- boot + capture the sandbox's own settled baseline (reproducible target) ----
|
||
start_daemon "$name" || die "daemon failed to start"
|
||
local sstats; sstats="$(sbx_stats "$name")"
|
||
local sbn sbe; sbn="$(stat_field "$sstats" node_count)"; sbe="$(stat_field "$sstats" edge_count)"
|
||
_capture_retrieval "$name" "$d/baseline/retrieval.json"
|
||
# fold sandbox baseline into manifest
|
||
python3 - "$(manifest "$name")" "$sbn" "$sbe" <<'PY'
|
||
import json,sys
|
||
mf,bn,be=sys.argv[1],sys.argv[2],sys.argv[3]
|
||
d=json.load(open(mf)); d["sbx_baseline"]={"node_count":int(bn or 0),"edge_count":int(be or 0)}
|
||
json.dump(d,open(mf,'w'),indent=2)
|
||
PY
|
||
log "created."
|
||
info "sandbox baseline (settled): nodes=$sbn edges=$sbe"
|
||
info "next: nsbx validate $name | nsbx run $name api /api/stats"
|
||
}
|
||
|
||
_live_real_bin(){
|
||
python3 - "$LIVE_PLIST" <<'PY' 2>/dev/null
|
||
import sys,plistlib
|
||
try:
|
||
d=plistlib.load(open(sys.argv[1],'rb'))
|
||
print(d.get("EnvironmentVariables",{}).get("ENGRAM_REAL_BIN",""))
|
||
except Exception: print("")
|
||
PY
|
||
}
|
||
|
||
_capture_retrieval(){ # <name> <outfile> : top-k ids for the fixed probe set
|
||
local name="$1" out="$2" q res
|
||
local port; port="$(mget "$name" "['port']")"; local key="sbx-$name"
|
||
{
|
||
echo "{"
|
||
local first=1
|
||
for q in "${PARITY_QUERIES[@]}"; do
|
||
res="$(curl -s -m10 -X POST -H 'Content-Type: application/json' \
|
||
-d "{\"_auth\":\"$key\",\"query\":\"$q\",\"limit\":5}" "http://127.0.0.1:$port/api/search" 2>/dev/null)"
|
||
local ids; ids="$(printf '%s' "$res" | python3 -c 'import sys,json
|
||
try:
|
||
d=json.load(sys.stdin)
|
||
rows=d if isinstance(d,list) else d.get("results",d.get("hits",[]))
|
||
print(json.dumps([r.get("id") for r in rows][:5]))
|
||
except Exception: print("[]")' 2>/dev/null)"
|
||
[ $first -eq 1 ] || echo ","; first=0
|
||
printf ' %s: %s' "$(python3 -c "import json,sys;print(json.dumps(sys.argv[1]))" "$q")" "${ids:-[]}"
|
||
done
|
||
echo ""; echo "}"
|
||
} > "$out"
|
||
}
|
||
|
||
# ================================================================ up ===========
|
||
# Dead-simple one-command dev environment: `nsbx up` gives you (or Tim, or anyone)
|
||
# a private, isolated copy of the live mind to build against. Creates it on first
|
||
# run with sane defaults (stock prod binary, auto-allocated port), just starts it
|
||
# thereafter. Prod on :$LIVE_BIND_PORT/:$SOUL_PORT is unreachable from here by design.
|
||
cmd_up(){
|
||
local name; if [ $# -gt 0 ] && [ "${1#-}" = "$1" ]; then name="$1"; shift; else name="${USER:-dev}-dev"; fi
|
||
if mexists "$name"; then daemon_alive "$name" || start_daemon "$name"; else cmd_create "$name" "$@"; fi
|
||
local port; port="$(mget "$name" "['port']")"
|
||
echo >&2
|
||
ok "your sandbox '$name' is ready at http://127.0.0.1:$port (a private copy of the mind — prod is untouchable)"
|
||
info "experiment: nsbx run $name api /api/stats"
|
||
info "prove it: nsbx validate $name"
|
||
info "tear down: nsbx destroy $name"
|
||
}
|
||
|
||
# ================================================================ build ========
|
||
# Rebuild an existing sandbox's runtime from a source tree/branch and hot-restart
|
||
# it on the SAME clone + port (the code-change dev loop, in place).
|
||
cmd_build(){
|
||
local name="$1"; shift || true
|
||
mexists "$name" || die "no such sandbox: $name"
|
||
local src="" branch="" repo="$EL_REPO"
|
||
while [ $# -gt 0 ]; do case "$1" in
|
||
--source) src="$2"; shift 2;; --branch) branch="$2"; shift 2;; --repo) repo="$2"; shift 2;;
|
||
*) die "unknown flag: $1";; esac; done
|
||
local d; d="$(sdir "$name")"
|
||
stop_daemon "$name"
|
||
if [ -n "$src" ]; then _build_binary "$src" "$d/bin/engram" "$d/build"
|
||
elif [ -n "$branch" ]; then
|
||
rm -rf "$d/build/worktree" 2>/dev/null; git -C "$repo" worktree prune 2>/dev/null
|
||
git -C "$repo" worktree add --detach "$d/build/worktree" "$branch" >/dev/null 2>&1 || die "worktree add failed"
|
||
_build_binary "$d/build/worktree" "$d/bin/engram" "$d/build"
|
||
else die "usage: nsbx build <name> --source DIR | --branch REF [--repo R]"; fi
|
||
# record new binary sha
|
||
python3 - "$(manifest "$name")" "$(sha "$d/bin/engram")" "${src:-branch:$branch}" <<'PY'
|
||
import json,sys; mf,s,src=sys.argv[1:4]
|
||
d=json.load(open(mf)); d["binary_sha256"]=s; d["source"]="rebuilt:"+src
|
||
json.dump(d,open(mf,'w'),indent=2)
|
||
PY
|
||
start_daemon "$name"
|
||
ok "rebuilt + restarted on :$(mget "$name" "['port']")"
|
||
}
|
||
|
||
# ================================================================ run ==========
|
||
cmd_run(){
|
||
local name="$1"; shift || true
|
||
mexists "$name" || die "no such sandbox: $name"
|
||
daemon_alive "$name" || start_daemon "$name"
|
||
local d port; d="$(sdir "$name")"; port="$(mget "$name" "['port']")"
|
||
# direct API form: nsbx run <name> api <path> [json]
|
||
if [ "${1:-}" = "api" ]; then
|
||
api "$name" "$2" "${3:-}"; echo; return 0
|
||
fi
|
||
[ "${1:-}" = "--" ] && shift # allow an explicit separator: nsbx run <name> -- <cmd...>
|
||
[ $# -gt 0 ] || die "usage: nsbx run <name> <cmd...> | nsbx run <name> api <path> [json]"
|
||
local ts log0; ts="$(now)"; log0="$d/logs/run-$ts.log"
|
||
local s0 t0 t1 s1
|
||
s0="$(sbx_stats "$name")"; t0="$(epoch)"
|
||
log "run experiment against sandbox '$name' (:$port)"
|
||
info "cmd: $*"
|
||
( export SBX_NAME="$name" SBX_PORT="$port" SBX_URL="http://127.0.0.1:$port" \
|
||
SBX_KEY="sbx-$name" SBX_DATA="$d/data" SBX_BIN="$d/bin/engram"
|
||
"$@" ) 2>&1 | tee "$log0"
|
||
local rc=${PIPESTATUS[0]}
|
||
t1="$(epoch)"; s1="$(sbx_stats "$name")"
|
||
{
|
||
echo "--- nsbx run metrics ---"
|
||
echo "exit_code: $rc"
|
||
printf 'wall_secs: %.3f\n' "$(python3 -c "print($t1-$t0)")"
|
||
echo "stats_before: $s0"
|
||
echo "stats_after: $s1"
|
||
} | tee -a "$log0" >&2
|
||
return $rc
|
||
}
|
||
|
||
# ================================================================ validate =====
|
||
# The rails as first-class checks. Baseline = the sandbox's own settled state at
|
||
# create (reproducible). zero-loss through sustained load AND reboot; reboot-prove;
|
||
# RSS bound; retrieval parity; keystone integrity.
|
||
cmd_validate(){
|
||
local name="$1"; shift || true
|
||
mexists "$name" || die "no such sandbox: $name"
|
||
daemon_alive "$name" || start_daemon "$name"
|
||
local d port key; d="$(sdir "$name")"; port="$(mget "$name" "['port']")"; key="sbx-$name"
|
||
local url="http://127.0.0.1:$port"
|
||
local bn be; bn="$(mget "$name" "['sbx_baseline']['node_count']")"; be="$(mget "$name" "['sbx_baseline']['edge_count']")"
|
||
log "validate '$name' against baseline nodes=$bn edges=$be"
|
||
local -a names=() results=() details=()
|
||
|
||
# 1) sustained load — no data loss under activity
|
||
local s cur_n cur_e i
|
||
log "check: sustained load (~15s: tick + reads) then zero-loss"
|
||
for i in $(seq 1 15); do
|
||
curl -s -m5 -X POST -H 'Content-Type: application/json' -d "{\"_auth\":\"$key\"}" "$url/api/tick" >/dev/null 2>&1
|
||
curl -s -m5 "$url/api/stats" >/dev/null 2>&1
|
||
done
|
||
s="$(sbx_stats "$name")"; cur_n="$(stat_field "$s" node_count)"; cur_e="$(stat_field "$s" edge_count)"
|
||
names+=("zero-loss-under-load"); if [ "${cur_n:-0}" -ge "${bn:-0}" ] && [ "${cur_e:-0}" -ge "${be:-0}" ]; then
|
||
results+=("PASS"); else results+=("FAIL"); fi
|
||
details+=("nodes $cur_n>=$bn, edges $cur_e>=$be")
|
||
|
||
# 2) reboot-prove — counts survive a real restart
|
||
log "check: reboot-prove (stop -> start -> compare)"
|
||
local pre_n pre_e; pre_n="$cur_n"; pre_e="$cur_e"
|
||
stop_daemon "$name"; start_daemon "$name" >/dev/null
|
||
s="$(sbx_stats "$name")"; cur_n="$(stat_field "$s" node_count)"; cur_e="$(stat_field "$s" edge_count)"
|
||
names+=("reboot-prove"); if [ "${cur_n:-0}" -ge "${bn:-0}" ] && [ "${cur_e:-0}" -ge "${be:-0}" ]; then
|
||
results+=("PASS"); else results+=("FAIL"); fi
|
||
details+=("post-reboot nodes=$cur_n edges=$cur_e (pre $pre_n/$pre_e)")
|
||
|
||
# 3) RSS bound
|
||
log "check: RSS bound (< ${RSS_BOUND_MB}MB)"
|
||
local pid rss_kb rss_mb; pid="$(daemon_pid "$name")"
|
||
rss_kb="$(ps -o rss= -p "$pid" 2>/dev/null | tr -d ' ')"; rss_mb=$(( ${rss_kb:-0} / 1024 ))
|
||
names+=("rss-bound"); if [ "$rss_mb" -lt "$RSS_BOUND_MB" ] && [ "$rss_mb" -gt 0 ]; then results+=("PASS"); else results+=("FAIL"); fi
|
||
details+=("RSS=${rss_mb}MB (bound ${RSS_BOUND_MB}MB)")
|
||
|
||
# 4) retrieval parity vs the create-time baseline
|
||
log "check: retrieval parity vs baseline probe set"
|
||
_capture_retrieval "$name" "$d/logs/retrieval-$( now ).json"
|
||
local latest; latest="$(ls -t "$d/logs"/retrieval-*.json 2>/dev/null | head -1)"
|
||
local parity; parity="$(python3 - "$d/baseline/retrieval.json" "$latest" <<'PY'
|
||
import json,sys
|
||
def load(p):
|
||
try: return json.load(open(p))
|
||
except Exception: return {}
|
||
b,c=load(sys.argv[1]),load(sys.argv[2])
|
||
tot=hit=0
|
||
for q,ids in b.items():
|
||
cb=set(ids or []); cc=set(c.get(q) or [])
|
||
if not cb: continue
|
||
tot+=len(cb); hit+=len(cb & cc)
|
||
print(f"{hit}/{tot}" if tot else "0/0")
|
||
PY
|
||
)"
|
||
local ph="${parity%/*}" pt="${parity#*/}"
|
||
names+=("retrieval-parity"); if [ "${pt:-0}" -gt 0 ] && [ "${ph:-0}" -eq "${pt:-0}" ]; then results+=("PASS"); else results+=("FAIL"); fi
|
||
details+=("top-k id overlap $parity vs baseline")
|
||
|
||
# 5) keystone integrity
|
||
log "check: keystone integrity"
|
||
local kfail=0 kid kres
|
||
for kid in "${KEYSTONES[@]}"; do
|
||
kres="$(curl -s -m5 "$url/api/node/$kid" 2>/dev/null)"
|
||
printf '%s' "$kres" | grep -q "\"$kid\"" || kfail=1
|
||
done
|
||
names+=("keystone-integrity"); [ "$kfail" -eq 0 ] && results+=("PASS") || results+=("FAIL")
|
||
details+=("kn-efeb4a5b + kn-5b606390 present")
|
||
|
||
# ---- report + stamp ----
|
||
echo >&2
|
||
printf '%s VALIDATION — %s%s\n' "$C_BLD" "$name" "$C_0" >&2
|
||
local allpass=1 j
|
||
for j in "${!names[@]}"; do
|
||
local r="${results[$j]}" c="$C_GRN"; [ "$r" = FAIL ] && { c="$C_RED"; allpass=0; }
|
||
printf ' %s%-6s%s %-22s %s%s%s\n' "$c" "$r" "$C_0" "${names[$j]}" "$C_DIM" "${details[$j]}" "$C_0" >&2
|
||
done
|
||
local status; [ "$allpass" -eq 1 ] && status="PASS" || status="FAIL"
|
||
python3 - "$d/validate.json" "$status" "$(sha "$d/bin/engram")" "$(now)" "${names[*]}" "${results[*]}" <<'PY'
|
||
import json,sys
|
||
out,status,binsha,ts,ns,rs=sys.argv[1:7]
|
||
checks=[{"name":n,"result":r} for n,r in zip(ns.split(),rs.split())]
|
||
json.dump({"status":status,"binary_sha256":binsha,"ts":ts,"checks":checks},open(out,'w'),indent=2)
|
||
PY
|
||
printf ' %s==> %s%s\n' "$([ "$allpass" -eq 1 ] && echo "$C_GRN" || echo "$C_RED")" "$status" "$C_0" >&2
|
||
[ "$allpass" -eq 1 ]
|
||
}
|
||
|
||
# ================================================================ promote ======
|
||
# The ONLY prod-touching op. Explicit, gated, per-use Will-approved. Rails ONLY:
|
||
# snapshot-first -> additive binary swap -> launchctl bootout -> settle-poll ->
|
||
# bootstrap -> verify -> auto-rollback on failure. NEVER pkill, NEVER kickstart -k.
|
||
# Default is a DRY-RUN plan; requires --i-approve-prod-cutover to actually cut over.
|
||
cmd_promote(){
|
||
local name="$1"; shift || true
|
||
mexists "$name" || die "no such sandbox: $name"
|
||
local approve=0 do_data=0
|
||
while [ $# -gt 0 ]; do case "$1" in
|
||
--i-approve-prod-cutover) approve=1; shift;;
|
||
--data) do_data=1; shift;;
|
||
*) die "unknown flag: $1";; esac; done
|
||
local d; d="$(sdir "$name")"
|
||
# GATE 1: validation must have passed for the CURRENT binary
|
||
[ -f "$d/validate.json" ] || die "GATE: no validation on record — run 'nsbx validate $name' first"
|
||
local vstatus vsha bsha
|
||
vstatus="$(python3 -c "import json;print(json.load(open('$d/validate.json'))['status'])")"
|
||
vsha="$(python3 -c "import json;print(json.load(open('$d/validate.json'))['binary_sha256'])")"
|
||
bsha="$(sha "$d/bin/engram")"
|
||
[ "$vstatus" = PASS ] || die "GATE: last validation status is $vstatus (must be PASS)"
|
||
[ "$vsha" = "$bsha" ] || die "GATE: validation is stale — binary changed since validate (re-run validate)"
|
||
|
||
local live_bin new_bin ts; ts="$(now)"
|
||
live_bin="$(_live_real_bin)"
|
||
new_bin="$HOME/.neuron/bin/engram.promote-$name-$ts" # additive: new file, old kept
|
||
local bkp="$BACKUP_ROOT/promote-$name-$ts"
|
||
|
||
log "PROMOTE PLAN for '$name' -> live :$LIVE_BIND_PORT"
|
||
info "current live ENGRAM_REAL_BIN : $live_bin"
|
||
info "sandbox binary (validated) : $d/bin/engram (sha ${bsha:0:12})"
|
||
info "will install as : $new_bin (additive; old binary retained)"
|
||
info "snapshot-first backup dir : $bkp (egm+wal+plist+rollback.txt)"
|
||
info "data promote : $([ $do_data -eq 1 ] && echo 'YES (--data: clone egm/wal -> live)' || echo 'no (binary only)')"
|
||
info "rails : launchctl bootout -> settle-poll -> bootstrap"
|
||
info "verify : /api/stats + edges>=baseline + keystones + retrieval; auto-rollback armed"
|
||
|
||
if [ "$approve" -ne 1 ]; then
|
||
warn "DRY-RUN — not touching prod. Re-run with --i-approve-prod-cutover to execute (per-use Will-approved)."
|
||
return 0
|
||
fi
|
||
|
||
need launchctl
|
||
local dom="gui/$(id -u)"
|
||
# ---- snapshot-first ----
|
||
log "snapshot-first backup -> $bkp"
|
||
mkdir -p "$bkp"
|
||
cp -p "$LIVE_DATA_DIR/neuron.egm" "$bkp/neuron.egm.bak"
|
||
cp -p "$LIVE_DATA_DIR/neuron.wal" "$bkp/neuron.wal.bak" 2>/dev/null || true
|
||
cp -p "$LIVE_PLIST" "$bkp/plist.bak"
|
||
printf 'rollback REAL_BIN=%s\nNEWBIN=%s\ndata_promote=%s\n' "$live_bin" "$new_bin" "$do_data" > "$bkp/rollback.txt"
|
||
ok "backup complete"
|
||
|
||
# ---- additive binary install + plist supersede ----
|
||
cp -p "$d/bin/engram" "$new_bin"
|
||
python3 - "$LIVE_PLIST" "$new_bin" <<'PY'
|
||
import sys,plistlib
|
||
p,new=sys.argv[1],sys.argv[2]
|
||
d=plistlib.load(open(p,'rb')); d.setdefault("EnvironmentVariables",{})["ENGRAM_REAL_BIN"]=new
|
||
plistlib.dump(d,open(p,'wb'))
|
||
PY
|
||
ok "installed $new_bin + updated plist ENGRAM_REAL_BIN"
|
||
|
||
# ---- optional data promote (after backup) ----
|
||
if [ $do_data -eq 1 ]; then
|
||
log "data promote: clone store -> live (backed up above)"
|
||
cp -p "$d/data/neuron.egm" "$LIVE_DATA_DIR/neuron.egm"
|
||
cp -p "$d/data/neuron.wal" "$LIVE_DATA_DIR/neuron.wal" 2>/dev/null || true
|
||
fi
|
||
|
||
# ---- rails cutover: bootout -> settle-poll -> bootstrap ----
|
||
log "rails: launchctl bootout $dom/$LIVE_LABEL"
|
||
launchctl bootout "$dom/$LIVE_LABEL" 2>/dev/null || true
|
||
local i
|
||
for i in $(seq 1 60); do
|
||
launchctl print "$dom/$LIVE_LABEL" >/dev/null 2>&1 || { ok "settle: job gone after ${i}x0.5s"; break; }
|
||
printf ' settle: job still present (%d)\n' "$i" >&2; sleep 0.5
|
||
done
|
||
log "rails: launchctl bootstrap $dom <plist>"
|
||
launchctl bootstrap "$dom" "$LIVE_PLIST" || warn "bootstrap returned nonzero"
|
||
|
||
# ---- verify ----
|
||
log "verify prod health"
|
||
local s="" ; for i in $(seq 1 60); do s="$(live_stats)"; [ -n "$s" ] && break; sleep 1; done
|
||
local ok_verify=1 le; le="$(stat_field "$s" edge_count)"
|
||
local base_e; base_e="$(mget "$name" "['live_baseline']['edge_count']")"
|
||
[ -n "$s" ] || ok_verify=0
|
||
[ "${le:-0}" -ge "${base_e:-0}" ] || ok_verify=0
|
||
local kid; for kid in "${KEYSTONES[@]}"; do curl -s -m5 "$LIVE_URL/api/node/$kid" 2>/dev/null | grep -q "\"$kid\"" || ok_verify=0; done
|
||
if [ "$ok_verify" -eq 1 ]; then
|
||
ok "PROMOTED. live stats: $s (rollback: $bkp)"; return 0
|
||
fi
|
||
|
||
# ---- auto-rollback ----
|
||
warn "verify FAILED — auto-rollback"
|
||
cp -p "$bkp/plist.bak" "$LIVE_PLIST"
|
||
[ $do_data -eq 1 ] && { cp -p "$bkp/neuron.egm.bak" "$LIVE_DATA_DIR/neuron.egm"; cp -p "$bkp/neuron.wal.bak" "$LIVE_DATA_DIR/neuron.wal" 2>/dev/null || true; }
|
||
launchctl bootout "$dom/$LIVE_LABEL" 2>/dev/null || true
|
||
for i in $(seq 1 60); do launchctl print "$dom/$LIVE_LABEL" >/dev/null 2>&1 || break; sleep 0.5; done
|
||
launchctl bootstrap "$dom" "$LIVE_PLIST" || true
|
||
die "ROLLED BACK to $live_bin. See $bkp"
|
||
}
|
||
|
||
# ================================================================ destroy ======
|
||
cmd_destroy(){
|
||
local name="$1"; shift || true
|
||
mexists "$name" || die "no such sandbox: $name"
|
||
local d; d="$(sdir "$name")"
|
||
stop_daemon "$name"
|
||
if [ -d "$d/build/worktree" ]; then
|
||
log "removing git worktree"
|
||
git -C "$EL_REPO" worktree remove --force "$d/build/worktree" 2>/dev/null || true
|
||
git -C "$EL_REPO" worktree prune 2>/dev/null || true
|
||
fi
|
||
log "removing $d"
|
||
rm -rf "$d"
|
||
ok "destroyed '$name' (live untouched)"
|
||
}
|
||
|
||
# ================================================================ list/status ==
|
||
cmd_list(){
|
||
[ -d "$SBX_ROOT" ] || { echo "no sandboxes"; return 0; }
|
||
printf '%-16s %-6s %-8s %-9s %s\n' NAME PORT STATE PID SOURCE
|
||
local m
|
||
for m in "$SBX_ROOT"/*/manifest.json; do
|
||
[ -f "$m" ] || continue
|
||
local n p src pid state
|
||
n="$(python3 -c "import json;print(json.load(open('$m'))['name'])")"
|
||
p="$(python3 -c "import json;print(json.load(open('$m'))['port'])")"
|
||
src="$(python3 -c "import json;print(json.load(open('$m'))['source'])")"
|
||
pid="$(daemon_pid "$n")"; state="stopped"; daemon_alive "$n" && state="running"
|
||
printf '%-16s %-6s %-8s %-9s %s\n' "$n" "$p" "$state" "${pid:-–}" "$src"
|
||
done
|
||
}
|
||
cmd_status(){
|
||
local name="$1"; mexists "$name" || die "no such sandbox: $name"
|
||
python3 -m json.tool "$(manifest "$name")"
|
||
daemon_alive "$name" && echo "state: running (pid $(daemon_pid "$name")) stats: $(sbx_stats "$name")" || echo "state: stopped"
|
||
[ -f "$(sdir "$name")/validate.json" ] && { echo "--- last validation ---"; python3 -m json.tool "$(sdir "$name")/validate.json"; }
|
||
}
|
||
|
||
usage(){ cat >&2 <<EOF
|
||
${C_BLD}nsbx${C_0} — Neuron Sandbox: experiments + code changes against the REAL engram
|
||
runtime on an isolated snapshot of the live mind, with a gated promote-to-prod path.
|
||
|
||
nsbx up [name] [flags…] one command: your private, isolated copy of the mind
|
||
(creates on first run, starts thereafter; prod untouchable)
|
||
nsbx create [name] [--port N] [--source DIR | --branch REF [--repo R] | --binary PATH]
|
||
clone live store+WAL+config, place/build the runtime, boot on an
|
||
isolated port (never :$LIVE_BIND_PORT/:$SOUL_PORT). Default runtime = stock prod binary.
|
||
nsbx build <name> --source DIR | --branch REF rebuild the runtime from a code change + hot-restart
|
||
nsbx run <name> <cmd...> | api <path> [json] run an experiment; capture output + metrics
|
||
nsbx validate <name> rails as checks: zero-loss(load+reboot), reboot-prove,
|
||
RSS bound, retrieval parity, keystone integrity
|
||
nsbx promote <name> [--data] [--i-approve-prod-cutover] GATED rails cutover to prod (DRY-RUN without approval)
|
||
nsbx destroy <name> stop daemon, free port, remove clone (live untouched)
|
||
nsbx list | nsbx status <name>
|
||
|
||
Env in 'run' cmds: \$SBX_URL \$SBX_PORT \$SBX_KEY \$SBX_DATA \$SBX_BIN \$SBX_NAME
|
||
EOF
|
||
}
|
||
|
||
main(){
|
||
local cmd="${1:-}"; shift || true
|
||
case "$cmd" in
|
||
up) cmd_up "$@";;
|
||
create) cmd_create "$@";;
|
||
build) cmd_build "$@";;
|
||
run) cmd_run "$@";;
|
||
validate) cmd_validate "$@";;
|
||
promote) cmd_promote "$@";;
|
||
destroy) cmd_destroy "$@";;
|
||
list|ls) cmd_list "$@";;
|
||
status) cmd_status "$@";;
|
||
""|-h|--help|help) usage;;
|
||
*) die "unknown command: $cmd (try: nsbx help)";;
|
||
esac
|
||
}
|
||
main "$@"
|