Merge pull request 'nsbx: fail loud on daemon-not-ready + el_seed.c standalone compile' (#118) from fix/nsbx-tooling-hardening into dev

lang/runtime/el_seed.c

@@ -37,6 +37,82 @@
 #include <dlfcn.h>
 #include <curl/curl.h>
 
+/* el_runtime.c bridge prototypes.
+ *
+ * A block of __-prefixed wrappers further down in this file (http serving,
+ * JSON access, key-val state, URL/HTML escaping, and the whole engram_*
+ * node/edge/layer/search surface -- 51 symbols in total) delegate to
+ * unprefixed counterparts that are implemented in el_runtime.c, not here.
+ * Porting them into native el_seed.c or El has not happened yet.
+ * tools/install.sh compiles el_seed.c and el_runtime.c as separate objects
+ * and archives both into libel.a, so the symbols are always present at link
+ * time. el_seed.c alone was just missing the prototypes, which made even a
+ * standalone -c compile of this one file fail on a toolchain that now treats
+ * an implicit function declaration as a hard error under C11.
+ *
+ * A plain include of el_runtime.h was tried first and rejected: it redefines
+ * el_to_float and el_from_float, which el_seed.h already provides. Narrow
+ * prototypes, copied verbatim from el_runtime.h, avoid that collision without
+ * pulling in the rest of the retiring runtime header.
+ */
+el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body);
+void     http_serve(el_val_t port, el_val_t handler);
+void     http_serve_v2(el_val_t port, el_val_t handler);
+el_val_t json_get(el_val_t json, el_val_t key);
+el_val_t json_get_string(el_val_t json_str, el_val_t key);
+el_val_t json_get_int(el_val_t json_str, el_val_t key);
+el_val_t json_get_float(el_val_t json_str, el_val_t key);
+el_val_t json_get_bool(el_val_t json_str, el_val_t key);
+el_val_t json_get_raw(el_val_t json_str, el_val_t key);
+el_val_t json_parse(el_val_t s);
+el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value);
+el_val_t json_stringify(el_val_t v);
+el_val_t json_array_len(el_val_t json_str);
+el_val_t json_array_get(el_val_t json_str, el_val_t index);
+el_val_t json_array_get_string(el_val_t json_str, el_val_t index);
+el_val_t state_set(el_val_t key, el_val_t value);
+el_val_t state_get(el_val_t key);
+el_val_t state_del(el_val_t key);
+el_val_t state_keys(void);
+el_val_t url_encode(el_val_t s);
+el_val_t url_decode(el_val_t s);
+el_val_t el_html_sanitize(el_val_t input_html, el_val_t allowlist_json);
+el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience);
+el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
+                           el_val_t salience, el_val_t importance, el_val_t confidence,
+                           el_val_t tier, el_val_t tags);
+el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t label,
+                              el_val_t salience, el_val_t certainty, el_val_t confidence,
+                              el_val_t status, el_val_t tags, el_val_t layer_id);
+el_val_t engram_add_layer(el_val_t name, el_val_t priority, el_val_t suppressible,
+                           el_val_t transparent, el_val_t injectable);
+el_val_t engram_remove_layer(el_val_t layer_id);
+el_val_t engram_list_layers(void);
+el_val_t engram_list_layers_json(void);
+el_val_t engram_get_node(el_val_t id);
+el_val_t engram_get_node_json(el_val_t id);
+el_val_t engram_get_node_by_label(el_val_t label);
+void     engram_strengthen(el_val_t node_id);
+void     engram_forget(el_val_t node_id);
+el_val_t engram_node_count(void);
+el_val_t engram_edge_count(void);
+el_val_t engram_scan_nodes(el_val_t limit, el_val_t offset);
+el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
+el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
+el_val_t engram_search(el_val_t query, el_val_t limit);
+el_val_t engram_search_json(el_val_t query, el_val_t limit);
+el_val_t engram_activate(el_val_t query, el_val_t depth);
+el_val_t engram_activate_json(el_val_t query, el_val_t depth);
+el_val_t engram_compile_layered_json(el_val_t intent, el_val_t depth);
+el_val_t engram_stats_json(void);
+void     engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation);
+el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id);
+el_val_t engram_neighbors(el_val_t node_id);
+el_val_t engram_neighbors_filtered(el_val_t node_id, el_val_t max_depth, el_val_t direction);
+el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
+el_val_t engram_load(el_val_t path);
+el_val_t engram_save(el_val_t path);
+
 /* ── Private allocator ───────────────────────────────────────────────────── */
 /*
  * el_seed.c carries its own arena for per-request allocation tracking.
@@ -831,6 +907,219 @@ void __mutex_unlock(el_val_t m) {
     pthread_mutex_unlock(&_el_mutexes[slot]);
 }
 
+/* ── Channels ─────────────────────────────────────────────────────────────── *
+ * Buffered MPMC channel backed by a mutex + condvar + circular buffer.
+ * Ported from the pre-restructure el_runtime.c (b2aac4b) — runtime/channel.el
+ * has always called these five primitives, but they were never carried
+ * forward into el_seed.c when el_runtime.c was consolidated onto the
+ * canonical release copy. Native channels were silently unlinkable on dev
+ * until this port.
+ *
+ * __channel_new(capacity) -> Int (handle)
+ * __channel_send(ch, msg)         — blocks if full (capacity > 0) or never (unbounded)
+ * __channel_recv(ch) -> String    — blocks until a message is available
+ * __channel_try_recv(ch) -> String — non-blocking, returns "" if empty
+ * __channel_close(ch)             — signal no more sends; recv drains remaining
+ *
+ * Bounded channels (cap > 0): circular buffer, sender blocks when full.
+ * Unbounded channels (cap == 0): dynamic array, sender never blocks.
+ */
+#define EL_CHANNEL_MAX 64
+#define EL_CHANNEL_BUF 1024
+
+typedef struct {
+    char**          buf;
+    int             cap;     /* 0 = unbounded (grows dynamically) */
+    int             head, tail, count;
+    int             dyn_cap; /* allocated slots for unbounded mode */
+    int             closed;
+    pthread_mutex_t mu;
+    pthread_cond_t  not_empty;
+    pthread_cond_t  not_full;
+} ElChannel;
+
+static ElChannel       _channels[EL_CHANNEL_MAX];
+static int             _channel_count = 0;
+static pthread_mutex_t _channel_alloc_mu = PTHREAD_MUTEX_INITIALIZER;
+
+el_val_t __channel_new(el_val_t capacity_v) {
+    int cap = (int)(int64_t)capacity_v;
+    if (cap < 0) cap = 0;
+
+    pthread_mutex_lock(&_channel_alloc_mu);
+    if (_channel_count >= EL_CHANNEL_MAX) {
+        pthread_mutex_unlock(&_channel_alloc_mu);
+        fprintf(stderr, "[__channel_new] channel table full\n");
+        return EL_INT(-1);
+    }
+    int slot = _channel_count++;
+    pthread_mutex_unlock(&_channel_alloc_mu);
+
+    ElChannel* ch = &_channels[slot];
+    memset(ch, 0, sizeof(*ch));
+    ch->cap     = cap;
+    ch->closed  = 0;
+    ch->head    = 0;
+    ch->tail    = 0;
+    ch->count   = 0;
+
+    if (cap > 0) {
+        /* Bounded: fixed circular buffer. */
+        ch->buf     = (char**)malloc((size_t)cap * sizeof(char*));
+        ch->dyn_cap = cap;
+    } else {
+        /* Unbounded: start with EL_CHANNEL_BUF slots, grow as needed. */
+        ch->buf     = (char**)malloc(EL_CHANNEL_BUF * sizeof(char*));
+        ch->dyn_cap = EL_CHANNEL_BUF;
+    }
+    if (!ch->buf) {
+        fprintf(stderr, "[__channel_new] out of memory\n");
+        return EL_INT(-1);
+    }
+
+    pthread_mutex_init(&ch->mu, NULL);
+    pthread_cond_init(&ch->not_empty, NULL);
+    pthread_cond_init(&ch->not_full, NULL);
+
+    return EL_INT(slot);
+}
+
+el_val_t __channel_send(el_val_t ch_v, el_val_t msg_v) {
+    int slot = (int)(int64_t)ch_v;
+    if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR("");
+    ElChannel* ch = &_channels[slot];
+
+    const char* msg = EL_CSTR(msg_v);
+    if (!msg) msg = "";
+    char* copy = strdup(msg); /* channel owns the string */
+
+    pthread_mutex_lock(&ch->mu);
+
+    if (ch->closed) {
+        /* Send on closed channel is a no-op (drop the message). */
+        pthread_mutex_unlock(&ch->mu);
+        free(copy);
+        return EL_STR("");
+    }
+
+    if (ch->cap > 0) {
+        /* Bounded: block while full. */
+        while (ch->count >= ch->cap && !ch->closed) {
+            pthread_cond_wait(&ch->not_full, &ch->mu);
+        }
+        if (ch->closed) {
+            pthread_mutex_unlock(&ch->mu);
+            free(copy);
+            return EL_STR("");
+        }
+        ch->buf[ch->tail] = copy;
+        ch->tail = (ch->tail + 1) % ch->cap;
+        ch->count++;
+    } else {
+        /* Unbounded: grow the buffer if needed. */
+        if (ch->count >= ch->dyn_cap) {
+            int new_cap = ch->dyn_cap * 2;
+            char** grown = (char**)realloc(ch->buf, (size_t)new_cap * sizeof(char*));
+            if (!grown) {
+                pthread_mutex_unlock(&ch->mu);
+                free(copy);
+                fprintf(stderr, "[__channel_send] out of memory growing channel\n");
+                return EL_STR("");
+            }
+            /* The circular buffer may have wrapped. Linearise it first.
+             * In unbounded mode head is always 0 (we append at tail, drain
+             * from head), so a simple memmove isn't needed — but if the
+             * buffer did wrap (tail < head after growth), we need to fix up.
+             * Simplest safe path: if tail wrapped, move the head..old_cap
+             * segment to new_cap..new_cap+(old_cap-head). */
+            if (ch->tail < ch->head) {
+                /* Wrapped: [head..old_cap) is the front, [0..tail) is the back. */
+                int front = ch->dyn_cap - ch->head;
+                memmove(grown + ch->dyn_cap, grown + ch->head, (size_t)front * sizeof(char*));
+                ch->head = ch->dyn_cap;
+            }
+            ch->buf     = grown;
+            ch->dyn_cap = new_cap;
+        }
+        ch->buf[ch->tail] = copy;
+        ch->tail = (ch->tail + 1) % ch->dyn_cap;
+        ch->count++;
+    }
+
+    pthread_cond_signal(&ch->not_empty);
+    pthread_mutex_unlock(&ch->mu);
+    return EL_STR("");
+}
+
+el_val_t __channel_recv(el_val_t ch_v) {
+    int slot = (int)(int64_t)ch_v;
+    if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR("");
+    ElChannel* ch = &_channels[slot];
+
+    pthread_mutex_lock(&ch->mu);
+
+    /* Block until there is a message or the channel is closed and drained. */
+    while (ch->count == 0 && !ch->closed) {
+        pthread_cond_wait(&ch->not_empty, &ch->mu);
+    }
+
+    if (ch->count == 0) {
+        /* Closed and empty — signal EOF. */
+        pthread_mutex_unlock(&ch->mu);
+        return EL_STR("");
+    }
+
+    int buf_cap = (ch->cap > 0) ? ch->cap : ch->dyn_cap;
+    char* msg = ch->buf[ch->head];
+    ch->head  = (ch->head + 1) % buf_cap;
+    ch->count--;
+
+    pthread_cond_signal(&ch->not_full);
+    pthread_mutex_unlock(&ch->mu);
+
+    /* Hand the string to the arena so it is freed after the request. */
+    seed_arena_track(msg);
+    return EL_STR(msg);
+}
+
+el_val_t __channel_try_recv(el_val_t ch_v) {
+    int slot = (int)(int64_t)ch_v;
+    if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR("");
+    ElChannel* ch = &_channels[slot];
+
+    pthread_mutex_lock(&ch->mu);
+
+    if (ch->count == 0) {
+        pthread_mutex_unlock(&ch->mu);
+        return EL_STR("");
+    }
+
+    int buf_cap = (ch->cap > 0) ? ch->cap : ch->dyn_cap;
+    char* msg = ch->buf[ch->head];
+    ch->head  = (ch->head + 1) % buf_cap;
+    ch->count--;
+
+    pthread_cond_signal(&ch->not_full);
+    pthread_mutex_unlock(&ch->mu);
+
+    seed_arena_track(msg);
+    return EL_STR(msg);
+}
+
+el_val_t __channel_close(el_val_t ch_v) {
+    int slot = (int)(int64_t)ch_v;
+    if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR("");
+    ElChannel* ch = &_channels[slot];
+
+    pthread_mutex_lock(&ch->mu);
+    ch->closed = 1;
+    /* Wake all blocked recvers and senders so they can observe the close. */
+    pthread_cond_broadcast(&ch->not_empty);
+    pthread_cond_broadcast(&ch->not_full);
+    pthread_mutex_unlock(&ch->mu);
+    return EL_STR("");
+}
+
 /* ── Subprocess ──────────────────────────────────────────────────────────── */
 
 el_val_t __exec(el_val_t cmd) {

lang/runtime/el_seed.h

@@ -139,6 +139,13 @@ el_val_t  __mutex_new(void);
 void      __mutex_lock(el_val_t m);
 void      __mutex_unlock(el_val_t m);
 
+/* Buffered MPMC channel (runtime/channel.el). capacity=0 means unbounded. */
+el_val_t  __channel_new(el_val_t capacity);
+el_val_t  __channel_send(el_val_t ch, el_val_t msg);     /* blocks if bounded+full */
+el_val_t  __channel_recv(el_val_t ch);                   /* blocks until available */
+el_val_t  __channel_try_recv(el_val_t ch);                /* non-blocking, "" if empty */
+el_val_t  __channel_close(el_val_t ch);
+
 /* ── Subprocess ──────────────────────────────────────────────────────────── */
 
 el_val_t  __exec(el_val_t cmd);            /* popen, capture all stdout, return String */

tools/neuron-sandbox/README.md

@@ -173,4 +173,7 @@ Each ad-hoc harness becomes `nsbx run <name> …` (or `--source` build) against
 ## Env knobs
 
 `NSBX_ROOT`, `NSBX_PORT_BASE`, `NSBX_RSS_BOUND_MB`, `NSBX_REMERGE_THRESHOLD`,
+`NSBX_READY_TIMEOUT_SECS` (default 15 — how long `up`/`create`/`build` wait for a
+daemon to answer `/api/stats` before reporting failure; raise it if a boot is
+legitimately slow under concurrent sandbox/CPU load rather than actually broken),
 `EL_REPO` (for `elc` + runtime sources), `ENGRAM_LIVE_DATA_DIR`, `ENGRAM_LIVE_PLIST`.

tools/neuron-sandbox/nsbx

@@ -32,6 +32,12 @@ EL_REPO="${EL_REPO:-$HOME/Development/neuron-technologies/foundation/el}"
 PORT_BASE="${NSBX_PORT_BASE:-8900}"
 RSS_BOUND_MB="${NSBX_RSS_BOUND_MB:-550}"      # from store-fix reboot-proof (aaf13f88)
 REMERGE_THRESHOLD="${NSBX_REMERGE_THRESHOLD:-40000}"
+# readiness-poll window for start_daemon (0.5s ticks). Default unchanged (15s) —
+# but a cold boot against the full live store, under concurrent CPU contention
+# from other running sandboxes, has been observed live to take well past that.
+# Bump per-invocation with NSBX_READY_TIMEOUT_SECS if `up`/`create` reports a
+# not-ready failure but the daemon looks otherwise fine (see its logs/daemon.log).
+READY_TICKS=$(( ${NSBX_READY_TIMEOUT_SECS:-15} * 2 ))
 KEYSTONES=( "kn-efeb4a5b-5aff-4759-8a97-7233099be6ee" "kn-5b606390-a52d-4ca2-8e0e-eba141d13440" )
 # fixed probe set for retrieval-parity (stable, identity-anchored)
 PARITY_QUERIES=( "who am I" "self identity core" "engram store durability" "keystone self anchor" "grounding honesty" )
@@ -77,7 +83,7 @@ _port_claimed(){ # is another sandbox already assigned this port?
 live_stats(){ curl -s -m5 "$LIVE_URL/api/stats" 2>/dev/null; }
 api(){ # api <name> <path> [json-body]
   local name="$1" path="$2" body="${3:-}"
-  local port; port="$(mget "$name" "['port']")"; [ -n "$port" ] || die "unknown sandbox: $name"
+  local port; port="$(mget "$name" "['port']")"; [ -n "$port" ] || die "unknown sandbox: $name (run: nsbx list)"
   local url="http://127.0.0.1:${port}${path}"
   if [ -n "$body" ]; then curl -s -m30 -X POST -H 'Content-Type: application/json' -d "$body" "$url"
   else curl -s -m30 "$url"; fi
@@ -88,6 +94,19 @@ stat_field(){ printf '%s' "$1" | sed -n "s/.*\"$2\":\([0-9]*\).*/\1/p"; }
 daemon_pid(){ local f; f="$(sdir "$1")/daemon.pid"; [ -f "$f" ] && cat "$f" || true; }
 daemon_alive(){ local p; p="$(daemon_pid "$1")"; [ -n "$p" ] && kill -0 "$p" 2>/dev/null; }
 
+# daemon_health <name> : prints "stopped" | "running" | "unresponsive" (to stdout).
+# "running" means the pid is alive AND /api/stats actually answered — process
+# liveness alone (daemon_alive) is not proof the HTTP server is serving; a pegged
+# or hung process still passes kill -0. Short timeout (2s) since this runs per-row
+# in `nsbx list`.
+daemon_health(){
+  local name="$1"
+  daemon_alive "$name" || { echo "stopped"; return 0; }
+  local port; port="$(mget "$name" "['port']")"
+  local s; s="$(curl -s -m2 "http://127.0.0.1:${port}/api/stats" 2>/dev/null)"
+  [ -n "$s" ] && echo "running" || echo "unresponsive"
+}
+
 # ---------------------------------------------------------------- elc/build ----
 find_elc(){
   command -v elc 2>/dev/null && return 0
@@ -123,6 +142,49 @@ _build_binary(){
   ok "built: $out ($(ls -lh "$out" | awk '{print $5}'), sha $(sha "$out" | cut -c1-12))"
 }
 
+# bin_built_at <path> : human-readable build timestamp. `cp -p` preserves mtime,
+# so this is the ORIGINAL build time even for binaries copied stock-prod into a
+# sandbox — not the copy time.
+bin_built_at(){ stat -f '%Sm' -t '%Y-%m-%d %H:%M:%S' "$1" 2>/dev/null || echo "unknown"; }
+
+# _binary_freshness <name> : best-effort staleness note (empty string if fresh/
+# unknown — never guesses). Two cases:
+#  - stock-prod: compares the sha recorded at create time against the CURRENTLY
+#    configured live real binary's sha (recomputed now, not cached) — catches
+#    "live prod moved on since this sandbox was cloned".
+#  - branch/source/rebuilt: compares the recorded source_commit against the
+#    LOCAL origin/dev ref (no fetch — reads whatever the repo already has) —
+#    catches "built from a commit that predates current dev tip".
+_binary_freshness(){
+  local name="$1" src; src="$(mget "$name" "['source']")"
+  case "$src" in
+    stock-prod:*)
+      local live_bin cur_sha rec_sha
+      live_bin="$(_live_real_bin)"
+      [ -n "$live_bin" ] && [ -x "$live_bin" ] || return 0
+      cur_sha="$(sha "$live_bin")"; rec_sha="$(mget "$name" "['binary_sha256']")"
+      [ -n "$cur_sha" ] && [ -n "$rec_sha" ] && [ "$cur_sha" != "$rec_sha" ] \
+        && printf 'stale: live prod binary has moved on since this sandbox was cloned (live is now %s, sha %s) — nsbx build %s --binary %s to catch up' \
+             "$(basename "$live_bin")" "${cur_sha:0:12}" "$name" "$live_bin"
+      ;;
+    branch:*|source:*|rebuilt:*)
+      local commit cur behind
+      commit="$(mget "$name" "['source_commit']")"
+      [ -n "$commit" ] || return 0
+      cur="$(git -C "$EL_REPO" rev-parse origin/dev 2>/dev/null)" || return 0
+      [ -n "$cur" ] && [ "$commit" != "$cur" ] || return 0
+      git -C "$EL_REPO" merge-base --is-ancestor "$commit" "$cur" 2>/dev/null || return 0
+      behind="$(git -C "$EL_REPO" rev-list --count "$commit..$cur" 2>/dev/null)"
+      printf 'stale: built from %s, %s commit(s) behind local origin/dev (%s) — nsbx build %s --branch origin/dev' \
+        "${commit:0:12}" "${behind:-?}" "${cur:0:12}" "$name"
+      ;;
+  esac
+}
+
+# _source_commit <dir> : best-effort git HEAD of a source tree used to build a
+# sandbox binary, empty if not a git repo (e.g. a prebuilt --binary path has none).
+_source_commit(){ git -C "$1" rev-parse HEAD 2>/dev/null || true; }
+
 # ---------------------------------------------------------------- daemon -------
 # start_daemon <name> : boots the sandbox's real engram binary on its isolated
 # port against its cloned data dir, with the SAME auto-remerge net the live soul
@@ -133,9 +195,9 @@ start_daemon(){
   local port bin data export key
   port="$(mget "$name" "['port']")"; bin="$d/bin/engram"; data="$d/data"
   key="sbx-$name"; export="$data/.scan-export.reseed-clean.json"
-  [ -x "$bin" ] || die "sandbox binary missing: $bin"
+  [ -x "$bin" ] || die "sandbox binary missing: $bin (run: nsbx build $name --source DIR | --branch REF, or nsbx destroy $name && nsbx create $name to reclone stock-prod)"
   [ "$port" != "$LIVE_BIND_PORT" ] && [ "$port" != "$SOUL_PORT" ] || die "refusing forbidden port $port"
-  [ -f "$data/neuron.egm" ] || die "sandbox has no cloned store: $data/neuron.egm"
+  [ -f "$data/neuron.egm" ] || die "sandbox has no cloned store: $data/neuron.egm (data dir is corrupt/incomplete — run: nsbx destroy $name && nsbx create $name)"
   # HARD guard: never point a sandbox daemon at the live data dir.
   [ "$(cd "$data" && pwd -P)" != "$(cd "$LIVE_DATA_DIR" && pwd -P)" ] || die "refusing: sandbox data dir resolves to LIVE store"
 
@@ -150,11 +212,11 @@ start_daemon(){
   echo "$pid" > "$d/daemon.pid"
   # readiness poll
   local url="http://127.0.0.1:$port" i s
-  for i in $(seq 1 30); do
+  for i in $(seq 1 "$READY_TICKS"); do
     s="$(curl -s -m3 "$url/api/stats" 2>/dev/null)"
     [ -n "$s" ] && break; sleep 0.5
   done
-  [ -n "$s" ] || { warn "daemon did not become ready (see $d/logs/daemon.log)"; return 1; }
+  [ -n "$s" ] || { warn "daemon did not become ready within ${NSBX_READY_TIMEOUT_SECS:-15}s (see $d/logs/daemon.log). pid $pid may still be alive and slow to boot under load — check: lsof -iTCP:$port -P, or retry with NSBX_READY_TIMEOUT_SECS=45"; return 1; }
   ok "ready pid=$pid  boot-stats: $s"
   # auto-remerge net (idempotent): match live edge population if the export is present
   if [ -f "$export" ]; then
@@ -231,33 +293,35 @@ cmd_create(){
   info "live baseline stats: ${lstats:-<unavailable>}"
 
   # ---- determine + place the runtime binary (versioned into the snapshot) ----
-  local source_desc live_bin
+  local source_desc live_bin source_commit=""
   live_bin="$(_live_real_bin)"
   if [ -n "$binpath" ]; then
     [ -x "$binpath" ] || die "not an executable binary: $binpath"
     cp -p "$binpath" "$d/bin/engram"; source_desc="prebuilt:$binpath"
   elif [ -n "$src" ]; then
     _build_binary "$src" "$d/bin/engram" "$d/build"; source_desc="source:$src"
+    source_commit="$(_source_commit "$src")"
   elif [ -n "$branch" ]; then
     log "worktree: $repo @ $branch -> $d/build/worktree"
     git -C "$repo" worktree add --detach "$d/build/worktree" "$branch" >/dev/null 2>&1 \
       || die "git worktree add failed ($repo @ $branch)"
     _build_binary "$d/build/worktree" "$d/bin/engram" "$d/build"; source_desc="branch:$branch@$repo"
+    source_commit="$(_source_commit "$d/build/worktree")"
   else
     [ -x "$live_bin" ] || die "cannot resolve live ENGRAM_REAL_BIN: $live_bin"
     cp -p "$live_bin" "$d/bin/engram"; source_desc="stock-prod:$live_bin"
   fi
   local bin_sha; bin_sha="$(sha "$d/bin/engram")"
-  info "runtime: $source_desc  (sha ${bin_sha:0:12})"
+  info "runtime: $source_desc  (sha ${bin_sha:0:12}, built $(bin_built_at "$d/bin/engram"))"
 
   # ---- write manifest ----
-  python3 - "$name" "$port" "$source_desc" "$bin_sha" "$egm_sha" "$base_nodes" "$base_edges" "$(sha "$live_bin" 2>/dev/null)" <<'PY' > "$(manifest "$name")"
+  python3 - "$name" "$port" "$source_desc" "$bin_sha" "$egm_sha" "$base_nodes" "$base_edges" "$(sha "$live_bin" 2>/dev/null)" "$source_commit" <<'PY' > "$(manifest "$name")"
 import json,sys,datetime
-name,port,src,binsha,egmsha,bn,be,livebinsha=sys.argv[1:9]
+name,port,src,binsha,egmsha,bn,be,livebinsha,source_commit=sys.argv[1:10]
 json.dump({
  "name":name,"port":int(port),"created_at":datetime.datetime.now(datetime.timezone.utc).isoformat(),
  "source":src,"binary_sha256":binsha,"clone_egm_sha256":egmsha,
- "live_binary_sha256":livebinsha,
+ "live_binary_sha256":livebinsha,"source_commit":source_commit,
  "live_baseline":{"node_count":int(bn or 0),"edge_count":int(be or 0)},
  "keystones":["kn-efeb4a5b-5aff-4759-8a97-7233099be6ee","kn-5b606390-a52d-4ca2-8e0e-eba141d13440"]
 }, sys.stdout, indent=2)
@@ -268,6 +332,17 @@ PY
   start_daemon "$name" || die "daemon failed to start"
   local sstats; sstats="$(sbx_stats "$name")"
   local sbn sbe; sbn="$(stat_field "$sstats" node_count)"; sbe="$(stat_field "$sstats" edge_count)"
+  # a boot immediately followed by an auto-remerge can leave the daemon briefly
+  # busy — retry rather than silently folding a 0/0 baseline into the manifest.
+  # `validate`'s zero-loss/reboot-prove checks compare current counts >= baseline,
+  # so a 0/0 baseline would make them trivially PASS regardless of real data loss.
+  local _bi
+  for _bi in 1 2 3 4 5; do
+    [ -n "$sbn" ] && [ "$sbn" != "0" ] && break
+    sleep 1
+    sstats="$(sbx_stats "$name")"; sbn="$(stat_field "$sstats" node_count)"; sbe="$(stat_field "$sstats" edge_count)"
+  done
+  [ -z "$sbn" ] || [ "$sbn" = "0" ] && warn "sandbox stats still empty/zero after retries — recording sbx_baseline 0/0. This makes 'nsbx validate $name' zero-loss checks trivially pass; investigate before trusting a validate PASS: nsbx status $name"
   _capture_retrieval "$name" "$d/baseline/retrieval.json"
   # fold sandbox baseline into manifest
   python3 - "$(manifest "$name")" "$sbn" "$sbe" <<'PY'
@@ -320,7 +395,12 @@ except Exception: print("[]")' 2>/dev/null)"
 # thereafter. Prod on :$LIVE_BIND_PORT/:$SOUL_PORT is unreachable from here by design.
 cmd_up(){
   local name; if [ $# -gt 0 ] && [ "${1#-}" = "$1" ]; then name="$1"; shift; else name="${USER:-dev}-dev"; fi
-  if mexists "$name"; then daemon_alive "$name" || start_daemon "$name"; else cmd_create "$name" "$@"; fi
+  if mexists "$name"; then
+    daemon_alive "$name" || start_daemon "$name" \
+      || die "daemon did not become ready — see $(sdir "$name")/logs/daemon.log (try: nsbx up $name  again once you've checked the log)"
+  else
+    cmd_create "$name" "$@"
+  fi
   local port; port="$(mget "$name" "['port']")"
   echo >&2
   ok "your sandbox '$name' is ready at http://127.0.0.1:$port  (a private copy of the mind — prod is untouchable)"
@@ -334,34 +414,37 @@ cmd_up(){
 # it on the SAME clone + port (the code-change dev loop, in place).
 cmd_build(){
   local name="$1"; shift || true
-  mexists "$name" || die "no such sandbox: $name"
+  mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)"
   local src="" branch="" repo="$EL_REPO"
   while [ $# -gt 0 ]; do case "$1" in
     --source) src="$2"; shift 2;; --branch) branch="$2"; shift 2;; --repo) repo="$2"; shift 2;;
     *) die "unknown flag: $1";; esac; done
   local d; d="$(sdir "$name")"
   stop_daemon "$name"
-  if [ -n "$src" ]; then _build_binary "$src" "$d/bin/engram" "$d/build"
+  local source_commit=""
+  if [ -n "$src" ]; then _build_binary "$src" "$d/bin/engram" "$d/build"; source_commit="$(_source_commit "$src")"
   elif [ -n "$branch" ]; then
     rm -rf "$d/build/worktree" 2>/dev/null; git -C "$repo" worktree prune 2>/dev/null
     git -C "$repo" worktree add --detach "$d/build/worktree" "$branch" >/dev/null 2>&1 || die "worktree add failed"
     _build_binary "$d/build/worktree" "$d/bin/engram" "$d/build"
+    source_commit="$(_source_commit "$d/build/worktree")"
   else die "usage: nsbx build <name> --source DIR | --branch REF [--repo R]"; fi
   # record new binary sha
-  python3 - "$(manifest "$name")" "$(sha "$d/bin/engram")" "${src:-branch:$branch}" <<'PY'
-import json,sys; mf,s,src=sys.argv[1:4]
-d=json.load(open(mf)); d["binary_sha256"]=s; d["source"]="rebuilt:"+src
+  python3 - "$(manifest "$name")" "$(sha "$d/bin/engram")" "${src:-branch:$branch}" "$source_commit" <<'PY'
+import json,sys; mf,s,src,source_commit=sys.argv[1:5]
+d=json.load(open(mf)); d["binary_sha256"]=s; d["source"]="rebuilt:"+src; d["source_commit"]=source_commit
 json.dump(d,open(mf,'w'),indent=2)
 PY
-  start_daemon "$name"
+  start_daemon "$name" \
+    || die "rebuilt binary did not become ready — see $d/logs/daemon.log (the old binary is gone; fix the code and re-run nsbx build $name ...)"
   ok "rebuilt + restarted on :$(mget "$name" "['port']")"
 }
 
 # ================================================================ run ==========
 cmd_run(){
   local name="$1"; shift || true
-  mexists "$name" || die "no such sandbox: $name"
-  daemon_alive "$name" || start_daemon "$name"
+  mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)"
+  daemon_alive "$name" || start_daemon "$name" || die "daemon not running and failed to start — see $(sdir "$name")/logs/daemon.log"
   local d port; d="$(sdir "$name")"; port="$(mget "$name" "['port']")"
   # direct API form: nsbx run <name> api <path> [json]
   if [ "${1:-}" = "api" ]; then
@@ -395,8 +478,8 @@ cmd_run(){
 # RSS bound; retrieval parity; keystone integrity.
 cmd_validate(){
   local name="$1"; shift || true
-  mexists "$name" || die "no such sandbox: $name"
-  daemon_alive "$name" || start_daemon "$name"
+  mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)"
+  daemon_alive "$name" || start_daemon "$name" || die "daemon not running and failed to start — see $(sdir "$name")/logs/daemon.log"
   local d port key; d="$(sdir "$name")"; port="$(mget "$name" "['port']")"; key="sbx-$name"
   local url="http://127.0.0.1:$port"
   local bn be; bn="$(mget "$name" "['sbx_baseline']['node_count']")"; be="$(mget "$name" "['sbx_baseline']['edge_count']")"
@@ -489,7 +572,7 @@ PY
 # Default is a DRY-RUN plan; requires --i-approve-prod-cutover to actually cut over.
 cmd_promote(){
   local name="$1"; shift || true
-  mexists "$name" || die "no such sandbox: $name"
+  mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)"
   local approve=0 do_data=0
   while [ $# -gt 0 ]; do case "$1" in
     --i-approve-prod-cutover) approve=1; shift;;
@@ -588,7 +671,7 @@ PY
 # ================================================================ destroy ======
 cmd_destroy(){
   local name="$1"; shift || true
-  mexists "$name" || die "no such sandbox: $name"
+  mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)"
   local d; d="$(sdir "$name")"
   stop_daemon "$name"
   if [ -d "$d/build/worktree" ]; then
@@ -604,22 +687,44 @@ cmd_destroy(){
 # ================================================================ list/status ==
 cmd_list(){
   [ -d "$SBX_ROOT" ] || { echo "no sandboxes"; return 0; }
-  printf '%-16s %-6s %-8s %-9s %s\n' NAME PORT STATE PID SOURCE
+  printf '%-16s %-6s %-13s %-9s %-19s %s\n' NAME PORT STATE PID "BUILT" SOURCE
   local m
   for m in "$SBX_ROOT"/*/manifest.json; do
     [ -f "$m" ] || continue
-    local n p src pid state
+    local n p src pid state bpath built fresh
     n="$(python3 -c "import json;print(json.load(open('$m'))['name'])")"
     p="$(python3 -c "import json;print(json.load(open('$m'))['port'])")"
     src="$(python3 -c "import json;print(json.load(open('$m'))['source'])")"
-    pid="$(daemon_pid "$n")"; state="stopped"; daemon_alive "$n" && state="running"
-    printf '%-16s %-6s %-8s %-9s %s\n' "$n" "$p" "$state" "${pid:-–}" "$src"
+    pid="$(daemon_pid "$n")"
+    case "$(daemon_health "$n")" in
+      running)      state="running";;
+      unresponsive) state="running(!resp)";;
+      *)            state="stopped";;
+    esac
+    bpath="$(sdir "$n")/bin/engram"; built="$([ -f "$bpath" ] && bin_built_at "$bpath" || echo unknown)"
+    fresh="$(_binary_freshness "$n")"; [ -n "$fresh" ] && src="[STALE] $src"
+    printf '%-16s %-6s %-13s %-9s %-19s %s\n' "$n" "$p" "$state" "${pid:-–}" "$built" "$src"
   done
+  info "state 'running(!resp)' = process alive but /api/stats didn't answer — see: nsbx status <name>"
 }
 cmd_status(){
-  local name="$1"; mexists "$name" || die "no such sandbox: $name"
+  local name="$1"; mexists "$name" || die "no such sandbox: $name (run: nsbx list to see what exists)"
   python3 -m json.tool "$(manifest "$name")"
-  daemon_alive "$name" && echo "state: running (pid $(daemon_pid "$name")) stats: $(sbx_stats "$name")" || echo "state: stopped"
+  local bpath; bpath="$(sdir "$name")/bin/engram"
+  if [ -f "$bpath" ]; then
+    echo "binary: sha=$(sha "$bpath" | cut -c1-12) built=$(bin_built_at "$bpath")"
+    local fresh; fresh="$(_binary_freshness "$name")"
+    [ -n "$fresh" ] && printf '%s%s%s\n' "$C_YEL" "$fresh" "$C_0"
+  fi
+  case "$(daemon_health "$name")" in
+    running)
+      echo "state: running (pid $(daemon_pid "$name")) stats: $(sbx_stats "$name")";;
+    unresponsive)
+      printf '%sstate: running but NOT RESPONDING%s (pid %s) — process alive, /api/stats returned nothing.\n' "$C_RED" "$C_0" "$(daemon_pid "$name")"
+      info "check: tail -50 $(sdir "$name")/logs/daemon.log   |   next: kill -9 $(daemon_pid "$name") && nsbx up $name"
+      ;;
+    *) echo "state: stopped";;
+  esac
   [ -f "$(sdir "$name")/validate.json" ] && { echo "--- last validation ---"; python3 -m json.tool "$(sdir "$name")/validate.json"; }
 }