feat(docker): switch soul image to engram HTTP mode with entrypoint script

Download both neuron-soul and engram binaries from Artifact Registry.
entrypoint.sh starts engram on :8742, waits for /health, then launches
the soul with ENGRAM_URL set. Removes SOUL_ENGRAM_PATH / file mode.

Dockerfile

@@ -1,24 +1,25 @@
 # Neuron Soul — GKE container image
 #
 # Build strategy:
-#   1. Download the pre-built linux/amd64 soul binary from Artifact Registry
+#   1. Download the pre-built linux/amd64 soul binary (package: neuron-soul)
-#      (package: neuron-soul, repository: foundation-dev).
+#      and engram HTTP server binary (package: engram) from Artifact Registry.
-#      The binary is built by CI from soul.el and published as a generic artifact.
+#      Both are built by CI and published as generic artifacts.
-#   2. Package it in a minimal Ubuntu 22.04 runtime with glibc and libcurl.
+#   2. Package both in a minimal Ubuntu 22.04 runtime.
-#
+#   3. entrypoint.sh starts engram on :8742, waits for it to be healthy,
-# The soul runs in file mode (no HTTP Engram sidecar):
+#      then starts the soul with ENGRAM_URL pointing at it (HTTP mode).
-#   - SOUL_ENGRAM_PATH=/data/snapshot.json  → reads/writes engram from mounted PVC
-#   - ENGRAM_URL must NOT be set            → absence triggers file mode
 #
 # Required env vars (injected via ExternalSecret at runtime):
 #   NEURON_PORT, NEURON_LLM_0_URL, NEURON_LLM_0_KEY, NEURON_LLM_0_FORMAT,
-#   SOUL_CGI_ID, SOUL_IDENTITY, NEURON_TOKEN, NEURON_API_URL, SOUL_ENGRAM_PATH
+#   SOUL_CGI_ID, SOUL_IDENTITY, NEURON_TOKEN, NEURON_API_URL, ENGRAM_URL,
+#   ENGRAM_DATA_DIR
 
 ARG SOUL_VERSION=latest
+ARG ENGRAM_VERSION=latest
 
 FROM ubuntu:22.04 AS downloader
 
 ARG SOUL_VERSION
+ARG ENGRAM_VERSION
 ARG GCP_SA_KEY
 
 RUN apt-get update -qq && \
@@ -35,9 +36,7 @@ RUN apt-get update -qq && \
     apt-get install -y --no-install-recommends google-cloud-cli && \
     rm -rf /var/lib/apt/lists/*
 
-# Authenticate and download the soul binary from Artifact Registry.
+# Authenticate and download both binaries from Artifact Registry.
-# SOUL_VERSION is the 8-char git SHA tag published by CI (e.g. ea271d5c).
-# The binary is stored as a generic artifact — download to /tmp/soul/neuron.
 RUN --mount=type=secret,id=gcp_sa_key \
     GCP_SA_KEY_FILE=$(cat /run/secrets/gcp_sa_key 2>/dev/null || echo "") && \
     if [ -n "$GCP_SA_KEY_FILE" ]; then \
@@ -45,9 +44,11 @@ RUN --mount=type=secret,id=gcp_sa_key \
       gcloud auth activate-service-account --key-file=/tmp/gcp-key.json; \
     fi && \
     gcloud config set project neuron-785695 && \
-    mkdir -p /tmp/soul && \
+    mkdir -p /tmp/soul /tmp/engram && \
+    \
+    # ── soul ──────────────────────────────────────────────────────────────── \
     if [ "${SOUL_VERSION}" = "latest" ]; then \
-      VERSION=$(gcloud artifacts versions list \
+      SOUL_VER=$(gcloud artifacts versions list \
         --repository=foundation-dev \
         --location=us-central1 \
         --project=neuron-785695 \
@@ -56,21 +57,46 @@ RUN --mount=type=secret,id=gcp_sa_key \
         --limit=1 \
         --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}'); \
     else \
-      VERSION="${SOUL_VERSION}"; \
+      SOUL_VER="${SOUL_VERSION}"; \
     fi && \
-    echo "Downloading neuron-soul@${VERSION}" && \
+    echo "Downloading neuron-soul@${SOUL_VER}" && \
     gcloud artifacts generic download \
       --repository=foundation-dev \
       --location=us-central1 \
       --project=neuron-785695 \
       --package=neuron-soul \
-      --version="${VERSION}" \
+      --version="${SOUL_VER}" \
       --destination=/tmp/soul/ && \
     mv /tmp/soul/neuron* /tmp/soul/neuron 2>/dev/null || true && \
     chmod +x /tmp/soul/neuron && \
+    \
+    # ── engram ────────────────────────────────────────────────────────────── \
+    if [ "${ENGRAM_VERSION}" = "latest" ]; then \
+      ENGRAM_VER=$(gcloud artifacts versions list \
+        --repository=foundation-dev \
+        --location=us-central1 \
+        --project=neuron-785695 \
+        --package=engram \
+        --sort-by="~createTime" \
+        --limit=1 \
+        --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}'); \
+    else \
+      ENGRAM_VER="${ENGRAM_VERSION}"; \
+    fi && \
+    echo "Downloading engram@${ENGRAM_VER}" && \
+    gcloud artifacts generic download \
+      --repository=foundation-dev \
+      --location=us-central1 \
+      --project=neuron-785695 \
+      --package=engram \
+      --version="${ENGRAM_VER}" \
+      --destination=/tmp/engram/ && \
+    mv /tmp/engram/engram* /tmp/engram/engram 2>/dev/null || true && \
+    chmod +x /tmp/engram/engram && \
+    \
     rm -f /tmp/gcp-key.json
 
-# Runtime image — minimal Ubuntu 22.04 with only what the soul binary needs.
+# Runtime image — minimal Ubuntu 22.04 with only what both binaries need.
 FROM ubuntu:22.04
 
 RUN apt-get update -qq && \
@@ -81,11 +107,13 @@ RUN apt-get update -qq && \
     rm -rf /var/lib/apt/lists/* && \
     useradd -r -u 1000 -m -s /bin/bash soul
 
-COPY --from=downloader /tmp/soul/neuron /usr/local/bin/neuron
+COPY --from=downloader /tmp/soul/neuron   /usr/local/bin/neuron
-RUN chmod +x /usr/local/bin/neuron
+COPY --from=downloader /tmp/engram/engram /usr/local/bin/engram
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+
+RUN chmod +x /usr/local/bin/neuron /usr/local/bin/engram /usr/local/bin/entrypoint.sh
 
 # /data is the engram mount point (PVC at runtime).
-# Create it owned by soul user so the binary can write snapshot.json.
 RUN mkdir -p /data && chown soul:soul /data
 
 USER soul
@@ -93,9 +121,10 @@ WORKDIR /home/soul
 
 EXPOSE 7770
 
-# SOUL_ENGRAM_PATH and other env vars are injected via k8s ExternalSecret.
+# ENGRAM_URL and ENGRAM_DATA_DIR trigger HTTP mode in the soul.
-# ENGRAM_URL must NOT be set — its absence triggers file mode.
+# SOUL_ENGRAM_PATH must NOT be set — its presence would enable legacy file mode.
 ENV NEURON_PORT=7770 \
-    SOUL_ENGRAM_PATH=/data/snapshot.json
+    ENGRAM_URL=http://localhost:8742 \
+    ENGRAM_DATA_DIR=/data
 
-ENTRYPOINT ["/usr/local/bin/neuron"]
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

entrypoint.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+# entrypoint.sh — start engram then soul, both in a single container.
+#
+# Engram runs as a background HTTP server on localhost:8742.
+# Soul starts once engram reports healthy, with ENGRAM_URL pointing at it.
+#
+# Data directory /data is the PVC mount point — engram reads/writes its
+# snapshot.json (and any future data files) there.
+
+set -eu
+
+ENGRAM_PORT="${ENGRAM_PORT:-8742}"
+ENGRAM_DATA_DIR="${ENGRAM_DATA_DIR:-/data}"
+ENGRAM_HEALTH_URL="http://localhost:${ENGRAM_PORT}/health"
+
+# Ensure the data directory exists (PVC may be mounted but empty on first boot)
+mkdir -p "$ENGRAM_DATA_DIR"
+
+# Start engram in the background
+echo "[entrypoint] starting engram on :${ENGRAM_PORT} data_dir=${ENGRAM_DATA_DIR}"
+ENGRAM_BIND=":${ENGRAM_PORT}" \
+ENGRAM_DATA_DIR="$ENGRAM_DATA_DIR" \
+  /usr/local/bin/engram &
+
+ENGRAM_PID=$!
+
+# Wait for engram to become healthy (up to 30s)
+echo "[entrypoint] waiting for engram..."
+TRIES=0
+until curl -sf "$ENGRAM_HEALTH_URL" > /dev/null 2>&1; do
+    TRIES=$((TRIES + 1))
+    if [ "$TRIES" -ge 30 ]; then
+        echo "[entrypoint] ERROR: engram did not become healthy after 30s" >&2
+        kill "$ENGRAM_PID" 2>/dev/null || true
+        exit 1
+    fi
+    sleep 1
+done
+echo "[entrypoint] engram ready"
+
+# Start soul — it takes over as PID 1's foreground process.
+# SOUL_ENGRAM_PATH must NOT be set; ENGRAM_URL triggers HTTP mode.
+exec /usr/local/bin/neuron