fix: build engram from source in Docker image

The engram binary was never published to Artifact Registry (foundation-dev/engram
package does not exist). Updated Dockerfile to build engram from source using the
El SDK packages (el-elc, el-runtime-c, el-runtime-h) from foundation-dev.

Also:
- Switch runtime base to Ubuntu 24.04 (GLIBC 2.39 required by elc-compiled binaries)
- Add -lm to engram link flags (el_runtime.c uses pow/sqrt/log/sin/cos/exp)
- Update deploy-gke.yaml to clone neuron-technologies/engram into build context

.gitea/workflows/deploy-gke.yaml

@@ -87,6 +87,15 @@ jobs:
           echo "slot=${SLOT}" >> "$GITEA_OUTPUT"
           echo "  Deploying to slot: ${SLOT}"
 
+      - name: Clone engram source for Docker build context
+        run: |
+          # The Dockerfile builds engram from source (no published AR package).
+          # Clone the engram repo into ./engram/ so it's available in the build context.
+          git clone http://34.31.145.131/neuron-technologies/engram.git \
+            --depth=1 --branch=main \
+            engram
+          echo "Engram source ready at ./engram/src/server.el"
+
       - name: Build and push Docker image
         env:
           GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}

Dockerfile

@@ -1,25 +1,27 @@
 # Neuron Soul — GKE container image
 #
 # Build strategy:
-#   1. Download the pre-built linux/amd64 soul binary from Artifact Registry
-#      (package: neuron-soul, repository: foundation-dev).
-#      The binary is built by CI from soul.el and published as a generic artifact.
-#   2. Package it in a minimal Ubuntu 22.04 runtime with glibc and libcurl.
-#
-# The soul runs in file mode (no HTTP Engram sidecar):
-#   - SOUL_ENGRAM_PATH=/data/snapshot.json  → reads/writes engram from mounted PVC
-#   - ENGRAM_URL must NOT be set            → absence triggers file mode
+#   1. Download the pre-built linux/amd64 soul binary (package: neuron-soul)
+#      from Artifact Registry (foundation-dev).
+#   2. Download the El SDK from Artifact Registry and build engram from source
+#      (the neuron-technologies/engram repo is a git submodule). Engram has
+#      never been published as a standalone Artifact Registry package.
+#   3. Package both in an Ubuntu 24.04 runtime image (GLIBC 2.39 required by
+#      binaries compiled on Ubuntu 24.04 CI runners).
+#   4. entrypoint.sh starts engram on :8742, waits for it to be healthy,
+#      then starts the soul with ENGRAM_URL pointing at it (HTTP mode).
 #
 # Required env vars (injected via ExternalSecret at runtime):
 #   NEURON_PORT, NEURON_LLM_0_URL, NEURON_LLM_0_KEY, NEURON_LLM_0_FORMAT,
-#   SOUL_CGI_ID, SOUL_IDENTITY, NEURON_TOKEN, NEURON_API_URL, SOUL_ENGRAM_PATH
+#   SOUL_CGI_ID, SOUL_IDENTITY, NEURON_TOKEN, NEURON_API_URL, ENGRAM_URL,
+#   ENGRAM_DATA_DIR
 
 ARG SOUL_VERSION=latest
 
-FROM ubuntu:22.04 AS downloader
+# ── Stage 1: Download neuron-soul + El SDK from Artifact Registry ─────────────
+FROM ubuntu:24.04 AS downloader
 
 ARG SOUL_VERSION
-ARG GCP_SA_KEY
 
 RUN apt-get update -qq && \
     apt-get install -y --no-install-recommends \
@@ -35,19 +37,18 @@ RUN apt-get update -qq && \
     apt-get install -y --no-install-recommends google-cloud-cli && \
     rm -rf /var/lib/apt/lists/*
 
-# Authenticate and download the soul binary from Artifact Registry.
-# SOUL_VERSION is the 8-char git SHA tag published by CI (e.g. ea271d5c).
-# The binary is stored as a generic artifact — download to /tmp/soul/neuron.
 RUN --mount=type=secret,id=gcp_sa_key \
-    GCP_SA_KEY_FILE=$(cat /run/secrets/gcp_sa_key 2>/dev/null || echo "") && \
-    if [ -n "$GCP_SA_KEY_FILE" ]; then \
-      echo "$GCP_SA_KEY_FILE" > /tmp/gcp-key.json && \
+    GCP_SA_KEY=$(cat /run/secrets/gcp_sa_key 2>/dev/null || echo "") && \
+    if [ -n "$GCP_SA_KEY" ]; then \
+      echo "$GCP_SA_KEY" > /tmp/gcp-key.json && \
       gcloud auth activate-service-account --key-file=/tmp/gcp-key.json; \
     fi && \
     gcloud config set project neuron-785695 && \
-    mkdir -p /tmp/soul && \
+    mkdir -p /tmp/soul /tmp/el-sdk && \
+    \
+    # ── soul ──────────────────────────────────────────────────────────────── \
     if [ "${SOUL_VERSION}" = "latest" ]; then \
-      VERSION=$(gcloud artifacts versions list \
+      SOUL_VER=$(gcloud artifacts versions list \
         --repository=foundation-dev \
         --location=us-central1 \
         --project=neuron-785695 \
@@ -56,36 +57,101 @@ RUN --mount=type=secret,id=gcp_sa_key \
         --limit=1 \
         --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}'); \
     else \
-      VERSION="${SOUL_VERSION}"; \
+      SOUL_VER="${SOUL_VERSION}"; \
     fi && \
-    echo "Downloading neuron-soul@${VERSION}" && \
+    echo "Downloading neuron-soul@${SOUL_VER}" && \
     gcloud artifacts generic download \
       --repository=foundation-dev \
       --location=us-central1 \
       --project=neuron-785695 \
       --package=neuron-soul \
-      --version="${VERSION}" \
+      --version="${SOUL_VER}" \
       --destination=/tmp/soul/ && \
     mv /tmp/soul/neuron* /tmp/soul/neuron 2>/dev/null || true && \
     chmod +x /tmp/soul/neuron && \
-    rm -f /tmp/gcp-key.json
+    \
+    # ── El SDK (needed to build engram from source) ────────────────────────── \
+    ELC_VER=$(gcloud artifacts versions list \
+      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
+      --package=el-elc --sort-by="~createTime" --limit=1 \
+      --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \
+    gcloud artifacts generic download \
+      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
+      --package=el-elc --version="${ELC_VER}" --destination=/tmp/el-sdk/ && \
+    mv /tmp/el-sdk/elc* /tmp/el-sdk/elc 2>/dev/null || true && \
+    chmod +x /tmp/el-sdk/elc && \
+    \
+    RC_VER=$(gcloud artifacts versions list \
+      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
+      --package=el-runtime-c --sort-by="~createTime" --limit=1 \
+      --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \
+    gcloud artifacts generic download \
+      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
+      --package=el-runtime-c --version="${RC_VER}" --destination=/tmp/el-sdk/ && \
+    mv /tmp/el-sdk/el_runtime.c* /tmp/el-sdk/el_runtime.c 2>/dev/null || true && \
+    \
+    RH_VER=$(gcloud artifacts versions list \
+      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
+      --package=el-runtime-h --sort-by="~createTime" --limit=1 \
+      --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \
+    gcloud artifacts generic download \
+      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
+      --package=el-runtime-h --version="${RH_VER}" --destination=/tmp/el-sdk/ && \
+    mv /tmp/el-sdk/el_runtime.h* /tmp/el-sdk/el_runtime.h 2>/dev/null || true && \
+    \
+    rm -f /tmp/gcp-key.json && \
+    echo "Downloads complete:" && ls -lh /tmp/soul/ /tmp/el-sdk/
 
-# Runtime image — minimal Ubuntu 22.04 with only what the soul binary needs.
-FROM ubuntu:22.04
+# ── Stage 2: Build engram from source ────────────────────────────────────────
+FROM ubuntu:24.04 AS engram-builder
 
 RUN apt-get update -qq && \
     apt-get install -y --no-install-recommends \
       ca-certificates \
-      libcurl4 \
+      gcc \
+      libc6-dev \
+      libcurl4-openssl-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY --from=downloader /tmp/el-sdk/elc           /usr/local/bin/elc
+COPY --from=downloader /tmp/el-sdk/el_runtime.c  /usr/local/lib/el/el_runtime.c
+COPY --from=downloader /tmp/el-sdk/el_runtime.h  /usr/local/lib/el/el_runtime.h
+
+# engram source is expected at ./engram/src/server.el in the build context.
+# The deploy-gke.yaml CI must clone neuron-technologies/engram alongside this repo.
+COPY engram/src/server.el /build/src/server.el
+
+RUN mkdir -p /build/dist && \
+    /usr/local/bin/elc /build/src/server.el > /build/dist/engram.c && \
+    echo "Compiled server.el -> engram.c ($(wc -l < /build/dist/engram.c) lines)" && \
+    cc -std=c11 -O2 \
+      -I /usr/local/lib/el \
+      -o /build/dist/engram \
+      /build/dist/engram.c \
+      /usr/local/lib/el/el_runtime.c \
+      -lcurl -lpthread -lm && \
+    echo "Built engram:" && ls -lh /build/dist/engram && \
+    chmod +x /build/dist/engram
+
+# ── Stage 3: Runtime image ───────────────────────────────────────────────────
+# Ubuntu 24.04: GLIBC 2.39 satisfies both neuron-soul and engram binary deps.
+FROM ubuntu:24.04
+
+RUN apt-get update -qq && \
+    apt-get install -y --no-install-recommends \
+      ca-certificates \
+      libcurl4t64 \
       curl && \
     rm -rf /var/lib/apt/lists/* && \
-    useradd -r -u 1000 -m -s /bin/bash soul
+    useradd -r -u 10000 -m -s /bin/bash soul
 
-COPY --from=downloader /tmp/soul/neuron /usr/local/bin/neuron
-RUN chmod +x /usr/local/bin/neuron
+COPY --from=downloader    /tmp/soul/neuron        /usr/local/bin/neuron
+COPY --from=engram-builder /build/dist/engram     /usr/local/bin/engram
+COPY entrypoint.sh                                /usr/local/bin/entrypoint.sh
+
+RUN chmod +x /usr/local/bin/neuron /usr/local/bin/engram /usr/local/bin/entrypoint.sh
 
 # /data is the engram mount point (PVC at runtime).
-# Create it owned by soul user so the binary can write snapshot.json.
 RUN mkdir -p /data && chown soul:soul /data
 
 USER soul
@@ -93,9 +159,10 @@ WORKDIR /home/soul
 
 EXPOSE 7770
 
-# SOUL_ENGRAM_PATH and other env vars are injected via k8s ExternalSecret.
-# ENGRAM_URL must NOT be set — its absence triggers file mode.
+# ENGRAM_URL and ENGRAM_DATA_DIR trigger HTTP mode in the soul.
+# SOUL_ENGRAM_PATH must NOT be set — its presence would enable legacy file mode.
 ENV NEURON_PORT=7770 \
-    SOUL_ENGRAM_PATH=/data/snapshot.json
+    ENGRAM_URL=http://localhost:8742 \
+    ENGRAM_DATA_DIR=/data
 
-ENTRYPOINT ["/usr/local/bin/neuron"]
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

entrypoint.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+# entrypoint.sh — start engram then soul, both in a single container.
+#
+# Engram runs as a background HTTP server on localhost:8742.
+# Soul starts once engram reports healthy, with ENGRAM_URL pointing at it.
+#
+# Data directory /data is the PVC mount point — engram reads/writes its
+# snapshot.json (and any future data files) there.
+
+set -eu
+
+ENGRAM_PORT="${ENGRAM_PORT:-8742}"
+ENGRAM_DATA_DIR="${ENGRAM_DATA_DIR:-/data}"
+ENGRAM_HEALTH_URL="http://localhost:${ENGRAM_PORT}/health"
+
+# Ensure the data directory exists (PVC may be mounted but empty on first boot)
+mkdir -p "$ENGRAM_DATA_DIR"
+
+# Start engram in the background
+echo "[entrypoint] starting engram on :${ENGRAM_PORT} data_dir=${ENGRAM_DATA_DIR}"
+ENGRAM_BIND=":${ENGRAM_PORT}" \
+ENGRAM_DATA_DIR="$ENGRAM_DATA_DIR" \
+  /usr/local/bin/engram &
+
+ENGRAM_PID=$!
+
+# Wait for engram to become healthy (up to 30s)
+echo "[entrypoint] waiting for engram..."
+TRIES=0
+until curl -sf "$ENGRAM_HEALTH_URL" > /dev/null 2>&1; do
+    TRIES=$((TRIES + 1))
+    if [ "$TRIES" -ge 30 ]; then
+        echo "[entrypoint] ERROR: engram did not become healthy after 30s" >&2
+        kill "$ENGRAM_PID" 2>/dev/null || true
+        exit 1
+    fi
+    sleep 1
+done
+echo "[entrypoint] engram ready"
+
+# Start soul — it takes over as PID 1's foreground process.
+# SOUL_ENGRAM_PATH must NOT be set; ENGRAM_URL triggers HTTP mode.
+exec /usr/local/bin/neuron