fix: build engram from source in Docker image

The engram binary was never published to Artifact Registry (foundation-dev/engram
package does not exist). Updated Dockerfile to build engram from source using the
El SDK packages (el-elc, el-runtime-c, el-runtime-h) from foundation-dev.

Also:
- Switch runtime base to Ubuntu 24.04 (GLIBC 2.39 required by elc-compiled binaries)
- Add -lm to engram link flags (el_runtime.c uses pow/sqrt/log/sin/cos/exp)
- Update deploy-gke.yaml to clone neuron-technologies/engram into build context

.gitea/workflows/deploy-gke.yaml

@@ -87,6 +87,15 @@ jobs:
           echo "slot=${SLOT}" >> "$GITEA_OUTPUT"
           echo "  Deploying to slot: ${SLOT}"
 
+      - name: Clone engram source for Docker build context
+        run: |
+          # The Dockerfile builds engram from source (no published AR package).
+          # Clone the engram repo into ./engram/ so it's available in the build context.
+          git clone http://34.31.145.131/neuron-technologies/engram.git \
+            --depth=1 --branch=main \
+            engram
+          echo "Engram source ready at ./engram/src/server.el"
+
       - name: Build and push Docker image
         env:
           GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}

Dockerfile

@@ -2,10 +2,13 @@
 #
 # Build strategy:
 #   1. Download the pre-built linux/amd64 soul binary (package: neuron-soul)
-#      and engram HTTP server binary (package: engram) from Artifact Registry.
+#      from Artifact Registry (foundation-dev).
-#      Both are built by CI and published as generic artifacts.
+#   2. Download the El SDK from Artifact Registry and build engram from source
-#   2. Package both in a minimal Ubuntu 22.04 runtime.
+#      (the neuron-technologies/engram repo is a git submodule). Engram has
-#   3. entrypoint.sh starts engram on :8742, waits for it to be healthy,
+#      never been published as a standalone Artifact Registry package.
+#   3. Package both in an Ubuntu 24.04 runtime image (GLIBC 2.39 required by
+#      binaries compiled on Ubuntu 24.04 CI runners).
+#   4. entrypoint.sh starts engram on :8742, waits for it to be healthy,
 #      then starts the soul with ENGRAM_URL pointing at it (HTTP mode).
 #
 # Required env vars (injected via ExternalSecret at runtime):
@@ -14,13 +17,11 @@
 #   ENGRAM_DATA_DIR
 
 ARG SOUL_VERSION=latest
-ARG ENGRAM_VERSION=latest
 
-FROM ubuntu:22.04 AS downloader
+# ── Stage 1: Download neuron-soul + El SDK from Artifact Registry ─────────────
+FROM ubuntu:24.04 AS downloader
 
 ARG SOUL_VERSION
-ARG ENGRAM_VERSION
-ARG GCP_SA_KEY
 
 RUN apt-get update -qq && \
     apt-get install -y --no-install-recommends \
@@ -36,15 +37,14 @@ RUN apt-get update -qq && \
     apt-get install -y --no-install-recommends google-cloud-cli && \
     rm -rf /var/lib/apt/lists/*
 
-# Authenticate and download both binaries from Artifact Registry.
 RUN --mount=type=secret,id=gcp_sa_key \
-    GCP_SA_KEY_FILE=$(cat /run/secrets/gcp_sa_key 2>/dev/null || echo "") && \
+    GCP_SA_KEY=$(cat /run/secrets/gcp_sa_key 2>/dev/null || echo "") && \
-    if [ -n "$GCP_SA_KEY_FILE" ]; then \
+    if [ -n "$GCP_SA_KEY" ]; then \
-      echo "$GCP_SA_KEY_FILE" > /tmp/gcp-key.json && \
+      echo "$GCP_SA_KEY" > /tmp/gcp-key.json && \
       gcloud auth activate-service-account --key-file=/tmp/gcp-key.json; \
     fi && \
     gcloud config set project neuron-785695 && \
-    mkdir -p /tmp/soul /tmp/engram && \
+    mkdir -p /tmp/soul /tmp/el-sdk && \
     \
     # ── soul ──────────────────────────────────────────────────────────────── \
     if [ "${SOUL_VERSION}" = "latest" ]; then \
@@ -70,46 +70,84 @@ RUN --mount=type=secret,id=gcp_sa_key \
     mv /tmp/soul/neuron* /tmp/soul/neuron 2>/dev/null || true && \
     chmod +x /tmp/soul/neuron && \
     \
-    # ── engram ────────────────────────────────────────────────────────────── \
+    # ── El SDK (needed to build engram from source) ────────────────────────── \
-    if [ "${ENGRAM_VERSION}" = "latest" ]; then \
+    ELC_VER=$(gcloud artifacts versions list \
-      ENGRAM_VER=$(gcloud artifacts versions list \
+      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
-        --repository=foundation-dev \
+      --package=el-elc --sort-by="~createTime" --limit=1 \
-        --location=us-central1 \
+      --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \
-        --project=neuron-785695 \
-        --package=engram \
-        --sort-by="~createTime" \
-        --limit=1 \
-        --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}'); \
-    else \
-      ENGRAM_VER="${ENGRAM_VERSION}"; \
-    fi && \
-    echo "Downloading engram@${ENGRAM_VER}" && \
     gcloud artifacts generic download \
-      --repository=foundation-dev \
+      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
-      --location=us-central1 \
+      --package=el-elc --version="${ELC_VER}" --destination=/tmp/el-sdk/ && \
-      --project=neuron-785695 \
+    mv /tmp/el-sdk/elc* /tmp/el-sdk/elc 2>/dev/null || true && \
-      --package=engram \
+    chmod +x /tmp/el-sdk/elc && \
-      --version="${ENGRAM_VER}" \
-      --destination=/tmp/engram/ && \
-    mv /tmp/engram/engram* /tmp/engram/engram 2>/dev/null || true && \
-    chmod +x /tmp/engram/engram && \
     \
-    rm -f /tmp/gcp-key.json
+    RC_VER=$(gcloud artifacts versions list \
+      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
+      --package=el-runtime-c --sort-by="~createTime" --limit=1 \
+      --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \
+    gcloud artifacts generic download \
+      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
+      --package=el-runtime-c --version="${RC_VER}" --destination=/tmp/el-sdk/ && \
+    mv /tmp/el-sdk/el_runtime.c* /tmp/el-sdk/el_runtime.c 2>/dev/null || true && \
+    \
+    RH_VER=$(gcloud artifacts versions list \
+      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
+      --package=el-runtime-h --sort-by="~createTime" --limit=1 \
+      --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \
+    gcloud artifacts generic download \
+      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
+      --package=el-runtime-h --version="${RH_VER}" --destination=/tmp/el-sdk/ && \
+    mv /tmp/el-sdk/el_runtime.h* /tmp/el-sdk/el_runtime.h 2>/dev/null || true && \
+    \
+    rm -f /tmp/gcp-key.json && \
+    echo "Downloads complete:" && ls -lh /tmp/soul/ /tmp/el-sdk/
 
-# Runtime image — minimal Ubuntu 22.04 with only what both binaries need.
+# ── Stage 2: Build engram from source ────────────────────────────────────────
-FROM ubuntu:22.04
+FROM ubuntu:24.04 AS engram-builder
 
 RUN apt-get update -qq && \
     apt-get install -y --no-install-recommends \
       ca-certificates \
-      libcurl4 \
+      gcc \
+      libc6-dev \
+      libcurl4-openssl-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY --from=downloader /tmp/el-sdk/elc           /usr/local/bin/elc
+COPY --from=downloader /tmp/el-sdk/el_runtime.c  /usr/local/lib/el/el_runtime.c
+COPY --from=downloader /tmp/el-sdk/el_runtime.h  /usr/local/lib/el/el_runtime.h
+
+# engram source is expected at ./engram/src/server.el in the build context.
+# The deploy-gke.yaml CI must clone neuron-technologies/engram alongside this repo.
+COPY engram/src/server.el /build/src/server.el
+
+RUN mkdir -p /build/dist && \
+    /usr/local/bin/elc /build/src/server.el > /build/dist/engram.c && \
+    echo "Compiled server.el -> engram.c ($(wc -l < /build/dist/engram.c) lines)" && \
+    cc -std=c11 -O2 \
+      -I /usr/local/lib/el \
+      -o /build/dist/engram \
+      /build/dist/engram.c \
+      /usr/local/lib/el/el_runtime.c \
+      -lcurl -lpthread -lm && \
+    echo "Built engram:" && ls -lh /build/dist/engram && \
+    chmod +x /build/dist/engram
+
+# ── Stage 3: Runtime image ───────────────────────────────────────────────────
+# Ubuntu 24.04: GLIBC 2.39 satisfies both neuron-soul and engram binary deps.
+FROM ubuntu:24.04
+
+RUN apt-get update -qq && \
+    apt-get install -y --no-install-recommends \
+      ca-certificates \
+      libcurl4t64 \
       curl && \
     rm -rf /var/lib/apt/lists/* && \
-    useradd -r -u 1000 -m -s /bin/bash soul
+    useradd -r -u 10000 -m -s /bin/bash soul
 
-COPY --from=downloader /tmp/soul/neuron   /usr/local/bin/neuron
+COPY --from=downloader    /tmp/soul/neuron        /usr/local/bin/neuron
-COPY --from=downloader /tmp/engram/engram /usr/local/bin/engram
+COPY --from=engram-builder /build/dist/engram     /usr/local/bin/engram
-COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+COPY entrypoint.sh                                /usr/local/bin/entrypoint.sh
 
 RUN chmod +x /usr/local/bin/neuron /usr/local/bin/engram /usr/local/bin/entrypoint.sh