fix(ci/docker): pre-download artifacts before build, remove --secret

The Dockerfile's --mount=type=secret path was corrupting the SA key JSON due to control character handling differences. Pre-download soul + El SDK in the CI workflow (using already-authenticated gcloud) and COPY them from the build context. No credentials needed inside the Docker build.
2026-06-18 14:04:03 -05:00
parent fdd946b3d4
commit b563fff062
2 changed files with 88 additions and 111 deletions
@@ -46,8 +46,7 @@ jobs:
        run: |
          apt-get update -qq
          apt-get install -y --no-install-recommends \
-            ca-certificates curl apt-transport-https kubectl \
-            docker-buildx-plugin
+            ca-certificates curl apt-transport-https kubectl
          echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" \
            > /etc/apt/sources.list.d/google-cloud-sdk.list
          apt-get update -qq && apt-get install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin
@@ -109,6 +108,66 @@ jobs:
          echo "slot=${SLOT}" >> "$GITEA_OUTPUT"
          echo "  Deploying to slot: ${SLOT}"

+      - name: Prepare build artifacts
+        run: |
+          # Pre-download soul binary and El SDK so the Dockerfile can COPY them
+          # from the build context instead of authenticating inside the build.
+          mkdir -p build-artifacts
+
+          # ── soul binary ────────────────────────────────────────────────────────
+          # ci.yaml publishes the soul binary to foundation-prod on every push.
+          # Download the latest version (the one just built by ci.yaml).
+          SOUL_VER=$(gcloud artifacts versions list \
+            --repository=foundation-prod \
+            --location=us-central1 \
+            --project=neuron-785695 \
+            --package=neuron-soul \
+            --sort-by="~createTime" \
+            --limit=1 \
+            --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}')
+          echo "Downloading neuron-soul@${SOUL_VER}"
+          gcloud artifacts generic download \
+            --repository=foundation-prod \
+            --location=us-central1 \
+            --project=neuron-785695 \
+            --package=neuron-soul \
+            --version="${SOUL_VER}" \
+            --destination=build-artifacts/
+          mv build-artifacts/neuron* build-artifacts/neuron 2>/dev/null || true
+          chmod +x build-artifacts/neuron
+
+          # ── El SDK (for engram source compilation inside the build) ────────────
+          ELC_VER=$(gcloud artifacts versions list \
+            --repository=foundation-prod --location=us-central1 --project=neuron-785695 \
+            --package=el-elc --sort-by="~createTime" --limit=1 \
+            --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}')
+          gcloud artifacts generic download \
+            --repository=foundation-prod --location=us-central1 --project=neuron-785695 \
+            --package=el-elc --version="${ELC_VER}" --destination=build-artifacts/
+          mv build-artifacts/elc* build-artifacts/elc 2>/dev/null || true
+          chmod +x build-artifacts/elc
+
+          RC_VER=$(gcloud artifacts versions list \
+            --repository=foundation-prod --location=us-central1 --project=neuron-785695 \
+            --package=el-runtime-c --sort-by="~createTime" --limit=1 \
+            --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}')
+          gcloud artifacts generic download \
+            --repository=foundation-prod --location=us-central1 --project=neuron-785695 \
+            --package=el-runtime-c --version="${RC_VER}" --destination=build-artifacts/
+          mv build-artifacts/el_runtime.c* build-artifacts/el_runtime.c 2>/dev/null || true
+
+          RH_VER=$(gcloud artifacts versions list \
+            --repository=foundation-prod --location=us-central1 --project=neuron-785695 \
+            --package=el-runtime-h --sort-by="~createTime" --limit=1 \
+            --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}')
+          gcloud artifacts generic download \
+            --repository=foundation-prod --location=us-central1 --project=neuron-785695 \
+            --package=el-runtime-h --version="${RH_VER}" --destination=build-artifacts/
+          mv build-artifacts/el_runtime.h* build-artifacts/el_runtime.h 2>/dev/null || true
+
+          echo "Build artifacts ready:"
+          ls -lh build-artifacts/
+
      - name: Clone engram source for Docker build context
        run: |
          # The Dockerfile builds engram from source (no published AR package).
@@ -119,17 +178,13 @@ jobs:
          echo "Engram source ready at ./engram/src/server.el"

      - name: Build and push Docker image
-        env:
-          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
-          DOCKER_BUILDKIT: "1"
        run: |
          IMAGE="${{ steps.vars.outputs.image }}"
-          SHA="${{ steps.vars.outputs.sha }}"

          echo "Building ${IMAGE}..."
+          # No --secret needed: artifacts are pre-downloaded into build-artifacts/
+          # and the Dockerfile uses COPY to include them.
          docker build \
-            --build-arg SOUL_VERSION="${SHA}" \
-            --secret id=gcp_sa_key,env=GCP_SA_KEY \
            --tag "${IMAGE}" \
            --tag "us-central1-docker.pkg.dev/neuron-785695/neuron-api/neuron-soul:latest" \
            .
@@ -1,108 +1,28 @@
 # Neuron Soul — GKE container image
 #
 # Build strategy:
-#   1. Download the pre-built linux/amd64 soul binary (package: neuron-soul)
-#      from Artifact Registry (foundation-dev).
-#   2. Download the El SDK from Artifact Registry and build engram from source
-#      (the neuron-technologies/engram repo is a git submodule). Engram has
-#      never been published as a standalone Artifact Registry package.
-#   3. Package both in an Ubuntu 24.04 runtime image (GLIBC 2.39 required by
-#      binaries compiled on Ubuntu 24.04 CI runners).
+#   1. CI pre-downloads all artifacts from Artifact Registry into build-artifacts/
+#      (neuron soul binary, El compiler, El runtime). No GCP credentials are needed
+#      inside the build — all AR access happens in the CI workflow before docker build.
+#   2. Build engram from source (neuron-technologies/engram, cloned by CI into ./engram/).
+#   3. Package soul + engram in an Ubuntu 24.04 runtime image (GLIBC 2.39).
 #   4. entrypoint.sh starts engram on :8742, waits for it to be healthy,
 #      then starts the soul with ENGRAM_URL pointing at it (HTTP mode).
 #
+# Expected build context layout (prepared by deploy-gke.yaml before docker build):
+#   build-artifacts/neuron         — pre-built linux/amd64 soul binary
+#   build-artifacts/elc            — El compiler (for engram source compilation)
+#   build-artifacts/el_runtime.c   — El C runtime
+#   build-artifacts/el_runtime.h   — El C runtime header
+#   engram/src/server.el           — engram source (cloned by CI)
+#   entrypoint.sh                  — container entrypoint
+#
 # Required env vars (injected via ExternalSecret at runtime):
 #   NEURON_PORT, NEURON_LLM_0_URL, NEURON_LLM_0_KEY, NEURON_LLM_0_FORMAT,
 #   SOUL_CGI_ID, SOUL_IDENTITY, NEURON_TOKEN, NEURON_API_URL, ENGRAM_URL,
 #   ENGRAM_DATA_DIR

-ARG SOUL_VERSION=latest
-
-# ── Stage 1: Download neuron-soul + El SDK from Artifact Registry ─────────────
-FROM ubuntu:24.04 AS downloader
-
-ARG SOUL_VERSION
-
-RUN apt-get update -qq && \
-    apt-get install -y --no-install-recommends \
-      ca-certificates \
-      curl \
-      gnupg \
-      apt-transport-https && \
-    echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \
-      > /etc/apt/sources.list.d/google-cloud-sdk.list && \
-    curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg \
-      | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg && \
-    apt-get update -qq && \
-    apt-get install -y --no-install-recommends google-cloud-cli && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN --mount=type=secret,id=gcp_sa_key \
-    GCP_SA_KEY=$(cat /run/secrets/gcp_sa_key 2>/dev/null || echo "") && \
-    if [ -n "$GCP_SA_KEY" ]; then \
-      echo "$GCP_SA_KEY" > /tmp/gcp-key.json && \
-      gcloud auth activate-service-account --key-file=/tmp/gcp-key.json; \
-    fi && \
-    gcloud config set project neuron-785695 && \
-    mkdir -p /tmp/soul /tmp/el-sdk && \
-    \
-    # ── soul ──────────────────────────────────────────────────────────────── \
-    if [ "${SOUL_VERSION}" = "latest" ]; then \
-      SOUL_VER=$(gcloud artifacts versions list \
-        --repository=foundation-dev \
-        --location=us-central1 \
-        --project=neuron-785695 \
-        --package=neuron-soul \
-        --sort-by="~createTime" \
-        --limit=1 \
-        --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}'); \
-    else \
-      SOUL_VER="${SOUL_VERSION}"; \
-    fi && \
-    echo "Downloading neuron-soul@${SOUL_VER}" && \
-    gcloud artifacts generic download \
-      --repository=foundation-dev \
-      --location=us-central1 \
-      --project=neuron-785695 \
-      --package=neuron-soul \
-      --version="${SOUL_VER}" \
-      --destination=/tmp/soul/ && \
-    mv /tmp/soul/neuron* /tmp/soul/neuron 2>/dev/null || true && \
-    chmod +x /tmp/soul/neuron && \
-    \
-    # ── El SDK (needed to build engram from source) ────────────────────────── \
-    ELC_VER=$(gcloud artifacts versions list \
-      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
-      --package=el-elc --sort-by="~createTime" --limit=1 \
-      --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \
-    gcloud artifacts generic download \
-      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
-      --package=el-elc --version="${ELC_VER}" --destination=/tmp/el-sdk/ && \
-    mv /tmp/el-sdk/elc* /tmp/el-sdk/elc 2>/dev/null || true && \
-    chmod +x /tmp/el-sdk/elc && \
-    \
-    RC_VER=$(gcloud artifacts versions list \
-      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
-      --package=el-runtime-c --sort-by="~createTime" --limit=1 \
-      --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \
-    gcloud artifacts generic download \
-      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
-      --package=el-runtime-c --version="${RC_VER}" --destination=/tmp/el-sdk/ && \
-    mv /tmp/el-sdk/el_runtime.c* /tmp/el-sdk/el_runtime.c 2>/dev/null || true && \
-    \
-    RH_VER=$(gcloud artifacts versions list \
-      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
-      --package=el-runtime-h --sort-by="~createTime" --limit=1 \
-      --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \
-    gcloud artifacts generic download \
-      --repository=foundation-dev --location=us-central1 --project=neuron-785695 \
-      --package=el-runtime-h --version="${RH_VER}" --destination=/tmp/el-sdk/ && \
-    mv /tmp/el-sdk/el_runtime.h* /tmp/el-sdk/el_runtime.h 2>/dev/null || true && \
-    \
-    rm -f /tmp/gcp-key.json && \
-    echo "Downloads complete:" && ls -lh /tmp/soul/ /tmp/el-sdk/
-
-# ── Stage 2: Build engram from source ────────────────────────────────────────
+# ── Stage 1: Build engram from source ────────────────────────────────────────
 FROM ubuntu:24.04 AS engram-builder

 RUN apt-get update -qq && \
@@ -113,12 +33,13 @@ RUN apt-get update -qq && \
      libcurl4-openssl-dev && \
    rm -rf /var/lib/apt/lists/*

-COPY --from=downloader /tmp/el-sdk/elc           /usr/local/bin/elc
-COPY --from=downloader /tmp/el-sdk/el_runtime.c  /usr/local/lib/el/el_runtime.c
-COPY --from=downloader /tmp/el-sdk/el_runtime.h  /usr/local/lib/el/el_runtime.h
+# El SDK pre-downloaded by CI into build-artifacts/
+COPY build-artifacts/elc           /usr/local/bin/elc
+COPY build-artifacts/el_runtime.c  /usr/local/lib/el/el_runtime.c
+COPY build-artifacts/el_runtime.h  /usr/local/lib/el/el_runtime.h
+RUN chmod +x /usr/local/bin/elc

-# engram source is expected at ./engram/src/server.el in the build context.
-# The deploy-gke.yaml CI must clone neuron-technologies/engram alongside this repo.
+# engram source cloned by CI into ./engram/
 COPY engram/src/server.el /build/src/server.el

 RUN mkdir -p /build/dist && \
@@ -133,7 +54,7 @@ RUN mkdir -p /build/dist && \
    echo "Built engram:" && ls -lh /build/dist/engram && \
    chmod +x /build/dist/engram

-# ── Stage 3: Runtime image ───────────────────────────────────────────────────
+# ── Stage 2: Runtime image ───────────────────────────────────────────────────
 # Ubuntu 24.04: GLIBC 2.39 satisfies both neuron-soul and engram binary deps.
 FROM ubuntu:24.04

@@ -145,9 +66,10 @@ RUN apt-get update -qq && \
    rm -rf /var/lib/apt/lists/* && \
    useradd -r -u 10000 -m -s /bin/bash soul

-COPY --from=downloader    /tmp/soul/neuron        /usr/local/bin/neuron
-COPY --from=engram-builder /build/dist/engram     /usr/local/bin/engram
-COPY entrypoint.sh                                /usr/local/bin/entrypoint.sh
+# soul binary pre-downloaded by CI into build-artifacts/
+COPY build-artifacts/neuron        /usr/local/bin/neuron
+COPY --from=engram-builder /build/dist/engram /usr/local/bin/engram
+COPY entrypoint.sh                 /usr/local/bin/entrypoint.sh

 RUN chmod +x /usr/local/bin/neuron /usr/local/bin/engram /usr/local/bin/entrypoint.sh