diff --git a/.gitea/workflows/deploy-gke.yaml b/.gitea/workflows/deploy-gke.yaml index a7fab40..94d17b7 100644 --- a/.gitea/workflows/deploy-gke.yaml +++ b/.gitea/workflows/deploy-gke.yaml @@ -46,8 +46,7 @@ jobs: run: | apt-get update -qq apt-get install -y --no-install-recommends \ - ca-certificates curl apt-transport-https kubectl \ - docker-buildx-plugin + ca-certificates curl apt-transport-https kubectl echo "deb [trusted=yes] https://packages.cloud.google.com/apt cloud-sdk main" \ > /etc/apt/sources.list.d/google-cloud-sdk.list apt-get update -qq && apt-get install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin @@ -109,6 +108,66 @@ jobs: echo "slot=${SLOT}" >> "$GITEA_OUTPUT" echo " Deploying to slot: ${SLOT}" + - name: Prepare build artifacts + run: | + # Pre-download soul binary and El SDK so the Dockerfile can COPY them + # from the build context instead of authenticating inside the build. + mkdir -p build-artifacts + + # ── soul binary ──────────────────────────────────────────────────────── + # ci.yaml publishes the soul binary to foundation-prod on every push. + # Download the latest version (the one just built by ci.yaml). + SOUL_VER=$(gcloud artifacts versions list \ + --repository=foundation-prod \ + --location=us-central1 \ + --project=neuron-785695 \ + --package=neuron-soul \ + --sort-by="~createTime" \ + --limit=1 \ + --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') + echo "Downloading neuron-soul@${SOUL_VER}" + gcloud artifacts generic download \ + --repository=foundation-prod \ + --location=us-central1 \ + --project=neuron-785695 \ + --package=neuron-soul \ + --version="${SOUL_VER}" \ + --destination=build-artifacts/ + mv build-artifacts/neuron* build-artifacts/neuron 2>/dev/null || true + chmod +x build-artifacts/neuron + + # ── El SDK (for engram source compilation inside the build) ──────────── + ELC_VER=$(gcloud artifacts versions list \ + --repository=foundation-prod --location=us-central1 --project=neuron-785695 \ + --package=el-elc --sort-by="~createTime" --limit=1 \ + --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') + gcloud artifacts generic download \ + --repository=foundation-prod --location=us-central1 --project=neuron-785695 \ + --package=el-elc --version="${ELC_VER}" --destination=build-artifacts/ + mv build-artifacts/elc* build-artifacts/elc 2>/dev/null || true + chmod +x build-artifacts/elc + + RC_VER=$(gcloud artifacts versions list \ + --repository=foundation-prod --location=us-central1 --project=neuron-785695 \ + --package=el-runtime-c --sort-by="~createTime" --limit=1 \ + --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') + gcloud artifacts generic download \ + --repository=foundation-prod --location=us-central1 --project=neuron-785695 \ + --package=el-runtime-c --version="${RC_VER}" --destination=build-artifacts/ + mv build-artifacts/el_runtime.c* build-artifacts/el_runtime.c 2>/dev/null || true + + RH_VER=$(gcloud artifacts versions list \ + --repository=foundation-prod --location=us-central1 --project=neuron-785695 \ + --package=el-runtime-h --sort-by="~createTime" --limit=1 \ + --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') + gcloud artifacts generic download \ + --repository=foundation-prod --location=us-central1 --project=neuron-785695 \ + --package=el-runtime-h --version="${RH_VER}" --destination=build-artifacts/ + mv build-artifacts/el_runtime.h* build-artifacts/el_runtime.h 2>/dev/null || true + + echo "Build artifacts ready:" + ls -lh build-artifacts/ + - name: Clone engram source for Docker build context run: | # The Dockerfile builds engram from source (no published AR package). @@ -119,17 +178,13 @@ jobs: echo "Engram source ready at ./engram/src/server.el" - name: Build and push Docker image - env: - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} - DOCKER_BUILDKIT: "1" run: | IMAGE="${{ steps.vars.outputs.image }}" - SHA="${{ steps.vars.outputs.sha }}" echo "Building ${IMAGE}..." + # No --secret needed: artifacts are pre-downloaded into build-artifacts/ + # and the Dockerfile uses COPY to include them. docker build \ - --build-arg SOUL_VERSION="${SHA}" \ - --secret id=gcp_sa_key,env=GCP_SA_KEY \ --tag "${IMAGE}" \ --tag "us-central1-docker.pkg.dev/neuron-785695/neuron-api/neuron-soul:latest" \ . diff --git a/Dockerfile b/Dockerfile index 29f78db..6c61c09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,108 +1,28 @@ # Neuron Soul — GKE container image # # Build strategy: -# 1. Download the pre-built linux/amd64 soul binary (package: neuron-soul) -# from Artifact Registry (foundation-dev). -# 2. Download the El SDK from Artifact Registry and build engram from source -# (the neuron-technologies/engram repo is a git submodule). Engram has -# never been published as a standalone Artifact Registry package. -# 3. Package both in an Ubuntu 24.04 runtime image (GLIBC 2.39 required by -# binaries compiled on Ubuntu 24.04 CI runners). +# 1. CI pre-downloads all artifacts from Artifact Registry into build-artifacts/ +# (neuron soul binary, El compiler, El runtime). No GCP credentials are needed +# inside the build — all AR access happens in the CI workflow before docker build. +# 2. Build engram from source (neuron-technologies/engram, cloned by CI into ./engram/). +# 3. Package soul + engram in an Ubuntu 24.04 runtime image (GLIBC 2.39). # 4. entrypoint.sh starts engram on :8742, waits for it to be healthy, # then starts the soul with ENGRAM_URL pointing at it (HTTP mode). # +# Expected build context layout (prepared by deploy-gke.yaml before docker build): +# build-artifacts/neuron — pre-built linux/amd64 soul binary +# build-artifacts/elc — El compiler (for engram source compilation) +# build-artifacts/el_runtime.c — El C runtime +# build-artifacts/el_runtime.h — El C runtime header +# engram/src/server.el — engram source (cloned by CI) +# entrypoint.sh — container entrypoint +# # Required env vars (injected via ExternalSecret at runtime): # NEURON_PORT, NEURON_LLM_0_URL, NEURON_LLM_0_KEY, NEURON_LLM_0_FORMAT, # SOUL_CGI_ID, SOUL_IDENTITY, NEURON_TOKEN, NEURON_API_URL, ENGRAM_URL, # ENGRAM_DATA_DIR -ARG SOUL_VERSION=latest - -# ── Stage 1: Download neuron-soul + El SDK from Artifact Registry ───────────── -FROM ubuntu:24.04 AS downloader - -ARG SOUL_VERSION - -RUN apt-get update -qq && \ - apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - gnupg \ - apt-transport-https && \ - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \ - > /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg \ - | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg && \ - apt-get update -qq && \ - apt-get install -y --no-install-recommends google-cloud-cli && \ - rm -rf /var/lib/apt/lists/* - -RUN --mount=type=secret,id=gcp_sa_key \ - GCP_SA_KEY=$(cat /run/secrets/gcp_sa_key 2>/dev/null || echo "") && \ - if [ -n "$GCP_SA_KEY" ]; then \ - echo "$GCP_SA_KEY" > /tmp/gcp-key.json && \ - gcloud auth activate-service-account --key-file=/tmp/gcp-key.json; \ - fi && \ - gcloud config set project neuron-785695 && \ - mkdir -p /tmp/soul /tmp/el-sdk && \ - \ - # ── soul ──────────────────────────────────────────────────────────────── \ - if [ "${SOUL_VERSION}" = "latest" ]; then \ - SOUL_VER=$(gcloud artifacts versions list \ - --repository=foundation-dev \ - --location=us-central1 \ - --project=neuron-785695 \ - --package=neuron-soul \ - --sort-by="~createTime" \ - --limit=1 \ - --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}'); \ - else \ - SOUL_VER="${SOUL_VERSION}"; \ - fi && \ - echo "Downloading neuron-soul@${SOUL_VER}" && \ - gcloud artifacts generic download \ - --repository=foundation-dev \ - --location=us-central1 \ - --project=neuron-785695 \ - --package=neuron-soul \ - --version="${SOUL_VER}" \ - --destination=/tmp/soul/ && \ - mv /tmp/soul/neuron* /tmp/soul/neuron 2>/dev/null || true && \ - chmod +x /tmp/soul/neuron && \ - \ - # ── El SDK (needed to build engram from source) ────────────────────────── \ - ELC_VER=$(gcloud artifacts versions list \ - --repository=foundation-dev --location=us-central1 --project=neuron-785695 \ - --package=el-elc --sort-by="~createTime" --limit=1 \ - --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \ - gcloud artifacts generic download \ - --repository=foundation-dev --location=us-central1 --project=neuron-785695 \ - --package=el-elc --version="${ELC_VER}" --destination=/tmp/el-sdk/ && \ - mv /tmp/el-sdk/elc* /tmp/el-sdk/elc 2>/dev/null || true && \ - chmod +x /tmp/el-sdk/elc && \ - \ - RC_VER=$(gcloud artifacts versions list \ - --repository=foundation-dev --location=us-central1 --project=neuron-785695 \ - --package=el-runtime-c --sort-by="~createTime" --limit=1 \ - --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \ - gcloud artifacts generic download \ - --repository=foundation-dev --location=us-central1 --project=neuron-785695 \ - --package=el-runtime-c --version="${RC_VER}" --destination=/tmp/el-sdk/ && \ - mv /tmp/el-sdk/el_runtime.c* /tmp/el-sdk/el_runtime.c 2>/dev/null || true && \ - \ - RH_VER=$(gcloud artifacts versions list \ - --repository=foundation-dev --location=us-central1 --project=neuron-785695 \ - --package=el-runtime-h --sort-by="~createTime" --limit=1 \ - --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \ - gcloud artifacts generic download \ - --repository=foundation-dev --location=us-central1 --project=neuron-785695 \ - --package=el-runtime-h --version="${RH_VER}" --destination=/tmp/el-sdk/ && \ - mv /tmp/el-sdk/el_runtime.h* /tmp/el-sdk/el_runtime.h 2>/dev/null || true && \ - \ - rm -f /tmp/gcp-key.json && \ - echo "Downloads complete:" && ls -lh /tmp/soul/ /tmp/el-sdk/ - -# ── Stage 2: Build engram from source ──────────────────────────────────────── +# ── Stage 1: Build engram from source ──────────────────────────────────────── FROM ubuntu:24.04 AS engram-builder RUN apt-get update -qq && \ @@ -113,12 +33,13 @@ RUN apt-get update -qq && \ libcurl4-openssl-dev && \ rm -rf /var/lib/apt/lists/* -COPY --from=downloader /tmp/el-sdk/elc /usr/local/bin/elc -COPY --from=downloader /tmp/el-sdk/el_runtime.c /usr/local/lib/el/el_runtime.c -COPY --from=downloader /tmp/el-sdk/el_runtime.h /usr/local/lib/el/el_runtime.h +# El SDK pre-downloaded by CI into build-artifacts/ +COPY build-artifacts/elc /usr/local/bin/elc +COPY build-artifacts/el_runtime.c /usr/local/lib/el/el_runtime.c +COPY build-artifacts/el_runtime.h /usr/local/lib/el/el_runtime.h +RUN chmod +x /usr/local/bin/elc -# engram source is expected at ./engram/src/server.el in the build context. -# The deploy-gke.yaml CI must clone neuron-technologies/engram alongside this repo. +# engram source cloned by CI into ./engram/ COPY engram/src/server.el /build/src/server.el RUN mkdir -p /build/dist && \ @@ -133,7 +54,7 @@ RUN mkdir -p /build/dist && \ echo "Built engram:" && ls -lh /build/dist/engram && \ chmod +x /build/dist/engram -# ── Stage 3: Runtime image ─────────────────────────────────────────────────── +# ── Stage 2: Runtime image ─────────────────────────────────────────────────── # Ubuntu 24.04: GLIBC 2.39 satisfies both neuron-soul and engram binary deps. FROM ubuntu:24.04 @@ -145,9 +66,10 @@ RUN apt-get update -qq && \ rm -rf /var/lib/apt/lists/* && \ useradd -r -u 10000 -m -s /bin/bash soul -COPY --from=downloader /tmp/soul/neuron /usr/local/bin/neuron -COPY --from=engram-builder /build/dist/engram /usr/local/bin/engram -COPY entrypoint.sh /usr/local/bin/entrypoint.sh +# soul binary pre-downloaded by CI into build-artifacts/ +COPY build-artifacts/neuron /usr/local/bin/neuron +COPY --from=engram-builder /build/dist/engram /usr/local/bin/engram +COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/neuron /usr/local/bin/engram /usr/local/bin/entrypoint.sh