From cc07648ae1f5eb83d8528807150ce54a45451ee4 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Mon, 11 May 2026 12:39:36 -0500 Subject: [PATCH] fix: build engram from source in Docker image The engram binary was never published to Artifact Registry (foundation-dev/engram package does not exist). Updated Dockerfile to build engram from source using the El SDK packages (el-elc, el-runtime-c, el-runtime-h) from foundation-dev. Also: - Switch runtime base to Ubuntu 24.04 (GLIBC 2.39 required by elc-compiled binaries) - Add -lm to engram link flags (el_runtime.c uses pow/sqrt/log/sin/cos/exp) - Update deploy-gke.yaml to clone neuron-technologies/engram into build context --- .gitea/workflows/deploy-gke.yaml | 9 +++ Dockerfile | 124 ++++++++++++++++++++----------- 2 files changed, 90 insertions(+), 43 deletions(-) diff --git a/.gitea/workflows/deploy-gke.yaml b/.gitea/workflows/deploy-gke.yaml index 0caa469..2081602 100644 --- a/.gitea/workflows/deploy-gke.yaml +++ b/.gitea/workflows/deploy-gke.yaml @@ -87,6 +87,15 @@ jobs: echo "slot=${SLOT}" >> "$GITEA_OUTPUT" echo " Deploying to slot: ${SLOT}" + - name: Clone engram source for Docker build context + run: | + # The Dockerfile builds engram from source (no published AR package). + # Clone the engram repo into ./engram/ so it's available in the build context. + git clone http://34.31.145.131/neuron-technologies/engram.git \ + --depth=1 --branch=main \ + engram + echo "Engram source ready at ./engram/src/server.el" + - name: Build and push Docker image env: GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} diff --git a/Dockerfile b/Dockerfile index 6a1b24a..29f78db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,13 @@ # # Build strategy: # 1. Download the pre-built linux/amd64 soul binary (package: neuron-soul) -# and engram HTTP server binary (package: engram) from Artifact Registry. -# Both are built by CI and published as generic artifacts. -# 2. Package both in a minimal Ubuntu 22.04 runtime. -# 3. entrypoint.sh starts engram on :8742, waits for it to be healthy, +# from Artifact Registry (foundation-dev). +# 2. Download the El SDK from Artifact Registry and build engram from source +# (the neuron-technologies/engram repo is a git submodule). Engram has +# never been published as a standalone Artifact Registry package. +# 3. Package both in an Ubuntu 24.04 runtime image (GLIBC 2.39 required by +# binaries compiled on Ubuntu 24.04 CI runners). +# 4. entrypoint.sh starts engram on :8742, waits for it to be healthy, # then starts the soul with ENGRAM_URL pointing at it (HTTP mode). # # Required env vars (injected via ExternalSecret at runtime): @@ -14,13 +17,11 @@ # ENGRAM_DATA_DIR ARG SOUL_VERSION=latest -ARG ENGRAM_VERSION=latest -FROM ubuntu:22.04 AS downloader +# ── Stage 1: Download neuron-soul + El SDK from Artifact Registry ───────────── +FROM ubuntu:24.04 AS downloader ARG SOUL_VERSION -ARG ENGRAM_VERSION -ARG GCP_SA_KEY RUN apt-get update -qq && \ apt-get install -y --no-install-recommends \ @@ -36,15 +37,14 @@ RUN apt-get update -qq && \ apt-get install -y --no-install-recommends google-cloud-cli && \ rm -rf /var/lib/apt/lists/* -# Authenticate and download both binaries from Artifact Registry. RUN --mount=type=secret,id=gcp_sa_key \ - GCP_SA_KEY_FILE=$(cat /run/secrets/gcp_sa_key 2>/dev/null || echo "") && \ - if [ -n "$GCP_SA_KEY_FILE" ]; then \ - echo "$GCP_SA_KEY_FILE" > /tmp/gcp-key.json && \ + GCP_SA_KEY=$(cat /run/secrets/gcp_sa_key 2>/dev/null || echo "") && \ + if [ -n "$GCP_SA_KEY" ]; then \ + echo "$GCP_SA_KEY" > /tmp/gcp-key.json && \ gcloud auth activate-service-account --key-file=/tmp/gcp-key.json; \ fi && \ gcloud config set project neuron-785695 && \ - mkdir -p /tmp/soul /tmp/engram && \ + mkdir -p /tmp/soul /tmp/el-sdk && \ \ # ── soul ──────────────────────────────────────────────────────────────── \ if [ "${SOUL_VERSION}" = "latest" ]; then \ @@ -70,46 +70,84 @@ RUN --mount=type=secret,id=gcp_sa_key \ mv /tmp/soul/neuron* /tmp/soul/neuron 2>/dev/null || true && \ chmod +x /tmp/soul/neuron && \ \ - # ── engram ────────────────────────────────────────────────────────────── \ - if [ "${ENGRAM_VERSION}" = "latest" ]; then \ - ENGRAM_VER=$(gcloud artifacts versions list \ - --repository=foundation-dev \ - --location=us-central1 \ - --project=neuron-785695 \ - --package=engram \ - --sort-by="~createTime" \ - --limit=1 \ - --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}'); \ - else \ - ENGRAM_VER="${ENGRAM_VERSION}"; \ - fi && \ - echo "Downloading engram@${ENGRAM_VER}" && \ + # ── El SDK (needed to build engram from source) ────────────────────────── \ + ELC_VER=$(gcloud artifacts versions list \ + --repository=foundation-dev --location=us-central1 --project=neuron-785695 \ + --package=el-elc --sort-by="~createTime" --limit=1 \ + --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \ gcloud artifacts generic download \ - --repository=foundation-dev \ - --location=us-central1 \ - --project=neuron-785695 \ - --package=engram \ - --version="${ENGRAM_VER}" \ - --destination=/tmp/engram/ && \ - mv /tmp/engram/engram* /tmp/engram/engram 2>/dev/null || true && \ - chmod +x /tmp/engram/engram && \ + --repository=foundation-dev --location=us-central1 --project=neuron-785695 \ + --package=el-elc --version="${ELC_VER}" --destination=/tmp/el-sdk/ && \ + mv /tmp/el-sdk/elc* /tmp/el-sdk/elc 2>/dev/null || true && \ + chmod +x /tmp/el-sdk/elc && \ \ - rm -f /tmp/gcp-key.json + RC_VER=$(gcloud artifacts versions list \ + --repository=foundation-dev --location=us-central1 --project=neuron-785695 \ + --package=el-runtime-c --sort-by="~createTime" --limit=1 \ + --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \ + gcloud artifacts generic download \ + --repository=foundation-dev --location=us-central1 --project=neuron-785695 \ + --package=el-runtime-c --version="${RC_VER}" --destination=/tmp/el-sdk/ && \ + mv /tmp/el-sdk/el_runtime.c* /tmp/el-sdk/el_runtime.c 2>/dev/null || true && \ + \ + RH_VER=$(gcloud artifacts versions list \ + --repository=foundation-dev --location=us-central1 --project=neuron-785695 \ + --package=el-runtime-h --sort-by="~createTime" --limit=1 \ + --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}') && \ + gcloud artifacts generic download \ + --repository=foundation-dev --location=us-central1 --project=neuron-785695 \ + --package=el-runtime-h --version="${RH_VER}" --destination=/tmp/el-sdk/ && \ + mv /tmp/el-sdk/el_runtime.h* /tmp/el-sdk/el_runtime.h 2>/dev/null || true && \ + \ + rm -f /tmp/gcp-key.json && \ + echo "Downloads complete:" && ls -lh /tmp/soul/ /tmp/el-sdk/ -# Runtime image — minimal Ubuntu 22.04 with only what both binaries need. -FROM ubuntu:22.04 +# ── Stage 2: Build engram from source ──────────────────────────────────────── +FROM ubuntu:24.04 AS engram-builder RUN apt-get update -qq && \ apt-get install -y --no-install-recommends \ ca-certificates \ - libcurl4 \ + gcc \ + libc6-dev \ + libcurl4-openssl-dev && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=downloader /tmp/el-sdk/elc /usr/local/bin/elc +COPY --from=downloader /tmp/el-sdk/el_runtime.c /usr/local/lib/el/el_runtime.c +COPY --from=downloader /tmp/el-sdk/el_runtime.h /usr/local/lib/el/el_runtime.h + +# engram source is expected at ./engram/src/server.el in the build context. +# The deploy-gke.yaml CI must clone neuron-technologies/engram alongside this repo. +COPY engram/src/server.el /build/src/server.el + +RUN mkdir -p /build/dist && \ + /usr/local/bin/elc /build/src/server.el > /build/dist/engram.c && \ + echo "Compiled server.el -> engram.c ($(wc -l < /build/dist/engram.c) lines)" && \ + cc -std=c11 -O2 \ + -I /usr/local/lib/el \ + -o /build/dist/engram \ + /build/dist/engram.c \ + /usr/local/lib/el/el_runtime.c \ + -lcurl -lpthread -lm && \ + echo "Built engram:" && ls -lh /build/dist/engram && \ + chmod +x /build/dist/engram + +# ── Stage 3: Runtime image ─────────────────────────────────────────────────── +# Ubuntu 24.04: GLIBC 2.39 satisfies both neuron-soul and engram binary deps. +FROM ubuntu:24.04 + +RUN apt-get update -qq && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + libcurl4t64 \ curl && \ rm -rf /var/lib/apt/lists/* && \ - useradd -r -u 1000 -m -s /bin/bash soul + useradd -r -u 10000 -m -s /bin/bash soul -COPY --from=downloader /tmp/soul/neuron /usr/local/bin/neuron -COPY --from=downloader /tmp/engram/engram /usr/local/bin/engram -COPY entrypoint.sh /usr/local/bin/entrypoint.sh +COPY --from=downloader /tmp/soul/neuron /usr/local/bin/neuron +COPY --from=engram-builder /build/dist/engram /usr/local/bin/engram +COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/neuron /usr/local/bin/engram /usr/local/bin/entrypoint.sh