feat(gke): add Dockerfile, deploy scripts, and GKE CI workflow
Deploy Soul to GKE / deploy (push) Failing after 33s
Neuron Soul CI / build (push) Successful in 1m56s

Dockerfile: downloads linux/amd64 soul binary from Artifact Registry
(foundation-dev/neuron-soul) into ubuntu:22.04 runtime image.
Pushes to neuron-api Docker repo as neuron-soul:<sha>.

scripts/blue-green-deploy.sh: swaps active slot on GKE — sets image,
scales new slot to 1, flips service selector, scales old slot to 0.

scripts/seed-engram-gke.sh: downloads latest GCS backup, extracts
snapshot.json, copies into neuron-engram-data PVC via a seed Job.

.gitea/workflows/deploy-gke.yaml: triggers on push to main, auto-detects
idle slot, builds Docker image from Artifact Registry binary, blue-green
deploys to neuron-prod on GKE neuron-platform cluster.
This commit is contained in:
2026-05-09 20:30:05 -05:00
parent ea271d5cc6
commit e1a7c71a87
4 changed files with 540 additions and 0 deletions
+132
View File
@@ -0,0 +1,132 @@
name: Deploy Soul to GKE
# Triggers on push to main — after the soul binary is built and published
# by ci.yaml, this workflow builds the Docker image and blue-green deploys
# to the neuron-prod namespace on GKE.
#
# This workflow runs AFTER ci.yaml has published the neuron-soul generic
# artifact to Artifact Registry. The Docker build downloads that binary.
on:
push:
branches:
- main
workflow_dispatch:
inputs:
slot:
description: "Target blue-green slot (blue or green)"
required: false
default: "green"
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install dependencies
run: |
apt-get update -qq
apt-get install -y --no-install-recommends \
ca-certificates curl gnupg apt-transport-https kubectl
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \
> /etc/apt/sources.list.d/google-cloud-sdk.list
curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg \
| gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
apt-get update -qq && apt-get install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin
- name: Authenticate to GCP
env:
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
run: |
echo "${GCP_SA_KEY}" > /tmp/gcp-key.json
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json
gcloud config set project neuron-785695
gcloud auth configure-docker us-central1-docker.pkg.dev --quiet
- name: Get GKE credentials
run: |
gcloud container clusters get-credentials neuron-platform \
--region=us-central1 \
--project=neuron-785695
- name: Determine image tag and slot
id: vars
run: |
SHA="${GITEA_SHA:0:8}"
IMAGE="us-central1-docker.pkg.dev/neuron-785695/neuron-api/neuron-soul:${SHA}"
echo "sha=${SHA}" >> "$GITEA_OUTPUT"
echo "image=${IMAGE}" >> "$GITEA_OUTPUT"
# Determine which slot is currently idle (0 replicas = idle slot)
# If both are at 0 (fresh deploy), default to blue
BLUE_REPLICAS=$(kubectl get deployment/neuron-mcp-blue \
-n neuron-prod \
-o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0")
GREEN_REPLICAS=$(kubectl get deployment/neuron-mcp-green \
-n neuron-prod \
-o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0")
echo " Blue replicas: ${BLUE_REPLICAS}"
echo " Green replicas: ${GREEN_REPLICAS}"
# Use workflow_dispatch override if provided, otherwise pick the idle slot
if [ "${{ github.event.inputs.slot }}" != "" ]; then
SLOT="${{ github.event.inputs.slot }}"
elif [ "${GREEN_REPLICAS}" -eq 0 ] && [ "${BLUE_REPLICAS}" -gt 0 ]; then
SLOT="green"
elif [ "${BLUE_REPLICAS}" -eq 0 ] && [ "${GREEN_REPLICAS}" -gt 0 ]; then
SLOT="blue"
else
# Fresh cluster — deploy to blue first
SLOT="blue"
fi
echo "slot=${SLOT}" >> "$GITEA_OUTPUT"
echo " Deploying to slot: ${SLOT}"
- name: Build and push Docker image
env:
GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
run: |
IMAGE="${{ steps.vars.outputs.image }}"
SHA="${{ steps.vars.outputs.sha }}"
echo "Building ${IMAGE}..."
docker build \
--build-arg SOUL_VERSION="${SHA}" \
--secret id=gcp_sa_key,env=GCP_SA_KEY \
--tag "${IMAGE}" \
--tag "us-central1-docker.pkg.dev/neuron-785695/neuron-api/neuron-soul:latest" \
.
echo "Pushing ${IMAGE}..."
docker push "${IMAGE}"
docker push "us-central1-docker.pkg.dev/neuron-785695/neuron-api/neuron-soul:latest"
- name: Blue-green deploy to GKE
run: |
chmod +x scripts/blue-green-deploy.sh
scripts/blue-green-deploy.sh \
--image "${{ steps.vars.outputs.image }}" \
--slot "${{ steps.vars.outputs.slot }}"
- name: Verify deployment
run: |
SLOT="${{ steps.vars.outputs.slot }}"
echo "Verifying neuron-mcp-${SLOT} is healthy..."
kubectl rollout status deployment/"neuron-mcp-${SLOT}" \
--namespace=neuron-prod \
--timeout=3m
echo "Active service endpoints:"
kubectl get endpoints neuron-mcp -n neuron-prod
echo "Pod status:"
kubectl get pods -n neuron-prod -l app=neuron-mcp
- name: Cleanup
if: always()
run: rm -f /tmp/gcp-key.json
+101
View File
@@ -0,0 +1,101 @@
# Neuron Soul — GKE container image
#
# Build strategy:
# 1. Download the pre-built linux/amd64 soul binary from Artifact Registry
# (package: neuron-soul, repository: foundation-dev).
# The binary is built by CI from soul.el and published as a generic artifact.
# 2. Package it in a minimal Ubuntu 22.04 runtime with glibc and libcurl.
#
# The soul runs in file mode (no HTTP Engram sidecar):
# - SOUL_ENGRAM_PATH=/data/snapshot.json → reads/writes engram from mounted PVC
# - ENGRAM_URL must NOT be set → absence triggers file mode
#
# Required env vars (injected via ExternalSecret at runtime):
# NEURON_PORT, NEURON_LLM_0_URL, NEURON_LLM_0_KEY, NEURON_LLM_0_FORMAT,
# SOUL_CGI_ID, SOUL_IDENTITY, NEURON_TOKEN, NEURON_API_URL, SOUL_ENGRAM_PATH
ARG SOUL_VERSION=latest
FROM ubuntu:22.04 AS downloader
ARG SOUL_VERSION
ARG GCP_SA_KEY
RUN apt-get update -qq && \
apt-get install -y --no-install-recommends \
ca-certificates \
curl \
gnupg \
apt-transport-https && \
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \
> /etc/apt/sources.list.d/google-cloud-sdk.list && \
curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg \
| gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg && \
apt-get update -qq && \
apt-get install -y --no-install-recommends google-cloud-cli && \
rm -rf /var/lib/apt/lists/*
# Authenticate and download the soul binary from Artifact Registry.
# SOUL_VERSION is the 8-char git SHA tag published by CI (e.g. ea271d5c).
# The binary is stored as a generic artifact — download to /tmp/soul/neuron.
RUN --mount=type=secret,id=gcp_sa_key \
GCP_SA_KEY_FILE=$(cat /run/secrets/gcp_sa_key 2>/dev/null || echo "") && \
if [ -n "$GCP_SA_KEY_FILE" ]; then \
echo "$GCP_SA_KEY_FILE" > /tmp/gcp-key.json && \
gcloud auth activate-service-account --key-file=/tmp/gcp-key.json; \
fi && \
gcloud config set project neuron-785695 && \
mkdir -p /tmp/soul && \
if [ "${SOUL_VERSION}" = "latest" ]; then \
VERSION=$(gcloud artifacts versions list \
--repository=foundation-dev \
--location=us-central1 \
--project=neuron-785695 \
--package=neuron-soul \
--sort-by="~createTime" \
--limit=1 \
--format="value(name)" 2>/dev/null | awk -F/ '{print $NF}'); \
else \
VERSION="${SOUL_VERSION}"; \
fi && \
echo "Downloading neuron-soul@${VERSION}" && \
gcloud artifacts generic download \
--repository=foundation-dev \
--location=us-central1 \
--project=neuron-785695 \
--package=neuron-soul \
--version="${VERSION}" \
--destination=/tmp/soul/ && \
mv /tmp/soul/neuron* /tmp/soul/neuron 2>/dev/null || true && \
chmod +x /tmp/soul/neuron && \
rm -f /tmp/gcp-key.json
# Runtime image — minimal Ubuntu 22.04 with only what the soul binary needs.
FROM ubuntu:22.04
RUN apt-get update -qq && \
apt-get install -y --no-install-recommends \
ca-certificates \
libcurl4 \
curl && \
rm -rf /var/lib/apt/lists/* && \
useradd -r -u 1000 -m -s /bin/bash soul
COPY --from=downloader /tmp/soul/neuron /usr/local/bin/neuron
RUN chmod +x /usr/local/bin/neuron
# /data is the engram mount point (PVC at runtime).
# Create it owned by soul user so the binary can write snapshot.json.
RUN mkdir -p /data && chown soul:soul /data
USER soul
WORKDIR /home/soul
EXPOSE 7770
# SOUL_ENGRAM_PATH and other env vars are injected via k8s ExternalSecret.
# ENGRAM_URL must NOT be set — its absence triggers file mode.
ENV NEURON_PORT=7770 \
SOUL_ENGRAM_PATH=/data/snapshot.json
ENTRYPOINT ["/usr/local/bin/neuron"]
+121
View File
@@ -0,0 +1,121 @@
#!/usr/bin/env bash
# blue-green-deploy.sh — swap the active soul slot on GKE
#
# Usage:
# blue-green-deploy.sh --image <image-tag> --slot <blue|green>
#
# What it does:
# 1. Sets the new image on the target slot deployment
# 2. Scales the target slot to 1 replica and waits for it to become ready
# 3. Patches the neuron-mcp Service selector to point at the new slot
# 4. Scales the old slot down to 0 replicas
#
# This script uses kubectl imperatively for the live traffic swap.
# After a successful swap, Argo CD manifests should be updated to reflect
# the new active slot (so Argo CD doesn't revert the replica counts on next sync).
#
# Prerequisites:
# - kubectl configured with access to neuron-platform cluster
# - Sufficient RBAC: get/patch/scale Deployments and Services in neuron-prod
#
# Examples:
# blue-green-deploy.sh --image us-central1-docker.pkg.dev/neuron-785695/neuron-api/neuron-soul:abc12345 --slot green
# blue-green-deploy.sh --image us-central1-docker.pkg.dev/neuron-785695/neuron-api/neuron-soul:latest --slot blue
set -euo pipefail
NAMESPACE="neuron-prod"
APP="neuron-mcp"
SERVICE="neuron-mcp"
IMAGE=""
NEW_SLOT=""
usage() {
echo "Usage: $0 --image <image-tag> --slot <blue|green>"
echo ""
echo " --image Full image URI including tag"
echo " --slot Target slot to deploy to: blue or green"
exit 1
}
while [[ $# -gt 0 ]]; do
case $1 in
--image)
IMAGE="$2"; shift 2 ;;
--slot)
NEW_SLOT="$2"; shift 2 ;;
-h|--help)
usage ;;
*)
echo "Unknown argument: $1"
usage ;;
esac
done
[[ -z "$IMAGE" ]] && { echo "ERROR: --image is required"; usage; }
[[ -z "$NEW_SLOT" ]] && { echo "ERROR: --slot is required"; usage; }
[[ "$NEW_SLOT" != "blue" && "$NEW_SLOT" != "green" ]] && {
echo "ERROR: --slot must be 'blue' or 'green'"; exit 1
}
# Determine the old (currently active) slot
if [[ "$NEW_SLOT" == "blue" ]]; then
OLD_SLOT="green"
else
OLD_SLOT="blue"
fi
NEW_DEPLOYMENT="${APP}-${NEW_SLOT}"
OLD_DEPLOYMENT="${APP}-${OLD_SLOT}"
echo "==> Deploying to slot: ${NEW_SLOT}"
echo " Image: ${IMAGE}"
echo " New deploy: ${NEW_DEPLOYMENT}"
echo " Old deploy: ${OLD_DEPLOYMENT}"
echo " Namespace: ${NAMESPACE}"
echo ""
# Step 1: Update the image on the new slot
echo "[1/4] Setting image on ${NEW_DEPLOYMENT}..."
kubectl set image deployment/"${NEW_DEPLOYMENT}" \
soul="${IMAGE}" \
--namespace="${NAMESPACE}"
# Step 2: Scale the new slot up (it may already be at 0)
echo "[2/4] Scaling ${NEW_DEPLOYMENT} to 1 replica..."
kubectl scale deployment/"${NEW_DEPLOYMENT}" \
--replicas=1 \
--namespace="${NAMESPACE}"
# Wait for rollout to complete (pod ready)
echo " Waiting for rollout to complete (timeout: 5m)..."
kubectl rollout status deployment/"${NEW_DEPLOYMENT}" \
--namespace="${NAMESPACE}" \
--timeout=5m
echo " ${NEW_DEPLOYMENT} is ready."
# Step 3: Patch the service selector to point at the new slot
echo "[3/4] Flipping service selector to slot=${NEW_SLOT}..."
kubectl patch service "${SERVICE}" \
--namespace="${NAMESPACE}" \
--type=merge \
--patch="{\"spec\":{\"selector\":{\"app\":\"${APP}\",\"slot\":\"${NEW_SLOT}\"}}}"
echo " Traffic now routing to ${NEW_SLOT}."
# Step 4: Scale down the old slot
echo "[4/4] Scaling ${OLD_DEPLOYMENT} down to 0 replicas..."
kubectl scale deployment/"${OLD_DEPLOYMENT}" \
--replicas=0 \
--namespace="${NAMESPACE}"
echo ""
echo "==> Blue-green swap complete."
echo " Active slot: ${NEW_SLOT} (${IMAGE})"
echo " Idle slot: ${OLD_SLOT} (scaled to 0)"
echo ""
echo "NOTE: Update Argo CD manifests to reflect new state:"
echo " - deployment-${NEW_SLOT}.yaml: replicas: 1, image: ${IMAGE}"
echo " - deployment-${OLD_SLOT}.yaml: replicas: 0"
+186
View File
@@ -0,0 +1,186 @@
#!/usr/bin/env bash
# seed-engram-gke.sh — seed the GKE soul's PVC with the latest engram backup
#
# Downloads the latest engram backup from GCS, extracts snapshot.json,
# and copies it into the neuron-prod PVC via a temporary seed Job.
#
# Usage:
# seed-engram-gke.sh [--hostname <hostname>] [--backup <gs://path>] [--dry-run]
#
# Options:
# --hostname Mac hostname used as the backup source prefix
# Default: auto-detected from GCS bucket listing
# --backup Full GCS path to a specific backup tarball
# Overrides --hostname; use if you want a specific backup
# --dry-run Print what would happen without executing
#
# Examples:
# seed-engram-gke.sh
# seed-engram-gke.sh --hostname Wills-MacBook-Pro.local
# seed-engram-gke.sh --backup gs://neuron-db-backup/local-mac/Wills-MacBook-Pro.local/engram-20260509T1200.tar.gz
set -euo pipefail
NAMESPACE="neuron-prod"
PVC_NAME="neuron-engram-data"
GCS_BUCKET="gs://neuron-db-backup/local-mac"
HOSTNAME_PREFIX=""
SPECIFIC_BACKUP=""
DRY_RUN=false
SEED_JOB_NAME="engram-seed-$(date +%s)"
usage() {
grep '^#' "$0" | grep -v '#!/' | sed 's/^# //' | sed 's/^#//'
exit 0
}
while [[ $# -gt 0 ]]; do
case $1 in
--hostname) HOSTNAME_PREFIX="$2"; shift 2 ;;
--backup) SPECIFIC_BACKUP="$2"; shift 2 ;;
--dry-run) DRY_RUN=true; shift ;;
-h|--help) usage ;;
*) echo "Unknown argument: $1"; exit 1 ;;
esac
done
run() {
if $DRY_RUN; then
echo "[dry-run] $*"
else
"$@"
fi
}
echo "==> Neuron Engram GKE Seed"
echo " Namespace: ${NAMESPACE}"
echo " PVC: ${PVC_NAME}"
echo ""
# Step 1: Locate the backup
if [[ -n "$SPECIFIC_BACKUP" ]]; then
BACKUP_PATH="$SPECIFIC_BACKUP"
echo "[1/5] Using specified backup: ${BACKUP_PATH}"
else
echo "[1/5] Finding latest backup in GCS..."
if [[ -z "$HOSTNAME_PREFIX" ]]; then
# Auto-detect: list prefixes and pick the first (most likely the only one)
HOSTNAME_PREFIX=$(gsutil ls "${GCS_BUCKET}/" 2>/dev/null \
| sed "s|${GCS_BUCKET}/||" \
| sed 's|/||' \
| head -1)
if [[ -z "$HOSTNAME_PREFIX" ]]; then
echo "ERROR: Could not find any backup prefixes in ${GCS_BUCKET}/"
echo " Run: gsutil ls ${GCS_BUCKET}/"
exit 1
fi
echo " Auto-detected hostname prefix: ${HOSTNAME_PREFIX}"
fi
BACKUP_PATH=$(gsutil ls "${GCS_BUCKET}/${HOSTNAME_PREFIX}/" 2>/dev/null \
| grep "engram-.*\.tar\.gz" \
| sort -r \
| head -1)
if [[ -z "$BACKUP_PATH" ]]; then
echo "ERROR: No backup found at ${GCS_BUCKET}/${HOSTNAME_PREFIX}/"
exit 1
fi
echo " Latest backup: ${BACKUP_PATH}"
fi
# Step 2: Download and extract snapshot.json
TMP_DIR=$(mktemp -d)
trap 'rm -rf "${TMP_DIR}"' EXIT
echo "[2/5] Downloading backup..."
run gsutil cp "${BACKUP_PATH}" "${TMP_DIR}/engram.tar.gz"
echo "[3/5] Extracting snapshot.json..."
if $DRY_RUN; then
echo "[dry-run] tar -xzf ${TMP_DIR}/engram.tar.gz -C ${TMP_DIR} --wildcards '*snapshot.json'"
SNAPSHOT_PATH="${TMP_DIR}/snapshot.json"
else
tar -xzf "${TMP_DIR}/engram.tar.gz" -C "${TMP_DIR}" 2>/dev/null || true
# Handle nested paths (backup may be engram/snapshot.json or just snapshot.json)
SNAPSHOT_PATH=$(find "${TMP_DIR}" -name "snapshot.json" | head -1)
if [[ -z "$SNAPSHOT_PATH" ]]; then
echo "ERROR: snapshot.json not found in backup archive"
echo " Archive contents:"
tar -tzf "${TMP_DIR}/engram.tar.gz" | head -20
exit 1
fi
SNAPSHOT_SIZE=$(du -h "$SNAPSHOT_PATH" | cut -f1)
echo " snapshot.json: ${SNAPSHOT_SIZE} at ${SNAPSHOT_PATH}"
fi
# Step 4: Copy into PVC via a seed Job
# The Job mounts the PVC, then we kubectl cp the snapshot.json into the pod.
echo "[4/5] Launching seed Job to access PVC..."
SEED_JOB_MANIFEST=$(cat <<EOF
apiVersion: batch/v1
kind: Job
metadata:
name: ${SEED_JOB_NAME}
namespace: ${NAMESPACE}
spec:
ttlSecondsAfterFinished: 300
template:
spec:
serviceAccountName: neuron-soul
restartPolicy: Never
containers:
- name: seed
image: ubuntu:22.04
command: ["/bin/bash", "-c", "echo waiting; sleep 600"]
volumeMounts:
- name: engram-data
mountPath: /data
volumes:
- name: engram-data
persistentVolumeClaim:
claimName: ${PVC_NAME}
EOF
)
if $DRY_RUN; then
echo "[dry-run] kubectl apply -f <job manifest>"
echo "[dry-run] kubectl wait --for=condition=ready pod -l job-name=${SEED_JOB_NAME} -n ${NAMESPACE}"
echo "[dry-run] kubectl cp ${SNAPSHOT_PATH} ${NAMESPACE}/<pod>:/data/snapshot.json"
echo "[dry-run] kubectl delete job/${SEED_JOB_NAME} -n ${NAMESPACE}"
else
echo "$SEED_JOB_MANIFEST" | kubectl apply -f -
echo " Waiting for seed pod to be ready (timeout: 2m)..."
kubectl wait --for=condition=ready pod \
-l "job-name=${SEED_JOB_NAME}" \
--namespace="${NAMESPACE}" \
--timeout=2m
SEED_POD=$(kubectl get pods \
-l "job-name=${SEED_JOB_NAME}" \
--namespace="${NAMESPACE}" \
--output=jsonpath='{.items[0].metadata.name}')
echo " Seed pod: ${SEED_POD}"
echo "[5/5] Copying snapshot.json into PVC..."
kubectl cp "${SNAPSHOT_PATH}" "${NAMESPACE}/${SEED_POD}:/data/snapshot.json"
# Verify
PVC_SIZE=$(kubectl exec "${SEED_POD}" --namespace="${NAMESPACE}" -- \
stat -c%s /data/snapshot.json 2>/dev/null || echo "unknown")
echo " PVC snapshot.json size: ${PVC_SIZE} bytes"
echo " Cleaning up seed Job..."
kubectl delete job/"${SEED_JOB_NAME}" --namespace="${NAMESPACE}" --ignore-not-found
fi
echo ""
echo "==> Engram seed complete."
echo " The soul pod will read /data/snapshot.json on next boot."
echo ""
echo " Scale the blue deployment to pick up the seeded engram:"
echo " kubectl scale deployment/neuron-mcp-blue --replicas=1 -n ${NAMESPACE}"