diff --git a/.gitea/workflows/deploy-gke.yaml b/.gitea/workflows/deploy-gke.yaml new file mode 100644 index 0000000..0caa469 --- /dev/null +++ b/.gitea/workflows/deploy-gke.yaml @@ -0,0 +1,132 @@ +name: Deploy Soul to GKE + +# Triggers on push to main — after the soul binary is built and published +# by ci.yaml, this workflow builds the Docker image and blue-green deploys +# to the neuron-prod namespace on GKE. +# +# This workflow runs AFTER ci.yaml has published the neuron-soul generic +# artifact to Artifact Registry. The Docker build downloads that binary. + +on: + push: + branches: + - main + workflow_dispatch: + inputs: + slot: + description: "Target blue-green slot (blue or green)" + required: false + default: "green" + +jobs: + deploy: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + apt-get update -qq + apt-get install -y --no-install-recommends \ + ca-certificates curl gnupg apt-transport-https kubectl + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \ + > /etc/apt/sources.list.d/google-cloud-sdk.list + curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg \ + | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg + apt-get update -qq && apt-get install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin + + - name: Authenticate to GCP + env: + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + run: | + echo "${GCP_SA_KEY}" > /tmp/gcp-key.json + gcloud auth activate-service-account --key-file=/tmp/gcp-key.json + gcloud config set project neuron-785695 + gcloud auth configure-docker us-central1-docker.pkg.dev --quiet + + - name: Get GKE credentials + run: | + gcloud container clusters get-credentials neuron-platform \ + --region=us-central1 \ + --project=neuron-785695 + + - name: Determine image tag and slot + id: vars + run: | + SHA="${GITEA_SHA:0:8}" + IMAGE="us-central1-docker.pkg.dev/neuron-785695/neuron-api/neuron-soul:${SHA}" + echo "sha=${SHA}" >> "$GITEA_OUTPUT" + echo "image=${IMAGE}" >> "$GITEA_OUTPUT" + + # Determine which slot is currently idle (0 replicas = idle slot) + # If both are at 0 (fresh deploy), default to blue + BLUE_REPLICAS=$(kubectl get deployment/neuron-mcp-blue \ + -n neuron-prod \ + -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0") + GREEN_REPLICAS=$(kubectl get deployment/neuron-mcp-green \ + -n neuron-prod \ + -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0") + + echo " Blue replicas: ${BLUE_REPLICAS}" + echo " Green replicas: ${GREEN_REPLICAS}" + + # Use workflow_dispatch override if provided, otherwise pick the idle slot + if [ "${{ github.event.inputs.slot }}" != "" ]; then + SLOT="${{ github.event.inputs.slot }}" + elif [ "${GREEN_REPLICAS}" -eq 0 ] && [ "${BLUE_REPLICAS}" -gt 0 ]; then + SLOT="green" + elif [ "${BLUE_REPLICAS}" -eq 0 ] && [ "${GREEN_REPLICAS}" -gt 0 ]; then + SLOT="blue" + else + # Fresh cluster — deploy to blue first + SLOT="blue" + fi + + echo "slot=${SLOT}" >> "$GITEA_OUTPUT" + echo " Deploying to slot: ${SLOT}" + + - name: Build and push Docker image + env: + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + run: | + IMAGE="${{ steps.vars.outputs.image }}" + SHA="${{ steps.vars.outputs.sha }}" + + echo "Building ${IMAGE}..." + docker build \ + --build-arg SOUL_VERSION="${SHA}" \ + --secret id=gcp_sa_key,env=GCP_SA_KEY \ + --tag "${IMAGE}" \ + --tag "us-central1-docker.pkg.dev/neuron-785695/neuron-api/neuron-soul:latest" \ + . + + echo "Pushing ${IMAGE}..." + docker push "${IMAGE}" + docker push "us-central1-docker.pkg.dev/neuron-785695/neuron-api/neuron-soul:latest" + + - name: Blue-green deploy to GKE + run: | + chmod +x scripts/blue-green-deploy.sh + scripts/blue-green-deploy.sh \ + --image "${{ steps.vars.outputs.image }}" \ + --slot "${{ steps.vars.outputs.slot }}" + + - name: Verify deployment + run: | + SLOT="${{ steps.vars.outputs.slot }}" + echo "Verifying neuron-mcp-${SLOT} is healthy..." + kubectl rollout status deployment/"neuron-mcp-${SLOT}" \ + --namespace=neuron-prod \ + --timeout=3m + + echo "Active service endpoints:" + kubectl get endpoints neuron-mcp -n neuron-prod + + echo "Pod status:" + kubectl get pods -n neuron-prod -l app=neuron-mcp + + - name: Cleanup + if: always() + run: rm -f /tmp/gcp-key.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b3a3d0e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,101 @@ +# Neuron Soul — GKE container image +# +# Build strategy: +# 1. Download the pre-built linux/amd64 soul binary from Artifact Registry +# (package: neuron-soul, repository: foundation-dev). +# The binary is built by CI from soul.el and published as a generic artifact. +# 2. Package it in a minimal Ubuntu 22.04 runtime with glibc and libcurl. +# +# The soul runs in file mode (no HTTP Engram sidecar): +# - SOUL_ENGRAM_PATH=/data/snapshot.json → reads/writes engram from mounted PVC +# - ENGRAM_URL must NOT be set → absence triggers file mode +# +# Required env vars (injected via ExternalSecret at runtime): +# NEURON_PORT, NEURON_LLM_0_URL, NEURON_LLM_0_KEY, NEURON_LLM_0_FORMAT, +# SOUL_CGI_ID, SOUL_IDENTITY, NEURON_TOKEN, NEURON_API_URL, SOUL_ENGRAM_PATH + +ARG SOUL_VERSION=latest + +FROM ubuntu:22.04 AS downloader + +ARG SOUL_VERSION +ARG GCP_SA_KEY + +RUN apt-get update -qq && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + gnupg \ + apt-transport-https && \ + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \ + > /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg \ + | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg && \ + apt-get update -qq && \ + apt-get install -y --no-install-recommends google-cloud-cli && \ + rm -rf /var/lib/apt/lists/* + +# Authenticate and download the soul binary from Artifact Registry. +# SOUL_VERSION is the 8-char git SHA tag published by CI (e.g. ea271d5c). +# The binary is stored as a generic artifact — download to /tmp/soul/neuron. +RUN --mount=type=secret,id=gcp_sa_key \ + GCP_SA_KEY_FILE=$(cat /run/secrets/gcp_sa_key 2>/dev/null || echo "") && \ + if [ -n "$GCP_SA_KEY_FILE" ]; then \ + echo "$GCP_SA_KEY_FILE" > /tmp/gcp-key.json && \ + gcloud auth activate-service-account --key-file=/tmp/gcp-key.json; \ + fi && \ + gcloud config set project neuron-785695 && \ + mkdir -p /tmp/soul && \ + if [ "${SOUL_VERSION}" = "latest" ]; then \ + VERSION=$(gcloud artifacts versions list \ + --repository=foundation-dev \ + --location=us-central1 \ + --project=neuron-785695 \ + --package=neuron-soul \ + --sort-by="~createTime" \ + --limit=1 \ + --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}'); \ + else \ + VERSION="${SOUL_VERSION}"; \ + fi && \ + echo "Downloading neuron-soul@${VERSION}" && \ + gcloud artifacts generic download \ + --repository=foundation-dev \ + --location=us-central1 \ + --project=neuron-785695 \ + --package=neuron-soul \ + --version="${VERSION}" \ + --destination=/tmp/soul/ && \ + mv /tmp/soul/neuron* /tmp/soul/neuron 2>/dev/null || true && \ + chmod +x /tmp/soul/neuron && \ + rm -f /tmp/gcp-key.json + +# Runtime image — minimal Ubuntu 22.04 with only what the soul binary needs. +FROM ubuntu:22.04 + +RUN apt-get update -qq && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + libcurl4 \ + curl && \ + rm -rf /var/lib/apt/lists/* && \ + useradd -r -u 1000 -m -s /bin/bash soul + +COPY --from=downloader /tmp/soul/neuron /usr/local/bin/neuron +RUN chmod +x /usr/local/bin/neuron + +# /data is the engram mount point (PVC at runtime). +# Create it owned by soul user so the binary can write snapshot.json. +RUN mkdir -p /data && chown soul:soul /data + +USER soul +WORKDIR /home/soul + +EXPOSE 7770 + +# SOUL_ENGRAM_PATH and other env vars are injected via k8s ExternalSecret. +# ENGRAM_URL must NOT be set — its absence triggers file mode. +ENV NEURON_PORT=7770 \ + SOUL_ENGRAM_PATH=/data/snapshot.json + +ENTRYPOINT ["/usr/local/bin/neuron"] diff --git a/scripts/blue-green-deploy.sh b/scripts/blue-green-deploy.sh new file mode 100755 index 0000000..c10b208 --- /dev/null +++ b/scripts/blue-green-deploy.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# blue-green-deploy.sh — swap the active soul slot on GKE +# +# Usage: +# blue-green-deploy.sh --image --slot +# +# What it does: +# 1. Sets the new image on the target slot deployment +# 2. Scales the target slot to 1 replica and waits for it to become ready +# 3. Patches the neuron-mcp Service selector to point at the new slot +# 4. Scales the old slot down to 0 replicas +# +# This script uses kubectl imperatively for the live traffic swap. +# After a successful swap, Argo CD manifests should be updated to reflect +# the new active slot (so Argo CD doesn't revert the replica counts on next sync). +# +# Prerequisites: +# - kubectl configured with access to neuron-platform cluster +# - Sufficient RBAC: get/patch/scale Deployments and Services in neuron-prod +# +# Examples: +# blue-green-deploy.sh --image us-central1-docker.pkg.dev/neuron-785695/neuron-api/neuron-soul:abc12345 --slot green +# blue-green-deploy.sh --image us-central1-docker.pkg.dev/neuron-785695/neuron-api/neuron-soul:latest --slot blue + +set -euo pipefail + +NAMESPACE="neuron-prod" +APP="neuron-mcp" +SERVICE="neuron-mcp" + +IMAGE="" +NEW_SLOT="" + +usage() { + echo "Usage: $0 --image --slot " + echo "" + echo " --image Full image URI including tag" + echo " --slot Target slot to deploy to: blue or green" + exit 1 +} + +while [[ $# -gt 0 ]]; do + case $1 in + --image) + IMAGE="$2"; shift 2 ;; + --slot) + NEW_SLOT="$2"; shift 2 ;; + -h|--help) + usage ;; + *) + echo "Unknown argument: $1" + usage ;; + esac +done + +[[ -z "$IMAGE" ]] && { echo "ERROR: --image is required"; usage; } +[[ -z "$NEW_SLOT" ]] && { echo "ERROR: --slot is required"; usage; } +[[ "$NEW_SLOT" != "blue" && "$NEW_SLOT" != "green" ]] && { + echo "ERROR: --slot must be 'blue' or 'green'"; exit 1 +} + +# Determine the old (currently active) slot +if [[ "$NEW_SLOT" == "blue" ]]; then + OLD_SLOT="green" +else + OLD_SLOT="blue" +fi + +NEW_DEPLOYMENT="${APP}-${NEW_SLOT}" +OLD_DEPLOYMENT="${APP}-${OLD_SLOT}" + +echo "==> Deploying to slot: ${NEW_SLOT}" +echo " Image: ${IMAGE}" +echo " New deploy: ${NEW_DEPLOYMENT}" +echo " Old deploy: ${OLD_DEPLOYMENT}" +echo " Namespace: ${NAMESPACE}" +echo "" + +# Step 1: Update the image on the new slot +echo "[1/4] Setting image on ${NEW_DEPLOYMENT}..." +kubectl set image deployment/"${NEW_DEPLOYMENT}" \ + soul="${IMAGE}" \ + --namespace="${NAMESPACE}" + +# Step 2: Scale the new slot up (it may already be at 0) +echo "[2/4] Scaling ${NEW_DEPLOYMENT} to 1 replica..." +kubectl scale deployment/"${NEW_DEPLOYMENT}" \ + --replicas=1 \ + --namespace="${NAMESPACE}" + +# Wait for rollout to complete (pod ready) +echo " Waiting for rollout to complete (timeout: 5m)..." +kubectl rollout status deployment/"${NEW_DEPLOYMENT}" \ + --namespace="${NAMESPACE}" \ + --timeout=5m + +echo " ${NEW_DEPLOYMENT} is ready." + +# Step 3: Patch the service selector to point at the new slot +echo "[3/4] Flipping service selector to slot=${NEW_SLOT}..." +kubectl patch service "${SERVICE}" \ + --namespace="${NAMESPACE}" \ + --type=merge \ + --patch="{\"spec\":{\"selector\":{\"app\":\"${APP}\",\"slot\":\"${NEW_SLOT}\"}}}" + +echo " Traffic now routing to ${NEW_SLOT}." + +# Step 4: Scale down the old slot +echo "[4/4] Scaling ${OLD_DEPLOYMENT} down to 0 replicas..." +kubectl scale deployment/"${OLD_DEPLOYMENT}" \ + --replicas=0 \ + --namespace="${NAMESPACE}" + +echo "" +echo "==> Blue-green swap complete." +echo " Active slot: ${NEW_SLOT} (${IMAGE})" +echo " Idle slot: ${OLD_SLOT} (scaled to 0)" +echo "" +echo "NOTE: Update Argo CD manifests to reflect new state:" +echo " - deployment-${NEW_SLOT}.yaml: replicas: 1, image: ${IMAGE}" +echo " - deployment-${OLD_SLOT}.yaml: replicas: 0" diff --git a/scripts/seed-engram-gke.sh b/scripts/seed-engram-gke.sh new file mode 100755 index 0000000..8efb112 --- /dev/null +++ b/scripts/seed-engram-gke.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# seed-engram-gke.sh — seed the GKE soul's PVC with the latest engram backup +# +# Downloads the latest engram backup from GCS, extracts snapshot.json, +# and copies it into the neuron-prod PVC via a temporary seed Job. +# +# Usage: +# seed-engram-gke.sh [--hostname ] [--backup ] [--dry-run] +# +# Options: +# --hostname Mac hostname used as the backup source prefix +# Default: auto-detected from GCS bucket listing +# --backup Full GCS path to a specific backup tarball +# Overrides --hostname; use if you want a specific backup +# --dry-run Print what would happen without executing +# +# Examples: +# seed-engram-gke.sh +# seed-engram-gke.sh --hostname Wills-MacBook-Pro.local +# seed-engram-gke.sh --backup gs://neuron-db-backup/local-mac/Wills-MacBook-Pro.local/engram-20260509T1200.tar.gz + +set -euo pipefail + +NAMESPACE="neuron-prod" +PVC_NAME="neuron-engram-data" +GCS_BUCKET="gs://neuron-db-backup/local-mac" +HOSTNAME_PREFIX="" +SPECIFIC_BACKUP="" +DRY_RUN=false +SEED_JOB_NAME="engram-seed-$(date +%s)" + +usage() { + grep '^#' "$0" | grep -v '#!/' | sed 's/^# //' | sed 's/^#//' + exit 0 +} + +while [[ $# -gt 0 ]]; do + case $1 in + --hostname) HOSTNAME_PREFIX="$2"; shift 2 ;; + --backup) SPECIFIC_BACKUP="$2"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage ;; + *) echo "Unknown argument: $1"; exit 1 ;; + esac +done + +run() { + if $DRY_RUN; then + echo "[dry-run] $*" + else + "$@" + fi +} + +echo "==> Neuron Engram GKE Seed" +echo " Namespace: ${NAMESPACE}" +echo " PVC: ${PVC_NAME}" +echo "" + +# Step 1: Locate the backup +if [[ -n "$SPECIFIC_BACKUP" ]]; then + BACKUP_PATH="$SPECIFIC_BACKUP" + echo "[1/5] Using specified backup: ${BACKUP_PATH}" +else + echo "[1/5] Finding latest backup in GCS..." + + if [[ -z "$HOSTNAME_PREFIX" ]]; then + # Auto-detect: list prefixes and pick the first (most likely the only one) + HOSTNAME_PREFIX=$(gsutil ls "${GCS_BUCKET}/" 2>/dev/null \ + | sed "s|${GCS_BUCKET}/||" \ + | sed 's|/||' \ + | head -1) + if [[ -z "$HOSTNAME_PREFIX" ]]; then + echo "ERROR: Could not find any backup prefixes in ${GCS_BUCKET}/" + echo " Run: gsutil ls ${GCS_BUCKET}/" + exit 1 + fi + echo " Auto-detected hostname prefix: ${HOSTNAME_PREFIX}" + fi + + BACKUP_PATH=$(gsutil ls "${GCS_BUCKET}/${HOSTNAME_PREFIX}/" 2>/dev/null \ + | grep "engram-.*\.tar\.gz" \ + | sort -r \ + | head -1) + + if [[ -z "$BACKUP_PATH" ]]; then + echo "ERROR: No backup found at ${GCS_BUCKET}/${HOSTNAME_PREFIX}/" + exit 1 + fi + echo " Latest backup: ${BACKUP_PATH}" +fi + +# Step 2: Download and extract snapshot.json +TMP_DIR=$(mktemp -d) +trap 'rm -rf "${TMP_DIR}"' EXIT + +echo "[2/5] Downloading backup..." +run gsutil cp "${BACKUP_PATH}" "${TMP_DIR}/engram.tar.gz" + +echo "[3/5] Extracting snapshot.json..." +if $DRY_RUN; then + echo "[dry-run] tar -xzf ${TMP_DIR}/engram.tar.gz -C ${TMP_DIR} --wildcards '*snapshot.json'" + SNAPSHOT_PATH="${TMP_DIR}/snapshot.json" +else + tar -xzf "${TMP_DIR}/engram.tar.gz" -C "${TMP_DIR}" 2>/dev/null || true + # Handle nested paths (backup may be engram/snapshot.json or just snapshot.json) + SNAPSHOT_PATH=$(find "${TMP_DIR}" -name "snapshot.json" | head -1) + if [[ -z "$SNAPSHOT_PATH" ]]; then + echo "ERROR: snapshot.json not found in backup archive" + echo " Archive contents:" + tar -tzf "${TMP_DIR}/engram.tar.gz" | head -20 + exit 1 + fi + SNAPSHOT_SIZE=$(du -h "$SNAPSHOT_PATH" | cut -f1) + echo " snapshot.json: ${SNAPSHOT_SIZE} at ${SNAPSHOT_PATH}" +fi + +# Step 4: Copy into PVC via a seed Job +# The Job mounts the PVC, then we kubectl cp the snapshot.json into the pod. +echo "[4/5] Launching seed Job to access PVC..." + +SEED_JOB_MANIFEST=$(cat <" + echo "[dry-run] kubectl wait --for=condition=ready pod -l job-name=${SEED_JOB_NAME} -n ${NAMESPACE}" + echo "[dry-run] kubectl cp ${SNAPSHOT_PATH} ${NAMESPACE}/:/data/snapshot.json" + echo "[dry-run] kubectl delete job/${SEED_JOB_NAME} -n ${NAMESPACE}" +else + echo "$SEED_JOB_MANIFEST" | kubectl apply -f - + + echo " Waiting for seed pod to be ready (timeout: 2m)..." + kubectl wait --for=condition=ready pod \ + -l "job-name=${SEED_JOB_NAME}" \ + --namespace="${NAMESPACE}" \ + --timeout=2m + + SEED_POD=$(kubectl get pods \ + -l "job-name=${SEED_JOB_NAME}" \ + --namespace="${NAMESPACE}" \ + --output=jsonpath='{.items[0].metadata.name}') + + echo " Seed pod: ${SEED_POD}" + + echo "[5/5] Copying snapshot.json into PVC..." + kubectl cp "${SNAPSHOT_PATH}" "${NAMESPACE}/${SEED_POD}:/data/snapshot.json" + + # Verify + PVC_SIZE=$(kubectl exec "${SEED_POD}" --namespace="${NAMESPACE}" -- \ + stat -c%s /data/snapshot.json 2>/dev/null || echo "unknown") + echo " PVC snapshot.json size: ${PVC_SIZE} bytes" + + echo " Cleaning up seed Job..." + kubectl delete job/"${SEED_JOB_NAME}" --namespace="${NAMESPACE}" --ignore-not-found +fi + +echo "" +echo "==> Engram seed complete." +echo " The soul pod will read /data/snapshot.json on next boot." +echo "" +echo " Scale the blue deployment to pick up the seeded engram:" +echo " kubectl scale deployment/neuron-mcp-blue --replicas=1 -n ${NAMESPACE}"