Will Anderson
f86ee04467
- Rename Deployment to neuron-mcp-blue with slot=blue label - Add neuron-mcp-green Deployment scaffold (replicas=0, ready to receive new image) - Service selector now routes by slot label; annotation tracks active slot - Memory: 768Mi → 1536Mi limit, 256Mi → 512Mi request (fixes OOM pod restarts / 502s) - Add JAVA_TOOL_OPTIONS: container-aware GC + ExitOnOutOfMemoryError - Add preStop sleep 8s so Traefik drains in-flight connections before pod terminates - terminationGracePeriodSeconds: 30 - Add scripts/blue-green-deploy.sh: automated slot-swap with health check and rollback
77 lines
3.2 KiB
Bash
Executable File
77 lines
3.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Blue/Green deploy for neuron-mcp
|
|
# Usage: ./blue-green-deploy.sh <new-image-tag>
|
|
# Example: ./blue-green-deploy.sh v0.8.4
|
|
#
|
|
# PVC is ReadWriteOnce — only ONE pod can hold it at a time.
|
|
# Process: scale IDLE slot to 1 (new image, it waits for PVC) →
|
|
# scale ACTIVE slot to 0 (releases PVC) → idle pod claims PVC + starts →
|
|
# verify health → switch service selector → done.
|
|
# Rollback: re-run with old tag, slots flip back.
|
|
set -euo pipefail
|
|
|
|
KUBECONFIG="${KUBECONFIG:-$HOME/.kube/legion-config}"
|
|
NS="neuron-prod"
|
|
SVC="neuron-mcp"
|
|
IMAGE_BASE="registry.neuralplatform.ai/neuron-technologies/neuron-mcp"
|
|
TAG="${1:-}"
|
|
|
|
if [[ -z "$TAG" ]]; then
|
|
echo "Usage: $0 <image-tag>"
|
|
exit 1
|
|
fi
|
|
|
|
KC="kubectl --kubeconfig=$KUBECONFIG -n $NS"
|
|
|
|
# Determine active and idle slots
|
|
ACTIVE_SLOT=$($KC get svc $SVC -o jsonpath='{.spec.selector.slot}')
|
|
if [[ "$ACTIVE_SLOT" == "blue" ]]; then
|
|
IDLE_SLOT="green"
|
|
else
|
|
IDLE_SLOT="blue"
|
|
fi
|
|
|
|
echo "→ Active slot: $ACTIVE_SLOT | Deploying to: $IDLE_SLOT | Image: $IMAGE_BASE:$TAG"
|
|
|
|
# 1. Update idle deployment with new image (still scaled to 0)
|
|
echo "→ Updating neuron-mcp-$IDLE_SLOT image..."
|
|
$KC set image deployment/neuron-mcp-$IDLE_SLOT neuron-mcp=$IMAGE_BASE:$TAG
|
|
|
|
# 2. Scale active down — releases the PVC
|
|
echo "→ Scaling neuron-mcp-$ACTIVE_SLOT to 0 (releases PVC)..."
|
|
$KC scale deployment/neuron-mcp-$ACTIVE_SLOT --replicas=0
|
|
$KC rollout status deployment/neuron-mcp-$ACTIVE_SLOT --timeout=60s || true
|
|
|
|
# 3. Scale idle up — claims PVC, starts with new image
|
|
echo "→ Scaling neuron-mcp-$IDLE_SLOT to 1..."
|
|
$KC scale deployment/neuron-mcp-$IDLE_SLOT --replicas=1
|
|
|
|
# 4. Wait for readiness (up to 3 min for JVM startup)
|
|
echo "→ Waiting for neuron-mcp-$IDLE_SLOT to be ready..."
|
|
$KC rollout status deployment/neuron-mcp-$IDLE_SLOT --timeout=180s
|
|
|
|
# 5. Health check
|
|
POD=$($KC get pod -l app=neuron-mcp,slot=$IDLE_SLOT -o jsonpath='{.items[0].metadata.name}')
|
|
HEALTH=$($KC exec "$POD" -- wget -qO- http://localhost:8080/actuator/health/readiness 2>/dev/null || echo '{"status":"DOWN"}')
|
|
STATUS=$(echo "$HEALTH" | python3 -c "import sys,json; print(json.load(sys.stdin).get('status','UNKNOWN'))" 2>/dev/null || echo "UNKNOWN")
|
|
|
|
if [[ "$STATUS" != "UP" ]]; then
|
|
echo "✗ Health check failed (status=$STATUS). Rolling back — scaling $IDLE_SLOT back to 0, $ACTIVE_SLOT back to 1."
|
|
$KC scale deployment/neuron-mcp-$IDLE_SLOT --replicas=0
|
|
$KC scale deployment/neuron-mcp-$ACTIVE_SLOT --replicas=1
|
|
$KC rollout status deployment/neuron-mcp-$ACTIVE_SLOT --timeout=180s
|
|
echo "✓ Rollback complete. Still on $ACTIVE_SLOT ($IMAGE_BASE:$(kubectl --kubeconfig=$KUBECONFIG -n $NS get deployment neuron-mcp-$ACTIVE_SLOT -o jsonpath='{.spec.template.spec.containers[0].image}' | cut -d: -f2))."
|
|
exit 1
|
|
fi
|
|
|
|
echo "✓ Health OK — switching service selector to $IDLE_SLOT..."
|
|
|
|
# 6. Switch service selector
|
|
$KC patch svc $SVC \
|
|
-p "{\"spec\":{\"selector\":{\"app\":\"neuron-mcp\",\"slot\":\"$IDLE_SLOT\"}},\"metadata\":{\"annotations\":{\"neuron.ai/active-slot\":\"$IDLE_SLOT\"}}}"
|
|
|
|
echo "✓ Traffic now on neuron-mcp-$IDLE_SLOT ($IMAGE_BASE:$TAG)"
|
|
echo " Previous slot neuron-mcp-$ACTIVE_SLOT is scaled to 0 (kept for instant rollback)."
|
|
echo ""
|
|
echo " To rollback: $0 <previous-tag> (will swap slots back)"
|