fix(neuron-mcp): blue/green deployment, memory limits, preStop drain hook
- Rename Deployment to neuron-mcp-blue with slot=blue label - Add neuron-mcp-green Deployment scaffold (replicas=0, ready to receive new image) - Service selector now routes by slot label; annotation tracks active slot - Memory: 768Mi → 1536Mi limit, 256Mi → 512Mi request (fixes OOM pod restarts / 502s) - Add JAVA_TOOL_OPTIONS: container-aware GC + ExitOnOutOfMemoryError - Add preStop sleep 8s so Traefik drains in-flight connections before pod terminates - terminationGracePeriodSeconds: 30 - Add scripts/blue-green-deploy.sh: automated slot-swap with health check and rollback
This commit is contained in:
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env bash
|
||||
# Blue/Green deploy for neuron-mcp
|
||||
# Usage: ./blue-green-deploy.sh <new-image-tag>
|
||||
# Example: ./blue-green-deploy.sh v0.8.4
|
||||
#
|
||||
# PVC is ReadWriteOnce — only ONE pod can hold it at a time.
|
||||
# Process: scale IDLE slot to 1 (new image, it waits for PVC) →
|
||||
# scale ACTIVE slot to 0 (releases PVC) → idle pod claims PVC + starts →
|
||||
# verify health → switch service selector → done.
|
||||
# Rollback: re-run with old tag, slots flip back.
|
||||
set -euo pipefail
|
||||
|
||||
KUBECONFIG="${KUBECONFIG:-$HOME/.kube/legion-config}"
|
||||
NS="neuron-prod"
|
||||
SVC="neuron-mcp"
|
||||
IMAGE_BASE="registry.neuralplatform.ai/neuron-technologies/neuron-mcp"
|
||||
TAG="${1:-}"
|
||||
|
||||
if [[ -z "$TAG" ]]; then
|
||||
echo "Usage: $0 <image-tag>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
KC="kubectl --kubeconfig=$KUBECONFIG -n $NS"
|
||||
|
||||
# Determine active and idle slots
|
||||
ACTIVE_SLOT=$($KC get svc $SVC -o jsonpath='{.spec.selector.slot}')
|
||||
if [[ "$ACTIVE_SLOT" == "blue" ]]; then
|
||||
IDLE_SLOT="green"
|
||||
else
|
||||
IDLE_SLOT="blue"
|
||||
fi
|
||||
|
||||
echo "→ Active slot: $ACTIVE_SLOT | Deploying to: $IDLE_SLOT | Image: $IMAGE_BASE:$TAG"
|
||||
|
||||
# 1. Update idle deployment with new image (still scaled to 0)
|
||||
echo "→ Updating neuron-mcp-$IDLE_SLOT image..."
|
||||
$KC set image deployment/neuron-mcp-$IDLE_SLOT neuron-mcp=$IMAGE_BASE:$TAG
|
||||
|
||||
# 2. Scale active down — releases the PVC
|
||||
echo "→ Scaling neuron-mcp-$ACTIVE_SLOT to 0 (releases PVC)..."
|
||||
$KC scale deployment/neuron-mcp-$ACTIVE_SLOT --replicas=0
|
||||
$KC rollout status deployment/neuron-mcp-$ACTIVE_SLOT --timeout=60s || true
|
||||
|
||||
# 3. Scale idle up — claims PVC, starts with new image
|
||||
echo "→ Scaling neuron-mcp-$IDLE_SLOT to 1..."
|
||||
$KC scale deployment/neuron-mcp-$IDLE_SLOT --replicas=1
|
||||
|
||||
# 4. Wait for readiness (up to 3 min for JVM startup)
|
||||
echo "→ Waiting for neuron-mcp-$IDLE_SLOT to be ready..."
|
||||
$KC rollout status deployment/neuron-mcp-$IDLE_SLOT --timeout=180s
|
||||
|
||||
# 5. Health check
|
||||
POD=$($KC get pod -l app=neuron-mcp,slot=$IDLE_SLOT -o jsonpath='{.items[0].metadata.name}')
|
||||
HEALTH=$($KC exec "$POD" -- wget -qO- http://localhost:8080/actuator/health/readiness 2>/dev/null || echo '{"status":"DOWN"}')
|
||||
STATUS=$(echo "$HEALTH" | python3 -c "import sys,json; print(json.load(sys.stdin).get('status','UNKNOWN'))" 2>/dev/null || echo "UNKNOWN")
|
||||
|
||||
if [[ "$STATUS" != "UP" ]]; then
|
||||
echo "✗ Health check failed (status=$STATUS). Rolling back — scaling $IDLE_SLOT back to 0, $ACTIVE_SLOT back to 1."
|
||||
$KC scale deployment/neuron-mcp-$IDLE_SLOT --replicas=0
|
||||
$KC scale deployment/neuron-mcp-$ACTIVE_SLOT --replicas=1
|
||||
$KC rollout status deployment/neuron-mcp-$ACTIVE_SLOT --timeout=180s
|
||||
echo "✓ Rollback complete. Still on $ACTIVE_SLOT ($IMAGE_BASE:$(kubectl --kubeconfig=$KUBECONFIG -n $NS get deployment neuron-mcp-$ACTIVE_SLOT -o jsonpath='{.spec.template.spec.containers[0].image}' | cut -d: -f2))."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ Health OK — switching service selector to $IDLE_SLOT..."
|
||||
|
||||
# 6. Switch service selector
|
||||
$KC patch svc $SVC \
|
||||
-p "{\"spec\":{\"selector\":{\"app\":\"neuron-mcp\",\"slot\":\"$IDLE_SLOT\"}},\"metadata\":{\"annotations\":{\"neuron.ai/active-slot\":\"$IDLE_SLOT\"}}}"
|
||||
|
||||
echo "✓ Traffic now on neuron-mcp-$IDLE_SLOT ($IMAGE_BASE:$TAG)"
|
||||
echo " Previous slot neuron-mcp-$ACTIVE_SLOT is scaled to 0 (kept for instant rollback)."
|
||||
echo ""
|
||||
echo " To rollback: $0 <previous-tag> (will swap slots back)"
|
||||
Reference in New Issue
Block a user