fix(neuron-mcp): blue/green deployment, memory limits, preStop drain hook
- Rename Deployment to neuron-mcp-blue with slot=blue label - Add neuron-mcp-green Deployment scaffold (replicas=0, ready to receive new image) - Service selector now routes by slot label; annotation tracks active slot - Memory: 768Mi → 1536Mi limit, 256Mi → 512Mi request (fixes OOM pod restarts / 502s) - Add JAVA_TOOL_OPTIONS: container-aware GC + ExitOnOutOfMemoryError - Add preStop sleep 8s so Traefik drains in-flight connections before pod terminates - terminationGracePeriodSeconds: 30 - Add scripts/blue-green-deploy.sh: automated slot-swap with health check and rollback
This commit is contained in:
Will Anderson
@@ -0,0 +1,76 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: neuron-mcp-green
|
||||
namespace: neuron-prod
|
||||
labels:
|
||||
app: neuron-mcp
|
||||
slot: green
|
||||
env: prod
|
||||
spec:
|
||||
# Scaled to 0 when inactive. Deploy process scales to 1, verifies health, then switches service.
|
||||
replicas: 0
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app: neuron-mcp
|
||||
slot: green
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: neuron-mcp
|
||||
slot: green
|
||||
env: prod
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 30
|
||||
containers:
|
||||
- name: neuron-mcp
|
||||
# IMAGE UPDATED HERE before scaling to 1
|
||||
image: registry.neuralplatform.ai/neuron-technologies/neuron-mcp:v0.8.3
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
env:
|
||||
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||||
value: "http://alloy.monitoring.svc.cluster.local:4318"
|
||||
- name: JAVA_TOOL_OPTIONS
|
||||
value: "-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0 -XX:+ExitOnOutOfMemoryError"
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: neuron-prod-config
|
||||
- secretRef:
|
||||
name: neuron-prod-secrets
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1500m
|
||||
memory: 1536Mi
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
command: ["sh", "-c", "sleep 8"]
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/readiness
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: neuron-prod-data
|
||||
@@ -1,10 +1,11 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: neuron-mcp
|
||||
name: neuron-mcp-blue
|
||||
namespace: neuron-prod
|
||||
labels:
|
||||
app: neuron-mcp
|
||||
slot: blue
|
||||
env: prod
|
||||
spec:
|
||||
replicas: 1
|
||||
@@ -13,14 +14,15 @@ spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: neuron-mcp
|
||||
slot: blue
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: neuron-mcp
|
||||
slot: blue
|
||||
env: prod
|
||||
annotations:
|
||||
kubectl.kubernetes.io/restartedAt: "2026-04-24T10:00:00Z"
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 30
|
||||
containers:
|
||||
- name: neuron-mcp
|
||||
image: registry.neuralplatform.ai/neuron-technologies/neuron-mcp:v0.8.3
|
||||
@@ -31,6 +33,8 @@ spec:
|
||||
env:
|
||||
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||||
value: "http://alloy.monitoring.svc.cluster.local:4318"
|
||||
- name: JAVA_TOOL_OPTIONS
|
||||
value: "-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0 -XX:+ExitOnOutOfMemoryError"
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: neuron-prod-config
|
||||
@@ -42,10 +46,15 @@ spec:
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 768Mi
|
||||
cpu: 1500m
|
||||
memory: 1536Mi
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
# Give Traefik time to drain in-flight connections before pod terminates
|
||||
command: ["sh", "-c", "sleep 8"]
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
@@ -57,7 +66,7 @@ spec:
|
||||
httpGet:
|
||||
path: /actuator/health/readiness
|
||||
port: http
|
||||
initialDelaySeconds: 15
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
volumes:
|
||||
|
||||
@@ -21,9 +21,14 @@ metadata:
|
||||
namespace: neuron-prod
|
||||
labels:
|
||||
app: neuron-mcp
|
||||
annotations:
|
||||
# Blue/green: change slot to "green" then back to "blue" to switch traffic.
|
||||
# Process: deploy green → verify health → patch this selector → scale blue to 0.
|
||||
neuron.ai/active-slot: "blue"
|
||||
spec:
|
||||
selector:
|
||||
app: neuron-mcp
|
||||
slot: blue
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env bash
|
||||
# Blue/Green deploy for neuron-mcp
|
||||
# Usage: ./blue-green-deploy.sh <new-image-tag>
|
||||
# Example: ./blue-green-deploy.sh v0.8.4
|
||||
#
|
||||
# PVC is ReadWriteOnce — only ONE pod can hold it at a time.
|
||||
# Process: scale IDLE slot to 1 (new image, it waits for PVC) →
|
||||
# scale ACTIVE slot to 0 (releases PVC) → idle pod claims PVC + starts →
|
||||
# verify health → switch service selector → done.
|
||||
# Rollback: re-run with old tag, slots flip back.
|
||||
set -euo pipefail
|
||||
|
||||
KUBECONFIG="${KUBECONFIG:-$HOME/.kube/legion-config}"
|
||||
NS="neuron-prod"
|
||||
SVC="neuron-mcp"
|
||||
IMAGE_BASE="registry.neuralplatform.ai/neuron-technologies/neuron-mcp"
|
||||
TAG="${1:-}"
|
||||
|
||||
if [[ -z "$TAG" ]]; then
|
||||
echo "Usage: $0 <image-tag>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
KC="kubectl --kubeconfig=$KUBECONFIG -n $NS"
|
||||
|
||||
# Determine active and idle slots
|
||||
ACTIVE_SLOT=$($KC get svc $SVC -o jsonpath='{.spec.selector.slot}')
|
||||
if [[ "$ACTIVE_SLOT" == "blue" ]]; then
|
||||
IDLE_SLOT="green"
|
||||
else
|
||||
IDLE_SLOT="blue"
|
||||
fi
|
||||
|
||||
echo "→ Active slot: $ACTIVE_SLOT | Deploying to: $IDLE_SLOT | Image: $IMAGE_BASE:$TAG"
|
||||
|
||||
# 1. Update idle deployment with new image (still scaled to 0)
|
||||
echo "→ Updating neuron-mcp-$IDLE_SLOT image..."
|
||||
$KC set image deployment/neuron-mcp-$IDLE_SLOT neuron-mcp=$IMAGE_BASE:$TAG
|
||||
|
||||
# 2. Scale active down — releases the PVC
|
||||
echo "→ Scaling neuron-mcp-$ACTIVE_SLOT to 0 (releases PVC)..."
|
||||
$KC scale deployment/neuron-mcp-$ACTIVE_SLOT --replicas=0
|
||||
$KC rollout status deployment/neuron-mcp-$ACTIVE_SLOT --timeout=60s || true
|
||||
|
||||
# 3. Scale idle up — claims PVC, starts with new image
|
||||
echo "→ Scaling neuron-mcp-$IDLE_SLOT to 1..."
|
||||
$KC scale deployment/neuron-mcp-$IDLE_SLOT --replicas=1
|
||||
|
||||
# 4. Wait for readiness (up to 3 min for JVM startup)
|
||||
echo "→ Waiting for neuron-mcp-$IDLE_SLOT to be ready..."
|
||||
$KC rollout status deployment/neuron-mcp-$IDLE_SLOT --timeout=180s
|
||||
|
||||
# 5. Health check
|
||||
POD=$($KC get pod -l app=neuron-mcp,slot=$IDLE_SLOT -o jsonpath='{.items[0].metadata.name}')
|
||||
HEALTH=$($KC exec "$POD" -- wget -qO- http://localhost:8080/actuator/health/readiness 2>/dev/null || echo '{"status":"DOWN"}')
|
||||
STATUS=$(echo "$HEALTH" | python3 -c "import sys,json; print(json.load(sys.stdin).get('status','UNKNOWN'))" 2>/dev/null || echo "UNKNOWN")
|
||||
|
||||
if [[ "$STATUS" != "UP" ]]; then
|
||||
echo "✗ Health check failed (status=$STATUS). Rolling back — scaling $IDLE_SLOT back to 0, $ACTIVE_SLOT back to 1."
|
||||
$KC scale deployment/neuron-mcp-$IDLE_SLOT --replicas=0
|
||||
$KC scale deployment/neuron-mcp-$ACTIVE_SLOT --replicas=1
|
||||
$KC rollout status deployment/neuron-mcp-$ACTIVE_SLOT --timeout=180s
|
||||
echo "✓ Rollback complete. Still on $ACTIVE_SLOT ($IMAGE_BASE:$(kubectl --kubeconfig=$KUBECONFIG -n $NS get deployment neuron-mcp-$ACTIVE_SLOT -o jsonpath='{.spec.template.spec.containers[0].image}' | cut -d: -f2))."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ Health OK — switching service selector to $IDLE_SLOT..."
|
||||
|
||||
# 6. Switch service selector
|
||||
$KC patch svc $SVC \
|
||||
-p "{\"spec\":{\"selector\":{\"app\":\"neuron-mcp\",\"slot\":\"$IDLE_SLOT\"}},\"metadata\":{\"annotations\":{\"neuron.ai/active-slot\":\"$IDLE_SLOT\"}}}"
|
||||
|
||||
echo "✓ Traffic now on neuron-mcp-$IDLE_SLOT ($IMAGE_BASE:$TAG)"
|
||||
echo " Previous slot neuron-mcp-$ACTIVE_SLOT is scaled to 0 (kept for instant rollback)."
|
||||
echo ""
|
||||
echo " To rollback: $0 <previous-tag> (will swap slots back)"
|
||||
Reference in New Issue
Block a user