fix(neuron-mcp): blue/green deployment, memory limits, preStop drain hook
- Rename Deployment to neuron-mcp-blue with slot=blue label - Add neuron-mcp-green Deployment scaffold (replicas=0, ready to receive new image) - Service selector now routes by slot label; annotation tracks active slot - Memory: 768Mi → 1536Mi limit, 256Mi → 512Mi request (fixes OOM pod restarts / 502s) - Add JAVA_TOOL_OPTIONS: container-aware GC + ExitOnOutOfMemoryError - Add preStop sleep 8s so Traefik drains in-flight connections before pod terminates - terminationGracePeriodSeconds: 30 - Add scripts/blue-green-deploy.sh: automated slot-swap with health check and rollback
This commit is contained in:
@@ -0,0 +1,76 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: neuron-mcp-green
|
||||||
|
namespace: neuron-prod
|
||||||
|
labels:
|
||||||
|
app: neuron-mcp
|
||||||
|
slot: green
|
||||||
|
env: prod
|
||||||
|
spec:
|
||||||
|
# Scaled to 0 when inactive. Deploy process scales to 1, verifies health, then switches service.
|
||||||
|
replicas: 0
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: neuron-mcp
|
||||||
|
slot: green
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: neuron-mcp
|
||||||
|
slot: green
|
||||||
|
env: prod
|
||||||
|
spec:
|
||||||
|
terminationGracePeriodSeconds: 30
|
||||||
|
containers:
|
||||||
|
- name: neuron-mcp
|
||||||
|
# IMAGE UPDATED HERE before scaling to 1
|
||||||
|
image: registry.neuralplatform.ai/neuron-technologies/neuron-mcp:v0.8.3
|
||||||
|
imagePullPolicy: Always
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 8080
|
||||||
|
env:
|
||||||
|
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||||||
|
value: "http://alloy.monitoring.svc.cluster.local:4318"
|
||||||
|
- name: JAVA_TOOL_OPTIONS
|
||||||
|
value: "-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0 -XX:+ExitOnOutOfMemoryError"
|
||||||
|
envFrom:
|
||||||
|
- configMapRef:
|
||||||
|
name: neuron-prod-config
|
||||||
|
- secretRef:
|
||||||
|
name: neuron-prod-secrets
|
||||||
|
volumeMounts:
|
||||||
|
- name: data
|
||||||
|
mountPath: /data
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 512Mi
|
||||||
|
limits:
|
||||||
|
cpu: 1500m
|
||||||
|
memory: 1536Mi
|
||||||
|
lifecycle:
|
||||||
|
preStop:
|
||||||
|
exec:
|
||||||
|
command: ["sh", "-c", "sleep 8"]
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /actuator/health/liveness
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 30
|
||||||
|
periodSeconds: 30
|
||||||
|
failureThreshold: 3
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /actuator/health/readiness
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 10
|
||||||
|
periodSeconds: 10
|
||||||
|
failureThreshold: 3
|
||||||
|
volumes:
|
||||||
|
- name: data
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: neuron-prod-data
|
||||||
@@ -1,10 +1,11 @@
|
|||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: Deployment
|
kind: Deployment
|
||||||
metadata:
|
metadata:
|
||||||
name: neuron-mcp
|
name: neuron-mcp-blue
|
||||||
namespace: neuron-prod
|
namespace: neuron-prod
|
||||||
labels:
|
labels:
|
||||||
app: neuron-mcp
|
app: neuron-mcp
|
||||||
|
slot: blue
|
||||||
env: prod
|
env: prod
|
||||||
spec:
|
spec:
|
||||||
replicas: 1
|
replicas: 1
|
||||||
@@ -13,14 +14,15 @@ spec:
|
|||||||
selector:
|
selector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
app: neuron-mcp
|
app: neuron-mcp
|
||||||
|
slot: blue
|
||||||
template:
|
template:
|
||||||
metadata:
|
metadata:
|
||||||
labels:
|
labels:
|
||||||
app: neuron-mcp
|
app: neuron-mcp
|
||||||
|
slot: blue
|
||||||
env: prod
|
env: prod
|
||||||
annotations:
|
|
||||||
kubectl.kubernetes.io/restartedAt: "2026-04-24T10:00:00Z"
|
|
||||||
spec:
|
spec:
|
||||||
|
terminationGracePeriodSeconds: 30
|
||||||
containers:
|
containers:
|
||||||
- name: neuron-mcp
|
- name: neuron-mcp
|
||||||
image: registry.neuralplatform.ai/neuron-technologies/neuron-mcp:v0.8.3
|
image: registry.neuralplatform.ai/neuron-technologies/neuron-mcp:v0.8.3
|
||||||
@@ -31,6 +33,8 @@ spec:
|
|||||||
env:
|
env:
|
||||||
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||||||
value: "http://alloy.monitoring.svc.cluster.local:4318"
|
value: "http://alloy.monitoring.svc.cluster.local:4318"
|
||||||
|
- name: JAVA_TOOL_OPTIONS
|
||||||
|
value: "-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0 -XX:+ExitOnOutOfMemoryError"
|
||||||
envFrom:
|
envFrom:
|
||||||
- configMapRef:
|
- configMapRef:
|
||||||
name: neuron-prod-config
|
name: neuron-prod-config
|
||||||
@@ -42,10 +46,15 @@ spec:
|
|||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: 250m
|
cpu: 250m
|
||||||
memory: 256Mi
|
memory: 512Mi
|
||||||
limits:
|
limits:
|
||||||
cpu: 1000m
|
cpu: 1500m
|
||||||
memory: 768Mi
|
memory: 1536Mi
|
||||||
|
lifecycle:
|
||||||
|
preStop:
|
||||||
|
exec:
|
||||||
|
# Give Traefik time to drain in-flight connections before pod terminates
|
||||||
|
command: ["sh", "-c", "sleep 8"]
|
||||||
livenessProbe:
|
livenessProbe:
|
||||||
httpGet:
|
httpGet:
|
||||||
path: /actuator/health/liveness
|
path: /actuator/health/liveness
|
||||||
@@ -57,7 +66,7 @@ spec:
|
|||||||
httpGet:
|
httpGet:
|
||||||
path: /actuator/health/readiness
|
path: /actuator/health/readiness
|
||||||
port: http
|
port: http
|
||||||
initialDelaySeconds: 15
|
initialDelaySeconds: 10
|
||||||
periodSeconds: 10
|
periodSeconds: 10
|
||||||
failureThreshold: 3
|
failureThreshold: 3
|
||||||
volumes:
|
volumes:
|
||||||
|
|||||||
@@ -21,9 +21,14 @@ metadata:
|
|||||||
namespace: neuron-prod
|
namespace: neuron-prod
|
||||||
labels:
|
labels:
|
||||||
app: neuron-mcp
|
app: neuron-mcp
|
||||||
|
annotations:
|
||||||
|
# Blue/green: change slot to "green" then back to "blue" to switch traffic.
|
||||||
|
# Process: deploy green → verify health → patch this selector → scale blue to 0.
|
||||||
|
neuron.ai/active-slot: "blue"
|
||||||
spec:
|
spec:
|
||||||
selector:
|
selector:
|
||||||
app: neuron-mcp
|
app: neuron-mcp
|
||||||
|
slot: blue
|
||||||
ports:
|
ports:
|
||||||
- name: http
|
- name: http
|
||||||
port: 8080
|
port: 8080
|
||||||
|
|||||||
Executable
+76
@@ -0,0 +1,76 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Blue/Green deploy for neuron-mcp
|
||||||
|
# Usage: ./blue-green-deploy.sh <new-image-tag>
|
||||||
|
# Example: ./blue-green-deploy.sh v0.8.4
|
||||||
|
#
|
||||||
|
# PVC is ReadWriteOnce — only ONE pod can hold it at a time.
|
||||||
|
# Process: scale IDLE slot to 1 (new image, it waits for PVC) →
|
||||||
|
# scale ACTIVE slot to 0 (releases PVC) → idle pod claims PVC + starts →
|
||||||
|
# verify health → switch service selector → done.
|
||||||
|
# Rollback: re-run with old tag, slots flip back.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
KUBECONFIG="${KUBECONFIG:-$HOME/.kube/legion-config}"
|
||||||
|
NS="neuron-prod"
|
||||||
|
SVC="neuron-mcp"
|
||||||
|
IMAGE_BASE="registry.neuralplatform.ai/neuron-technologies/neuron-mcp"
|
||||||
|
TAG="${1:-}"
|
||||||
|
|
||||||
|
if [[ -z "$TAG" ]]; then
|
||||||
|
echo "Usage: $0 <image-tag>"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
KC="kubectl --kubeconfig=$KUBECONFIG -n $NS"
|
||||||
|
|
||||||
|
# Determine active and idle slots
|
||||||
|
ACTIVE_SLOT=$($KC get svc $SVC -o jsonpath='{.spec.selector.slot}')
|
||||||
|
if [[ "$ACTIVE_SLOT" == "blue" ]]; then
|
||||||
|
IDLE_SLOT="green"
|
||||||
|
else
|
||||||
|
IDLE_SLOT="blue"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "→ Active slot: $ACTIVE_SLOT | Deploying to: $IDLE_SLOT | Image: $IMAGE_BASE:$TAG"
|
||||||
|
|
||||||
|
# 1. Update idle deployment with new image (still scaled to 0)
|
||||||
|
echo "→ Updating neuron-mcp-$IDLE_SLOT image..."
|
||||||
|
$KC set image deployment/neuron-mcp-$IDLE_SLOT neuron-mcp=$IMAGE_BASE:$TAG
|
||||||
|
|
||||||
|
# 2. Scale active down — releases the PVC
|
||||||
|
echo "→ Scaling neuron-mcp-$ACTIVE_SLOT to 0 (releases PVC)..."
|
||||||
|
$KC scale deployment/neuron-mcp-$ACTIVE_SLOT --replicas=0
|
||||||
|
$KC rollout status deployment/neuron-mcp-$ACTIVE_SLOT --timeout=60s || true
|
||||||
|
|
||||||
|
# 3. Scale idle up — claims PVC, starts with new image
|
||||||
|
echo "→ Scaling neuron-mcp-$IDLE_SLOT to 1..."
|
||||||
|
$KC scale deployment/neuron-mcp-$IDLE_SLOT --replicas=1
|
||||||
|
|
||||||
|
# 4. Wait for readiness (up to 3 min for JVM startup)
|
||||||
|
echo "→ Waiting for neuron-mcp-$IDLE_SLOT to be ready..."
|
||||||
|
$KC rollout status deployment/neuron-mcp-$IDLE_SLOT --timeout=180s
|
||||||
|
|
||||||
|
# 5. Health check
|
||||||
|
POD=$($KC get pod -l app=neuron-mcp,slot=$IDLE_SLOT -o jsonpath='{.items[0].metadata.name}')
|
||||||
|
HEALTH=$($KC exec "$POD" -- wget -qO- http://localhost:8080/actuator/health/readiness 2>/dev/null || echo '{"status":"DOWN"}')
|
||||||
|
STATUS=$(echo "$HEALTH" | python3 -c "import sys,json; print(json.load(sys.stdin).get('status','UNKNOWN'))" 2>/dev/null || echo "UNKNOWN")
|
||||||
|
|
||||||
|
if [[ "$STATUS" != "UP" ]]; then
|
||||||
|
echo "✗ Health check failed (status=$STATUS). Rolling back — scaling $IDLE_SLOT back to 0, $ACTIVE_SLOT back to 1."
|
||||||
|
$KC scale deployment/neuron-mcp-$IDLE_SLOT --replicas=0
|
||||||
|
$KC scale deployment/neuron-mcp-$ACTIVE_SLOT --replicas=1
|
||||||
|
$KC rollout status deployment/neuron-mcp-$ACTIVE_SLOT --timeout=180s
|
||||||
|
echo "✓ Rollback complete. Still on $ACTIVE_SLOT ($IMAGE_BASE:$(kubectl --kubeconfig=$KUBECONFIG -n $NS get deployment neuron-mcp-$ACTIVE_SLOT -o jsonpath='{.spec.template.spec.containers[0].image}' | cut -d: -f2))."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "✓ Health OK — switching service selector to $IDLE_SLOT..."
|
||||||
|
|
||||||
|
# 6. Switch service selector
|
||||||
|
$KC patch svc $SVC \
|
||||||
|
-p "{\"spec\":{\"selector\":{\"app\":\"neuron-mcp\",\"slot\":\"$IDLE_SLOT\"}},\"metadata\":{\"annotations\":{\"neuron.ai/active-slot\":\"$IDLE_SLOT\"}}}"
|
||||||
|
|
||||||
|
echo "✓ Traffic now on neuron-mcp-$IDLE_SLOT ($IMAGE_BASE:$TAG)"
|
||||||
|
echo " Previous slot neuron-mcp-$ACTIVE_SLOT is scaled to 0 (kept for instant rollback)."
|
||||||
|
echo ""
|
||||||
|
echo " To rollback: $0 <previous-tag> (will swap slots back)"
|
||||||
Reference in New Issue
Block a user