diff --git a/servers/legion/k8s/neuron-technologies/prod/mcp-deployment-green.yaml b/servers/legion/k8s/neuron-technologies/prod/mcp-deployment-green.yaml new file mode 100644 index 0000000..f3a682b --- /dev/null +++ b/servers/legion/k8s/neuron-technologies/prod/mcp-deployment-green.yaml @@ -0,0 +1,76 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: neuron-mcp-green + namespace: neuron-prod + labels: + app: neuron-mcp + slot: green + env: prod +spec: + # Scaled to 0 when inactive. Deploy process scales to 1, verifies health, then switches service. + replicas: 0 + strategy: + type: Recreate + selector: + matchLabels: + app: neuron-mcp + slot: green + template: + metadata: + labels: + app: neuron-mcp + slot: green + env: prod + spec: + terminationGracePeriodSeconds: 30 + containers: + - name: neuron-mcp + # IMAGE UPDATED HERE before scaling to 1 + image: registry.neuralplatform.ai/neuron-technologies/neuron-mcp:v0.8.3 + imagePullPolicy: Always + ports: + - name: http + containerPort: 8080 + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://alloy.monitoring.svc.cluster.local:4318" + - name: JAVA_TOOL_OPTIONS + value: "-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0 -XX:+ExitOnOutOfMemoryError" + envFrom: + - configMapRef: + name: neuron-prod-config + - secretRef: + name: neuron-prod-secrets + volumeMounts: + - name: data + mountPath: /data + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1500m + memory: 1536Mi + lifecycle: + preStop: + exec: + command: ["sh", "-c", "sleep 8"] + livenessProbe: + httpGet: + path: /actuator/health/liveness + port: http + initialDelaySeconds: 30 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /actuator/health/readiness + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + volumes: + - name: data + persistentVolumeClaim: + claimName: neuron-prod-data diff --git a/servers/legion/k8s/neuron-technologies/prod/mcp-deployment.yaml b/servers/legion/k8s/neuron-technologies/prod/mcp-deployment.yaml index 235c601..891938b 100644 --- a/servers/legion/k8s/neuron-technologies/prod/mcp-deployment.yaml +++ b/servers/legion/k8s/neuron-technologies/prod/mcp-deployment.yaml @@ -1,10 +1,11 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: neuron-mcp + name: neuron-mcp-blue namespace: neuron-prod labels: app: neuron-mcp + slot: blue env: prod spec: replicas: 1 @@ -13,14 +14,15 @@ spec: selector: matchLabels: app: neuron-mcp + slot: blue template: metadata: labels: app: neuron-mcp + slot: blue env: prod - annotations: - kubectl.kubernetes.io/restartedAt: "2026-04-24T10:00:00Z" spec: + terminationGracePeriodSeconds: 30 containers: - name: neuron-mcp image: registry.neuralplatform.ai/neuron-technologies/neuron-mcp:v0.8.3 @@ -31,6 +33,8 @@ spec: env: - name: OTEL_EXPORTER_OTLP_ENDPOINT value: "http://alloy.monitoring.svc.cluster.local:4318" + - name: JAVA_TOOL_OPTIONS + value: "-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0 -XX:+ExitOnOutOfMemoryError" envFrom: - configMapRef: name: neuron-prod-config @@ -42,10 +46,15 @@ spec: resources: requests: cpu: 250m - memory: 256Mi + memory: 512Mi limits: - cpu: 1000m - memory: 768Mi + cpu: 1500m + memory: 1536Mi + lifecycle: + preStop: + exec: + # Give Traefik time to drain in-flight connections before pod terminates + command: ["sh", "-c", "sleep 8"] livenessProbe: httpGet: path: /actuator/health/liveness @@ -57,7 +66,7 @@ spec: httpGet: path: /actuator/health/readiness port: http - initialDelaySeconds: 15 + initialDelaySeconds: 10 periodSeconds: 10 failureThreshold: 3 volumes: diff --git a/servers/legion/k8s/neuron-technologies/prod/services.yaml b/servers/legion/k8s/neuron-technologies/prod/services.yaml index e983751..2671a64 100644 --- a/servers/legion/k8s/neuron-technologies/prod/services.yaml +++ b/servers/legion/k8s/neuron-technologies/prod/services.yaml @@ -21,9 +21,14 @@ metadata: namespace: neuron-prod labels: app: neuron-mcp + annotations: + # Blue/green: change slot to "green" then back to "blue" to switch traffic. + # Process: deploy green → verify health → patch this selector → scale blue to 0. + neuron.ai/active-slot: "blue" spec: selector: app: neuron-mcp + slot: blue ports: - name: http port: 8080 diff --git a/servers/legion/scripts/blue-green-deploy.sh b/servers/legion/scripts/blue-green-deploy.sh new file mode 100755 index 0000000..37e84f6 --- /dev/null +++ b/servers/legion/scripts/blue-green-deploy.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Blue/Green deploy for neuron-mcp +# Usage: ./blue-green-deploy.sh +# Example: ./blue-green-deploy.sh v0.8.4 +# +# PVC is ReadWriteOnce — only ONE pod can hold it at a time. +# Process: scale IDLE slot to 1 (new image, it waits for PVC) → +# scale ACTIVE slot to 0 (releases PVC) → idle pod claims PVC + starts → +# verify health → switch service selector → done. +# Rollback: re-run with old tag, slots flip back. +set -euo pipefail + +KUBECONFIG="${KUBECONFIG:-$HOME/.kube/legion-config}" +NS="neuron-prod" +SVC="neuron-mcp" +IMAGE_BASE="registry.neuralplatform.ai/neuron-technologies/neuron-mcp" +TAG="${1:-}" + +if [[ -z "$TAG" ]]; then + echo "Usage: $0 " + exit 1 +fi + +KC="kubectl --kubeconfig=$KUBECONFIG -n $NS" + +# Determine active and idle slots +ACTIVE_SLOT=$($KC get svc $SVC -o jsonpath='{.spec.selector.slot}') +if [[ "$ACTIVE_SLOT" == "blue" ]]; then + IDLE_SLOT="green" +else + IDLE_SLOT="blue" +fi + +echo "→ Active slot: $ACTIVE_SLOT | Deploying to: $IDLE_SLOT | Image: $IMAGE_BASE:$TAG" + +# 1. Update idle deployment with new image (still scaled to 0) +echo "→ Updating neuron-mcp-$IDLE_SLOT image..." +$KC set image deployment/neuron-mcp-$IDLE_SLOT neuron-mcp=$IMAGE_BASE:$TAG + +# 2. Scale active down — releases the PVC +echo "→ Scaling neuron-mcp-$ACTIVE_SLOT to 0 (releases PVC)..." +$KC scale deployment/neuron-mcp-$ACTIVE_SLOT --replicas=0 +$KC rollout status deployment/neuron-mcp-$ACTIVE_SLOT --timeout=60s || true + +# 3. Scale idle up — claims PVC, starts with new image +echo "→ Scaling neuron-mcp-$IDLE_SLOT to 1..." +$KC scale deployment/neuron-mcp-$IDLE_SLOT --replicas=1 + +# 4. Wait for readiness (up to 3 min for JVM startup) +echo "→ Waiting for neuron-mcp-$IDLE_SLOT to be ready..." +$KC rollout status deployment/neuron-mcp-$IDLE_SLOT --timeout=180s + +# 5. Health check +POD=$($KC get pod -l app=neuron-mcp,slot=$IDLE_SLOT -o jsonpath='{.items[0].metadata.name}') +HEALTH=$($KC exec "$POD" -- wget -qO- http://localhost:8080/actuator/health/readiness 2>/dev/null || echo '{"status":"DOWN"}') +STATUS=$(echo "$HEALTH" | python3 -c "import sys,json; print(json.load(sys.stdin).get('status','UNKNOWN'))" 2>/dev/null || echo "UNKNOWN") + +if [[ "$STATUS" != "UP" ]]; then + echo "✗ Health check failed (status=$STATUS). Rolling back — scaling $IDLE_SLOT back to 0, $ACTIVE_SLOT back to 1." + $KC scale deployment/neuron-mcp-$IDLE_SLOT --replicas=0 + $KC scale deployment/neuron-mcp-$ACTIVE_SLOT --replicas=1 + $KC rollout status deployment/neuron-mcp-$ACTIVE_SLOT --timeout=180s + echo "✓ Rollback complete. Still on $ACTIVE_SLOT ($IMAGE_BASE:$(kubectl --kubeconfig=$KUBECONFIG -n $NS get deployment neuron-mcp-$ACTIVE_SLOT -o jsonpath='{.spec.template.spec.containers[0].image}' | cut -d: -f2))." + exit 1 +fi + +echo "✓ Health OK — switching service selector to $IDLE_SLOT..." + +# 6. Switch service selector +$KC patch svc $SVC \ + -p "{\"spec\":{\"selector\":{\"app\":\"neuron-mcp\",\"slot\":\"$IDLE_SLOT\"}},\"metadata\":{\"annotations\":{\"neuron.ai/active-slot\":\"$IDLE_SLOT\"}}}" + +echo "✓ Traffic now on neuron-mcp-$IDLE_SLOT ($IMAGE_BASE:$TAG)" +echo " Previous slot neuron-mcp-$ACTIVE_SLOT is scaled to 0 (kept for instant rollback)." +echo "" +echo " To rollback: $0 (will swap slots back)"