fix(neuron-mcp): blue/green deployment, memory limits, preStop drain hook

- Rename Deployment to neuron-mcp-blue with slot=blue label
- Add neuron-mcp-green Deployment scaffold (replicas=0, ready to receive new image)
- Service selector now routes by slot label; annotation tracks active slot
- Memory: 768Mi → 1536Mi limit, 256Mi → 512Mi request (fixes OOM pod restarts / 502s)
- Add JAVA_TOOL_OPTIONS: container-aware GC + ExitOnOutOfMemoryError
- Add preStop sleep 8s so Traefik drains in-flight connections before pod terminates
- terminationGracePeriodSeconds: 30
- Add scripts/blue-green-deploy.sh: automated slot-swap with health check and rollback
This commit is contained in:
Will Anderson
2026-04-24 22:08:12 -05:00
parent 5d4cc17829
commit f86ee04467
4 changed files with 173 additions and 7 deletions
@@ -0,0 +1,76 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: neuron-mcp-green
namespace: neuron-prod
labels:
app: neuron-mcp
slot: green
env: prod
spec:
# Scaled to 0 when inactive. Deploy process scales to 1, verifies health, then switches service.
replicas: 0
strategy:
type: Recreate
selector:
matchLabels:
app: neuron-mcp
slot: green
template:
metadata:
labels:
app: neuron-mcp
slot: green
env: prod
spec:
terminationGracePeriodSeconds: 30
containers:
- name: neuron-mcp
# IMAGE UPDATED HERE before scaling to 1
image: registry.neuralplatform.ai/neuron-technologies/neuron-mcp:v0.8.3
imagePullPolicy: Always
ports:
- name: http
containerPort: 8080
env:
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://alloy.monitoring.svc.cluster.local:4318"
- name: JAVA_TOOL_OPTIONS
value: "-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0 -XX:+ExitOnOutOfMemoryError"
envFrom:
- configMapRef:
name: neuron-prod-config
- secretRef:
name: neuron-prod-secrets
volumeMounts:
- name: data
mountPath: /data
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: 1500m
memory: 1536Mi
lifecycle:
preStop:
exec:
command: ["sh", "-c", "sleep 8"]
livenessProbe:
httpGet:
path: /actuator/health/liveness
port: http
initialDelaySeconds: 30
periodSeconds: 30
failureThreshold: 3
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: http
initialDelaySeconds: 10
periodSeconds: 10
failureThreshold: 3
volumes:
- name: data
persistentVolumeClaim:
claimName: neuron-prod-data
@@ -1,10 +1,11 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: neuron-mcp
name: neuron-mcp-blue
namespace: neuron-prod
labels:
app: neuron-mcp
slot: blue
env: prod
spec:
replicas: 1
@@ -13,14 +14,15 @@ spec:
selector:
matchLabels:
app: neuron-mcp
slot: blue
template:
metadata:
labels:
app: neuron-mcp
slot: blue
env: prod
annotations:
kubectl.kubernetes.io/restartedAt: "2026-04-24T10:00:00Z"
spec:
terminationGracePeriodSeconds: 30
containers:
- name: neuron-mcp
image: registry.neuralplatform.ai/neuron-technologies/neuron-mcp:v0.8.3
@@ -31,6 +33,8 @@ spec:
env:
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://alloy.monitoring.svc.cluster.local:4318"
- name: JAVA_TOOL_OPTIONS
value: "-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0 -XX:+ExitOnOutOfMemoryError"
envFrom:
- configMapRef:
name: neuron-prod-config
@@ -42,10 +46,15 @@ spec:
resources:
requests:
cpu: 250m
memory: 256Mi
memory: 512Mi
limits:
cpu: 1000m
memory: 768Mi
cpu: 1500m
memory: 1536Mi
lifecycle:
preStop:
exec:
# Give Traefik time to drain in-flight connections before pod terminates
command: ["sh", "-c", "sleep 8"]
livenessProbe:
httpGet:
path: /actuator/health/liveness
@@ -57,7 +66,7 @@ spec:
httpGet:
path: /actuator/health/readiness
port: http
initialDelaySeconds: 15
initialDelaySeconds: 10
periodSeconds: 10
failureThreshold: 3
volumes:
@@ -21,9 +21,14 @@ metadata:
namespace: neuron-prod
labels:
app: neuron-mcp
annotations:
# Blue/green: change slot to "green" then back to "blue" to switch traffic.
# Process: deploy green → verify health → patch this selector → scale blue to 0.
neuron.ai/active-slot: "blue"
spec:
selector:
app: neuron-mcp
slot: blue
ports:
- name: http
port: 8080