Harden prod: security, autoscaling, observability, BuildKit CI

Security:
- Drop ALL capabilities, enforce non-root, RuntimeDefault seccomp on
  neuron-mcp, neuron-rest, neuron-marketing pods
- Add startup probes (150s window for JVM) so liveness doesn't fire early
- Replace docker-sock hostPath with BuildKit rootless TCP endpoint
  (moby/buildkit:v0.19.0-rootless) — removes node root access from CI
- Document full ESO AppRole migration path in cluster-secret-store.yaml

Autoscaling & availability:
- HPAs on mcp (1–6), rest (1–4), marketing (2–8) at 65–70% CPU
- PodDisruptionBudgets (minAvailable: 1) on all three services
- NetworkPolicy: default-deny-all in neuron-prod, explicit allow rules
  for Traefik ingress, intra-namespace, and egress to DNS/platform/vault

Observability:
- ServiceMonitors for mcp, rest, marketing (cross-namespace enabled in
  kube-prometheus-stack with serviceMonitorSelectorNilUsesHelmValues:false)
- PrometheusRules: high error rate, high latency, crash loops, replica
  shortage, Postgres down/connections, backup failure, backup staleness

Chart version pinning:
- kube-prometheus-stack, loki, tempo, redis, alloy, postgres — all pinned
  to major-version ranges to block silent breaking upgrades

Backup hardening:
- restic:latest → restic:0.17.3 (deterministic image)
- Weekly backup-verify CronJob: restores latest snapshot and validates
  SQL dump structure (≥5 CREATE TABLE, pg_dump header check)

ArgoCD:
- neuron-prod AppProject: scopes deploys to neuron-prod + platform ns,
  blacklists ClusterRole/ClusterRoleBinding/Namespace creation,
  automated sync window 2–6am UTC, manual always allowed

servers/legion/apps/alloy.yaml

@@ -8,7 +8,7 @@ spec:
   source:
     repoURL: https://grafana.github.io/helm-charts
     chart: alloy
-    targetRevision: "*"
+    targetRevision: ">=0.9.0, <1.0.0"
     helm:
       values: |
         controller:

servers/legion/apps/gitea-runner.yaml

@@ -1,3 +1,7 @@
+---
+# Gitea CI runner — general-purpose (legion)
+# Docker socket REMOVED. Builds go through buildkitd TCP endpoint.
+# Set DOCKER_HOST=tcp://buildkitd.ci.svc.cluster.local:1234 in build steps.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -15,8 +19,10 @@ spec:
       labels:
         app: gitea-runner
       annotations:
-        config-version: "2026-04-23-nodeport-url"
+        config-version: "2026-04-25-buildkit-no-sock"
     spec:
+      securityContext:
+        runAsNonRoot: false  # act_runner needs root for container management
       initContainers:
         - name: register
           image: registry.neuralplatform.ai/ci-base:latest
@@ -36,10 +42,9 @@ spec:
                 timeout: 3h
               container:
                 network: host
-                docker_host: "unix:///var/run/docker.sock"
+                docker_host: "tcp://buildkitd.ci.svc.cluster.local:1234"
                 force_pull: false
-                valid_volumes:
+                valid_volumes: []
-                  - /var/run/docker.sock
                 default_image: "registry.neuralplatform.ai/ci-base:latest"
                 extra_hosts:
                   - "gitea.git.svc.cluster.local:10.43.1.53"
@@ -55,14 +60,16 @@ spec:
           image: registry.neuralplatform.ai/ci-base:latest
           workingDir: /data
           command: ["act_runner", "daemon", "--config", "/data/config.yaml"]
+          env:
+            - name: DOCKER_HOST
+              value: "tcp://buildkitd.ci.svc.cluster.local:1234"
           envFrom:
             - secretRef:
                 name: gitea-runner-secret
           volumeMounts:
             - name: data
               mountPath: /data
-            - name: docker-sock
+          # docker-sock volume intentionally removed — use buildkitd TCP instead
-              mountPath: /var/run/docker.sock
           resources:
             requests:
               memory: 512Mi
@@ -73,11 +80,9 @@ spec:
       volumes:
         - name: data
           emptyDir: {}
-        - name: docker-sock
+        # docker-sock hostPath intentionally removed
-          hostPath:
-            path: /var/run/docker.sock
-            type: Socket
 ---
+# Neuron Technologies CI runner
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -95,7 +100,7 @@ spec:
       labels:
         app: neuron-technologies-runner
       annotations:
-        config-version: "2026-04-24-initial"
+        config-version: "2026-04-25-buildkit-no-sock"
     spec:
       initContainers:
         - name: register
@@ -116,10 +121,9 @@ spec:
                 timeout: 3h
               container:
                 network: host
-                docker_host: "unix:///var/run/docker.sock"
+                docker_host: "tcp://buildkitd.ci.svc.cluster.local:1234"
                 force_pull: false
-                valid_volumes:
+                valid_volumes: []
-                  - /var/run/docker.sock
                 default_image: "registry.neuralplatform.ai/ci-base:latest"
                 extra_hosts:
                   - "gitea.git.svc.cluster.local:10.43.1.53"
@@ -135,14 +139,15 @@ spec:
           image: registry.neuralplatform.ai/ci-base:latest
           workingDir: /data
           command: ["act_runner", "daemon", "--config", "/data/config.yaml"]
+          env:
+            - name: DOCKER_HOST
+              value: "tcp://buildkitd.ci.svc.cluster.local:1234"
           envFrom:
             - secretRef:
                 name: neuron-technologies-runner-secret
           volumeMounts:
             - name: data
               mountPath: /data
-            - name: docker-sock
-              mountPath: /var/run/docker.sock
           resources:
             requests:
               memory: 512Mi
@@ -153,7 +158,3 @@ spec:
       volumes:
         - name: data
           emptyDir: {}
-        - name: docker-sock
-          hostPath:
-            path: /var/run/docker.sock
-            type: Socket

servers/legion/apps/kube-prometheus-stack.yaml

@@ -8,7 +8,7 @@ spec:
   source:
     repoURL: https://prometheus-community.github.io/helm-charts
     chart: kube-prometheus-stack
-    targetRevision: "*"
+    targetRevision: ">=60.0.0, <70.0.0"
     helm:
       values: |
         grafana:
@@ -37,6 +37,13 @@ spec:
               cert-manager.io/cluster-issuer: letsencrypt-prod
         prometheus:
           prometheusSpec:
+            # Discover ServiceMonitors and PrometheusRules from ALL namespaces.
+            # Without this, Prometheus only sees resources in the monitoring namespace.
+            serviceMonitorSelectorNilUsesHelmValues: false
+            serviceMonitorNamespaceSelector: {}
+            serviceMonitorSelector: {}
+            ruleNamespaceSelector: {}
+            podMonitorNamespaceSelector: {}
             storageSpec:
               volumeClaimTemplate:
                 spec:

servers/legion/apps/loki.yaml

@@ -8,7 +8,7 @@ spec:
   source:
     repoURL: https://grafana.github.io/helm-charts
     chart: loki
-    targetRevision: "*"
+    targetRevision: ">=6.0.0, <7.0.0"
     helm:
       values: |
         loki:

servers/legion/apps/neuron-prod-project.yaml

@@ -0,0 +1,89 @@
+---
+# ArgoCD AppProject — neuron-prod
+# Restricts which repos can deploy to which namespaces.
+# Prevents misconfigured apps from deploying cluster-admin RBAC or
+# accidentally targeting personal/hobby namespaces.
+apiVersion: argoproj.io/v1alpha1
+kind: AppProject
+metadata:
+  name: neuron-prod
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  description: "Neuron Technologies production services"
+
+  # Only allow deployments sourced from the infrastructure repo.
+  sourceRepos:
+    - "http://10.43.1.53:3000/will/infrastructure.git"
+    - "https://code.forgejo.org"
+    - "registry.neuralplatform.ai"
+
+  # Only allow deploying to neuron-prod and platform namespaces.
+  destinations:
+    - server: https://kubernetes.default.svc
+      namespace: neuron-prod
+    - server: https://kubernetes.default.svc
+      namespace: platform
+
+  # Cluster-scoped resources this project may NOT create.
+  clusterResourceBlacklist:
+    - group: rbac.authorization.k8s.io
+      kind: ClusterRole
+    - group: rbac.authorization.k8s.io
+      kind: ClusterRoleBinding
+    - group: ""
+      kind: Namespace
+
+  # Namespace-scoped resources this project may create.
+  namespaceResourceWhitelist:
+    - group: "apps"
+      kind: Deployment
+    - group: "apps"
+      kind: ReplicaSet
+    - group: "batch"
+      kind: CronJob
+    - group: "batch"
+      kind: Job
+    - group: ""
+      kind: Service
+    - group: ""
+      kind: ConfigMap
+    - group: ""
+      kind: Secret
+    - group: ""
+      kind: PersistentVolumeClaim
+    - group: ""
+      kind: ServiceAccount
+    - group: "networking.k8s.io"
+      kind: Ingress
+    - group: "networking.k8s.io"
+      kind: NetworkPolicy
+    - group: "policy"
+      kind: PodDisruptionBudget
+    - group: "autoscaling"
+      kind: HorizontalPodAutoscaler
+    - group: "traefik.io"
+      kind: IngressRoute
+    - group: "external-secrets.io"
+      kind: ExternalSecret
+    - group: "monitoring.coreos.com"
+      kind: ServiceMonitor
+    - group: "monitoring.coreos.com"
+      kind: PrometheusRule
+
+  # Sync windows: only allow automated syncs during low-traffic hours.
+  # Manual syncs allowed any time.
+  syncWindows:
+    - kind: allow
+      schedule: "0 2-6 * * *"   # 2am-6am UTC (off-peak)
+      duration: 4h
+      applications:
+        - "*"
+      manualSync: true
+    - kind: allow
+      schedule: "* * * * *"
+      duration: "1m"
+      manualSync: true           # Always allow manual deploys
+      applications:
+        - "*"

servers/legion/apps/postgres.yaml

@@ -9,7 +9,7 @@ spec:
     # Helm chart from Docker Hub OCI registry
     - repoURL: registry-1.docker.io/bitnamicharts
       chart: postgresql
-      targetRevision: "*"
+      targetRevision: ">=16.0.0, <17.0.0"
       helm:
         values: |
           auth:

servers/legion/apps/redis.yaml

@@ -8,7 +8,7 @@ spec:
   source:
     repoURL: registry-1.docker.io/bitnamicharts
     chart: redis
-    targetRevision: "*"
+    targetRevision: ">=20.0.0, <21.0.0"
     helm:
       values: |
         auth:

servers/legion/apps/tempo.yaml

@@ -8,7 +8,7 @@ spec:
   source:
     repoURL: https://grafana.github.io/helm-charts
     chart: tempo
-    targetRevision: "*"
+    targetRevision: ">=1.0.0, <2.0.0"
     helm:
       values: |
         tempo:

servers/legion/k8s/backup/cronjob.yaml

@@ -43,8 +43,10 @@ spec:
                 limits:
                   memory: 256Mi
           containers:
+            # Pin to explicit version — never use latest for backup tooling.
+            # Bump this intentionally after testing; do not let it float.
             - name: backup
-              image: restic/restic:latest
+              image: restic/restic:0.17.3
               command:
                 - /bin/sh
                 - -c
@@ -93,3 +95,77 @@ spec:
                 readOnly: true
             - name: dump
               emptyDir: {}
+---
+# Weekly restore-verify job — proves the backup is actually restorable.
+# Runs Sunday 4am. Restores latest snapshot to /tmp, validates the SQL dump.
+# Alerts via Alertmanager if it fails (BackupJobFailed PrometheusRule watches git namespace).
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: backup-verify
+  namespace: git
+spec:
+  schedule: "0 4 * * 0"
+  concurrencyPolicy: Forbid
+  failedJobsHistoryLimit: 2
+  successfulJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      activeDeadlineSeconds: 1800  # 30 min max
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+            - name: verify
+              image: restic/restic:0.17.3
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  set -e
+
+                  echo "=== Backup Restore Verification ==="
+                  echo "Listing latest snapshots..."
+                  restic snapshots --latest 3
+
+                  echo "Restoring latest snapshot to /tmp/restore-verify..."
+                  mkdir -p /tmp/restore-verify
+                  restic restore latest \
+                    --target /tmp/restore-verify \
+                    --include /all-databases.sql
+
+                  SQL_FILE="/tmp/restore-verify/all-databases.sql"
+                  if [ ! -f "$SQL_FILE" ]; then
+                    echo "ERROR: SQL dump not found in restored snapshot"
+                    exit 1
+                  fi
+
+                  SIZE=$(du -sh "$SQL_FILE" | cut -f1)
+                  echo "SQL dump restored: $SIZE"
+
+                  # Basic structural validation
+                  if ! grep -q "PostgreSQL database cluster dump" "$SQL_FILE"; then
+                    echo "ERROR: SQL dump does not look like a valid pg_dumpall output"
+                    exit 1
+                  fi
+
+                  TABLE_COUNT=$(grep -c "^CREATE TABLE" "$SQL_FILE" || true)
+                  echo "Tables found in dump: $TABLE_COUNT"
+
+                  if [ "$TABLE_COUNT" -lt 5 ]; then
+                    echo "ERROR: Too few tables in dump (expected >= 5, got $TABLE_COUNT)"
+                    exit 1
+                  fi
+
+                  echo "=== Verification PASSED: backup is restorable ==="
+                  rm -rf /tmp/restore-verify
+              envFrom:
+                - secretRef:
+                    name: backup-credentials
+              resources:
+                requests:
+                  memory: 256Mi
+                  cpu: 100m
+                limits:
+                  memory: 512Mi

servers/legion/k8s/external-secrets/cluster-secret-store.yaml

@@ -1,11 +1,47 @@
 ---
-# ClusterSecretStore — connects ESO to Vault using the root token Secret.
+# ClusterSecretStore — connects ESO to Vault
-# The vault-token Secret (name: vault-token, key: token) must be created
-# via kubectl as a one-time bootstrap step before this store can sync:
 #
-#   kubectl create secret generic vault-token \
+# CURRENT STATE: token auth (root token). This works but the root token
-#     --namespace external-secrets \
+# has no expiry and full Vault access — it's a single credential that
-#     --from-literal=token=$(cat ~/Secrets/tokens/vault-root-token)
+# compromises everything if leaked.
+#
+# MIGRATION TO APPROLE (do this before GCP migration):
+# ─────────────────────────────────────────────────────
+# 1. Enable AppRole on Vault (if not already):
+#      vault auth enable approle
+#
+# 2. Create a policy scoped to only the paths ESO reads:
+#      vault policy write eso-read - <<EOF
+#      path "secret/data/neuron-technologies/*" { capabilities = ["read"] }
+#      path "secret/data/r2"                    { capabilities = ["read"] }
+#      path "secret/data/slack"                 { capabilities = ["read"] }
+#      path "secret/data/legion-db"             { capabilities = ["read"] }
+#      EOF
+#
+# 3. Create the AppRole:
+#      vault write auth/approle/role/eso \
+#        token_policies="eso-read" \
+#        token_ttl=1h \
+#        token_max_ttl=4h \
+#        secret_id_ttl=0  # no expiry for the secretId
+#
+# 4. Get the roleId and secretId:
+#      vault read auth/approle/role/eso/role-id    → ROLE_ID
+#      vault write -f auth/approle/role/eso/secret-id → SECRET_ID
+#
+# 5. Create the k8s secret (one-time bootstrap, same as the root token was):
+#      kubectl create secret generic vault-approle \
+#        --namespace external-secrets \
+#        --from-literal=roleId=<ROLE_ID> \
+#        --from-literal=secretId=<SECRET_ID>
+#
+# 6. Flip this file to the appRole stanza below, commit, push.
+#    ESO will hot-reload when the manifest changes.
+#
+# 7. Delete the old vault-token secret:
+#      kubectl delete secret vault-token -n external-secrets
+# ─────────────────────────────────────────────────────
+
 apiVersion: external-secrets.io/v1beta1
 kind: ClusterSecretStore
 metadata:
@@ -17,7 +53,17 @@ spec:
       path: "secret"
       version: "v2"
       auth:
+        # ── Current: root token auth ──────────────────────────────────────────
+        # TODO: replace with appRole stanza once AppRole is bootstrapped (see above).
         tokenSecretRef:
           name: vault-token
           namespace: external-secrets
           key: token
+        # ── Target: AppRole auth (uncomment after bootstrap) ──────────────────
+        # appRole:
+        #   path: "approle"
+        #   roleId: "eso"
+        #   secretRef:
+        #     name: vault-approle
+        #     namespace: external-secrets
+        #     key: secretId

servers/legion/k8s/gitea-runner/buildkitd.yaml

@@ -0,0 +1,71 @@
+---
+# BuildKit daemon — rootless image builder for CI runners.
+# Replaces the /var/run/docker.sock hostPath mount that gave containers
+# full root access to the Legion host node.
+#
+# Runners connect via TCP: DOCKER_HOST=tcp://buildkitd.ci.svc.cluster.local:1234
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: buildkitd
+  namespace: ci
+  labels:
+    app: buildkitd
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: buildkitd
+  template:
+    metadata:
+      labels:
+        app: buildkitd
+    spec:
+      containers:
+        - name: buildkitd
+          image: moby/buildkit:v0.19.0-rootless
+          args:
+            - --oci-worker-no-process-sandbox
+            - --addr
+            - tcp://0.0.0.0:1234
+            - --addr
+            - unix:///run/buildkit/buildkitd.sock
+          ports:
+            - name: tcp
+              containerPort: 1234
+          securityContext:
+            seccompProfile:
+              type: Unconfined       # BuildKit rootless requires this
+            runAsUser: 1000
+            runAsGroup: 1000
+          readinessProbe:
+            exec:
+              command: ["buildctl", "debug", "workers"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          resources:
+            requests:
+              cpu: 200m
+              memory: 256Mi
+            limits:
+              cpu: "4"
+              memory: 4Gi
+          volumeMounts:
+            - name: buildkit-storage
+              mountPath: /home/user/.local/share/buildkit
+      volumes:
+        - name: buildkit-storage
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: buildkitd
+  namespace: ci
+spec:
+  selector:
+    app: buildkitd
+  ports:
+    - name: tcp
+      port: 1234
+      targetPort: 1234

servers/legion/k8s/monitoring/alerting-rules.yaml

@@ -0,0 +1,153 @@
+---
+# PrometheusRules — SLO and operational alerts for neuron-prod
+# These fire into Alertmanager which routes to #infrastructure-alerts on Slack.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: neuron-slo
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  groups:
+
+    # ── Neuron API SLOs ──────────────────────────────────────��───────────────
+    - name: neuron.slo
+      interval: 60s
+      rules:
+        - alert: NeuronMCPHighErrorRate
+          expr: |
+            (
+              sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-mcp",status=~"5.."}[5m]))
+              /
+              sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-mcp"}[5m]))
+            ) > 0.01
+          for: 2m
+          labels:
+            severity: critical
+            namespace: neuron-prod
+          annotations:
+            summary: "neuron-mcp 5xx error rate > 1%"
+            description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes."
+
+        - alert: NeuronRestHighErrorRate
+          expr: |
+            (
+              sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-rest",status=~"5.."}[5m]))
+              /
+              sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-rest"}[5m]))
+            ) > 0.01
+          for: 2m
+          labels:
+            severity: critical
+            namespace: neuron-prod
+          annotations:
+            summary: "neuron-rest 5xx error rate > 1%"
+            description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes."
+
+        - alert: NeuronMCPHighLatency
+          expr: |
+            histogram_quantile(0.99,
+              sum(rate(http_server_requests_seconds_bucket{namespace="neuron-prod",app="neuron-mcp"}[10m]))
+              by (le)
+            ) > 2.0
+          for: 5m
+          labels:
+            severity: warning
+            namespace: neuron-prod
+          annotations:
+            summary: "neuron-mcp p99 latency > 2s"
+            description: "p99 latency is {{ $value | humanizeDuration }} over the last 10 minutes."
+
+        - alert: NeuronRestHighLatency
+          expr: |
+            histogram_quantile(0.99,
+              sum(rate(http_server_requests_seconds_bucket{namespace="neuron-prod",app="neuron-rest"}[10m]))
+              by (le)
+            ) > 2.0
+          for: 5m
+          labels:
+            severity: warning
+            namespace: neuron-prod
+          annotations:
+            summary: "neuron-rest p99 latency > 2s"
+            description: "p99 latency is {{ $value | humanizeDuration }} over the last 10 minutes."
+
+    # ── Pod health ───────────────────────────────────────────────────────────
+    - name: neuron.pods
+      rules:
+        - alert: NeuronPodCrashLooping
+          expr: |
+            rate(kube_pod_container_status_restarts_total{namespace="neuron-prod"}[15m]) * 60 * 15 > 3
+          for: 5m
+          labels:
+            severity: critical
+            namespace: neuron-prod
+          annotations:
+            summary: "Pod {{ $labels.pod }} is crash looping"
+            description: "Container {{ $labels.container }} restarted {{ $value | humanize }} times in 15 minutes."
+
+        - alert: NeuronPodNotReady
+          expr: |
+            sum by (pod) (kube_pod_status_ready{namespace="neuron-prod",condition="false"}) > 0
+          for: 5m
+          labels:
+            severity: warning
+            namespace: neuron-prod
+          annotations:
+            summary: "Pod {{ $labels.pod }} not ready for 5 minutes"
+
+        - alert: NeuronDeploymentReplicasMissing
+          expr: |
+            kube_deployment_status_replicas_available{namespace="neuron-prod"}
+            < kube_deployment_spec_replicas{namespace="neuron-prod"}
+          for: 3m
+          labels:
+            severity: warning
+            namespace: neuron-prod
+          annotations:
+            summary: "Deployment {{ $labels.deployment }} has fewer replicas than desired"
+
+    # ── Postgres ─────────────────────────���─────────────────────────────────��─
+    - name: postgres.health
+      rules:
+        - alert: PostgresDown
+          expr: pg_up{namespace="platform"} == 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Postgres is down"
+
+        - alert: PostgresHighConnections
+          expr: |
+            (pg_stat_database_numbackends{namespace="platform"} / pg_settings_max_connections{namespace="platform"})
+            > 0.80
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Postgres connection pool > 80%"
+            description: "Connection utilization is {{ $value | humanizePercentage }}."
+
+    # ── Backup health ────────────────────────────���───────────────────────────
+    - name: backup.health
+      rules:
+        - alert: BackupJobFailed
+          expr: |
+            kube_job_status_failed{namespace="git",job_name=~"gitea-backup.*"} > 0
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Gitea backup job failed"
+            description: "Check: kubectl logs -n git -l job-name=gitea-backup --previous"
+
+        - alert: BackupNotRunRecently
+          expr: |
+            (time() - kube_job_status_completion_time{namespace="git",job_name=~"gitea-backup.*"}) > 90000
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Last successful backup was more than 25 hours ago"

servers/legion/k8s/monitoring/servicemonitor.yaml

@@ -0,0 +1,64 @@
+---
+# ServiceMonitors for neuron-prod — scraped by kube-prometheus-stack Prometheus
+# Requires: kube-prometheus-stack prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false
+# OR a serviceMonitorNamespaceSelector that includes neuron-prod.
+# We set this in the Prometheus stack values (see apps/kube-prometheus-stack.yaml).
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: neuron-mcp
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  namespaceSelector:
+    matchNames:
+      - neuron-prod
+  selector:
+    matchLabels:
+      app: neuron-mcp
+  endpoints:
+    - port: http
+      path: /actuator/prometheus
+      interval: 30s
+      scrapeTimeout: 10s
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: neuron-rest
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  namespaceSelector:
+    matchNames:
+      - neuron-prod
+  selector:
+    matchLabels:
+      app: neuron-rest
+  endpoints:
+    - port: http
+      path: /actuator/prometheus
+      interval: 30s
+      scrapeTimeout: 10s
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: neuron-marketing
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  namespaceSelector:
+    matchNames:
+      - neuron-prod
+  selector:
+    matchLabels:
+      app: neuron-marketing
+  endpoints:
+    - port: http
+      path: /api/metrics
+      interval: 60s
+      scrapeTimeout: 15s

servers/legion/k8s/neuron-technologies/marketing/deployment.yaml

@@ -17,6 +17,11 @@ spec:
         app: neuron-marketing
         env: prod
     spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1001
+        seccompProfile:
+          type: RuntimeDefault
       containers:
         - name: neuron-marketing
           image: registry.neuralplatform.ai/neuron-technologies/marketing:1e94e8ae
@@ -24,6 +29,12 @@ spec:
           ports:
             - name: http
               containerPort: 3000
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: false  # Next.js writes .next/cache at runtime
+            runAsNonRoot: true
+            capabilities:
+              drop: ["ALL"]
           env:
             - name: NODE_ENV
               value: production

servers/legion/k8s/neuron-technologies/prod/hpa.yaml

@@ -0,0 +1,98 @@
+---
+# HorizontalPodAutoscalers for neuron-prod
+# Single Legion node today — minReplicas=1 prevents unnecessary idle waste.
+# When GCP GKE is live these minimums should go to 2 for HA.
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: neuron-mcp-hpa
+  namespace: neuron-prod
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: neuron-mcp-blue
+  minReplicas: 1
+  maxReplicas: 6
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 65
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: 80
+  behavior:
+    scaleDown:
+      stabilizationWindowSeconds: 300  # Don't thrash — wait 5m before scaling down
+      policies:
+        - type: Pods
+          value: 1
+          periodSeconds: 120
+    scaleUp:
+      stabilizationWindowSeconds: 30
+      policies:
+        - type: Pods
+          value: 2
+          periodSeconds: 60
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: neuron-rest-hpa
+  namespace: neuron-prod
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: neuron-rest
+  minReplicas: 1
+  maxReplicas: 4
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 65
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: 80
+  behavior:
+    scaleDown:
+      stabilizationWindowSeconds: 300
+    scaleUp:
+      stabilizationWindowSeconds: 30
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: neuron-marketing-hpa
+  namespace: neuron-prod
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: neuron-marketing
+  minReplicas: 2
+  maxReplicas: 8
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 70
+  behavior:
+    scaleDown:
+      stabilizationWindowSeconds: 120
+    scaleUp:
+      stabilizationWindowSeconds: 15

servers/legion/k8s/neuron-technologies/prod/kustomization.yaml

@@ -15,3 +15,7 @@ resources:
   - backup-cronjob.yaml
   - runpod-inference.yaml
   - runpod-lb-configmap.yaml
+  # Hardening additions
+  - hpa.yaml
+  - pdb.yaml
+  - network-policy.yaml

servers/legion/k8s/neuron-technologies/prod/mcp-deployment.yaml

@@ -25,6 +25,10 @@ spec:
         kubectl.kubernetes.io/restartedAt: "2026-04-25T08:40:00Z"
     spec:
       terminationGracePeriodSeconds: 30
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
       containers:
         - name: neuron-mcp
           image: registry.neuralplatform.ai/neuron-technologies/neuron-mcp:v0.15.3
@@ -32,6 +36,12 @@ spec:
           ports:
             - name: http
               containerPort: 8080
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: false  # JVM writes temp files; restrict paths via allowedPaths once profiled
+            runAsNonRoot: true
+            capabilities:
+              drop: ["ALL"]
           env:
             - name: OTEL_EXPORTER_OTLP_ENDPOINT
               value: "http://alloy-otlp.monitoring.svc.cluster.local:4318"
@@ -57,18 +67,25 @@ spec:
               exec:
                 # Give Traefik time to drain in-flight connections before pod terminates
                 command: ["sh", "-c", "sleep 8"]
+          # startupProbe gives the JVM up to 150s to start before liveness takes over.
+          # Prevents false crash loops during slow cold starts.
+          startupProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: http
+            failureThreshold: 30
+            periodSeconds: 5
           livenessProbe:
             httpGet:
               path: /actuator/health/liveness
               port: http
-            initialDelaySeconds: 30
             periodSeconds: 30
             failureThreshold: 3
           readinessProbe:
             httpGet:
               path: /actuator/health/readiness
               port: http
-            initialDelaySeconds: 10
+            initialDelaySeconds: 5
             periodSeconds: 10
             failureThreshold: 3
       volumes:

servers/legion/k8s/neuron-technologies/prod/network-policy.yaml

@@ -0,0 +1,124 @@
+---
+# NetworkPolicies for neuron-prod
+# Default-deny all ingress and egress, then explicitly allow only what's needed.
+# Enforced by k3s's built-in NetworkPolicy controller.
+
+# ── Baseline: deny all traffic in namespace ───────────────────────────────────
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: default-deny-all
+  namespace: neuron-prod
+spec:
+  podSelector: {}
+  policyTypes:
+    - Ingress
+    - Egress
+---
+# ── neuron-mcp: accept traffic from Traefik (kube-system) only ───────────────
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-mcp-ingress
+  namespace: neuron-prod
+spec:
+  podSelector:
+    matchLabels:
+      app: neuron-mcp
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: neuron-prod
+---
+# ── neuron-rest: accept from mcp, kube-system (Traefik), monitoring (Alloy) ──
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-rest-ingress
+  namespace: neuron-prod
+spec:
+  podSelector:
+    matchLabels:
+      app: neuron-rest
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: neuron-prod
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: monitoring
+---
+# ── neuron-marketing: accept from Traefik only ───────────────────────────────
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-marketing-ingress
+  namespace: neuron-prod
+spec:
+  podSelector:
+    matchLabels:
+      app: neuron-marketing
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+---
+# ── Egress: all prod pods may reach platform (postgres/redis), vault,
+#            monitoring (alloy OTLP), kube-dns, and the internet (external APIs) ─
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-prod-egress
+  namespace: neuron-prod
+spec:
+  podSelector: {}
+  policyTypes:
+    - Egress
+  egress:
+    # Kubernetes DNS
+    - ports:
+        - port: 53
+          protocol: UDP
+        - port: 53
+          protocol: TCP
+    # platform namespace (Postgres, Redis, accounts service)
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: platform
+    # vault namespace
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: vault
+    # monitoring namespace (Alloy OTLP)
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: monitoring
+    # intra-namespace (mcp ↔ rest ↔ license)
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: neuron-prod
+    # external internet (Stripe, OAuth providers, Cloudflare, etc.)
+    - ports:
+        - port: 443
+          protocol: TCP
+        - port: 80
+          protocol: TCP

servers/legion/k8s/neuron-technologies/prod/pdb.yaml

@@ -0,0 +1,39 @@
+---
+# PodDisruptionBudgets for neuron-prod
+# Ensures at least 1 replica of each service survives node drains and upgrades.
+# With replicas=1 today, minAvailable=1 means drain will block until HPA scales up
+# to 2+ — this is intentional and provides a natural safety gate.
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: neuron-mcp-pdb
+  namespace: neuron-prod
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: neuron-mcp
+      slot: blue
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: neuron-rest-pdb
+  namespace: neuron-prod
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: neuron-rest
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: neuron-marketing-pdb
+  namespace: neuron-prod
+spec:
+  # Marketing runs 2 replicas — keep at least 1 alive during drains.
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: neuron-marketing

servers/legion/k8s/neuron-technologies/prod/rest-deployment.yaml

@@ -19,6 +19,10 @@ spec:
       annotations:
         kubectl.kubernetes.io/restartedAt: "2026-04-24T10:00:00Z"
     spec:
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
       containers:
         - name: neuron-rest
           image: registry.neuralplatform.ai/neuron-technologies/neuron-rest:v0.15.3
@@ -26,6 +30,12 @@ spec:
           ports:
             - name: http
               containerPort: 8081
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+            capabilities:
+              drop: ["ALL"]
           envFrom:
             - configMapRef:
                 name: neuron-prod-config
@@ -41,18 +51,23 @@ spec:
             limits:
               cpu: 1000m
               memory: 1Gi
+          startupProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: http
+            failureThreshold: 30
+            periodSeconds: 5
           livenessProbe:
             httpGet:
               path: /actuator/health/liveness
               port: http
-            initialDelaySeconds: 30
             periodSeconds: 30
             failureThreshold: 3
           readinessProbe:
             httpGet:
               path: /actuator/health/readiness
               port: http
-            initialDelaySeconds: 15
+            initialDelaySeconds: 5
             periodSeconds: 10
             failureThreshold: 3
       volumes: