diff --git a/servers/legion/apps/alloy.yaml b/servers/legion/apps/alloy.yaml index 77dbfc0..245b88d 100644 --- a/servers/legion/apps/alloy.yaml +++ b/servers/legion/apps/alloy.yaml @@ -8,7 +8,7 @@ spec: source: repoURL: https://grafana.github.io/helm-charts chart: alloy - targetRevision: "*" + targetRevision: ">=0.9.0, <1.0.0" helm: values: | controller: diff --git a/servers/legion/apps/gitea-runner.yaml b/servers/legion/apps/gitea-runner.yaml index 2902f60..29ff958 100644 --- a/servers/legion/apps/gitea-runner.yaml +++ b/servers/legion/apps/gitea-runner.yaml @@ -1,3 +1,7 @@ +--- +# Gitea CI runner — general-purpose (legion) +# Docker socket REMOVED. Builds go through buildkitd TCP endpoint. +# Set DOCKER_HOST=tcp://buildkitd.ci.svc.cluster.local:1234 in build steps. apiVersion: apps/v1 kind: Deployment metadata: @@ -15,8 +19,10 @@ spec: labels: app: gitea-runner annotations: - config-version: "2026-04-23-nodeport-url" + config-version: "2026-04-25-buildkit-no-sock" spec: + securityContext: + runAsNonRoot: false # act_runner needs root for container management initContainers: - name: register image: registry.neuralplatform.ai/ci-base:latest @@ -36,10 +42,9 @@ spec: timeout: 3h container: network: host - docker_host: "unix:///var/run/docker.sock" + docker_host: "tcp://buildkitd.ci.svc.cluster.local:1234" force_pull: false - valid_volumes: - - /var/run/docker.sock + valid_volumes: [] default_image: "registry.neuralplatform.ai/ci-base:latest" extra_hosts: - "gitea.git.svc.cluster.local:10.43.1.53" @@ -55,14 +60,16 @@ spec: image: registry.neuralplatform.ai/ci-base:latest workingDir: /data command: ["act_runner", "daemon", "--config", "/data/config.yaml"] + env: + - name: DOCKER_HOST + value: "tcp://buildkitd.ci.svc.cluster.local:1234" envFrom: - secretRef: name: gitea-runner-secret volumeMounts: - name: data mountPath: /data - - name: docker-sock - mountPath: /var/run/docker.sock + # docker-sock volume intentionally removed — use buildkitd TCP instead resources: requests: memory: 512Mi @@ -73,11 +80,9 @@ spec: volumes: - name: data emptyDir: {} - - name: docker-sock - hostPath: - path: /var/run/docker.sock - type: Socket + # docker-sock hostPath intentionally removed --- +# Neuron Technologies CI runner apiVersion: apps/v1 kind: Deployment metadata: @@ -95,7 +100,7 @@ spec: labels: app: neuron-technologies-runner annotations: - config-version: "2026-04-24-initial" + config-version: "2026-04-25-buildkit-no-sock" spec: initContainers: - name: register @@ -116,10 +121,9 @@ spec: timeout: 3h container: network: host - docker_host: "unix:///var/run/docker.sock" + docker_host: "tcp://buildkitd.ci.svc.cluster.local:1234" force_pull: false - valid_volumes: - - /var/run/docker.sock + valid_volumes: [] default_image: "registry.neuralplatform.ai/ci-base:latest" extra_hosts: - "gitea.git.svc.cluster.local:10.43.1.53" @@ -135,14 +139,15 @@ spec: image: registry.neuralplatform.ai/ci-base:latest workingDir: /data command: ["act_runner", "daemon", "--config", "/data/config.yaml"] + env: + - name: DOCKER_HOST + value: "tcp://buildkitd.ci.svc.cluster.local:1234" envFrom: - secretRef: name: neuron-technologies-runner-secret volumeMounts: - name: data mountPath: /data - - name: docker-sock - mountPath: /var/run/docker.sock resources: requests: memory: 512Mi @@ -153,7 +158,3 @@ spec: volumes: - name: data emptyDir: {} - - name: docker-sock - hostPath: - path: /var/run/docker.sock - type: Socket diff --git a/servers/legion/apps/kube-prometheus-stack.yaml b/servers/legion/apps/kube-prometheus-stack.yaml index b3ff096..a584459 100644 --- a/servers/legion/apps/kube-prometheus-stack.yaml +++ b/servers/legion/apps/kube-prometheus-stack.yaml @@ -8,7 +8,7 @@ spec: source: repoURL: https://prometheus-community.github.io/helm-charts chart: kube-prometheus-stack - targetRevision: "*" + targetRevision: ">=60.0.0, <70.0.0" helm: values: | grafana: @@ -37,6 +37,13 @@ spec: cert-manager.io/cluster-issuer: letsencrypt-prod prometheus: prometheusSpec: + # Discover ServiceMonitors and PrometheusRules from ALL namespaces. + # Without this, Prometheus only sees resources in the monitoring namespace. + serviceMonitorSelectorNilUsesHelmValues: false + serviceMonitorNamespaceSelector: {} + serviceMonitorSelector: {} + ruleNamespaceSelector: {} + podMonitorNamespaceSelector: {} storageSpec: volumeClaimTemplate: spec: diff --git a/servers/legion/apps/loki.yaml b/servers/legion/apps/loki.yaml index 9105f7b..e9891ad 100644 --- a/servers/legion/apps/loki.yaml +++ b/servers/legion/apps/loki.yaml @@ -8,7 +8,7 @@ spec: source: repoURL: https://grafana.github.io/helm-charts chart: loki - targetRevision: "*" + targetRevision: ">=6.0.0, <7.0.0" helm: values: | loki: diff --git a/servers/legion/apps/neuron-prod-project.yaml b/servers/legion/apps/neuron-prod-project.yaml new file mode 100644 index 0000000..ec3e501 --- /dev/null +++ b/servers/legion/apps/neuron-prod-project.yaml @@ -0,0 +1,89 @@ +--- +# ArgoCD AppProject — neuron-prod +# Restricts which repos can deploy to which namespaces. +# Prevents misconfigured apps from deploying cluster-admin RBAC or +# accidentally targeting personal/hobby namespaces. +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: neuron-prod + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + description: "Neuron Technologies production services" + + # Only allow deployments sourced from the infrastructure repo. + sourceRepos: + - "http://10.43.1.53:3000/will/infrastructure.git" + - "https://code.forgejo.org" + - "registry.neuralplatform.ai" + + # Only allow deploying to neuron-prod and platform namespaces. + destinations: + - server: https://kubernetes.default.svc + namespace: neuron-prod + - server: https://kubernetes.default.svc + namespace: platform + + # Cluster-scoped resources this project may NOT create. + clusterResourceBlacklist: + - group: rbac.authorization.k8s.io + kind: ClusterRole + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + - group: "" + kind: Namespace + + # Namespace-scoped resources this project may create. + namespaceResourceWhitelist: + - group: "apps" + kind: Deployment + - group: "apps" + kind: ReplicaSet + - group: "batch" + kind: CronJob + - group: "batch" + kind: Job + - group: "" + kind: Service + - group: "" + kind: ConfigMap + - group: "" + kind: Secret + - group: "" + kind: PersistentVolumeClaim + - group: "" + kind: ServiceAccount + - group: "networking.k8s.io" + kind: Ingress + - group: "networking.k8s.io" + kind: NetworkPolicy + - group: "policy" + kind: PodDisruptionBudget + - group: "autoscaling" + kind: HorizontalPodAutoscaler + - group: "traefik.io" + kind: IngressRoute + - group: "external-secrets.io" + kind: ExternalSecret + - group: "monitoring.coreos.com" + kind: ServiceMonitor + - group: "monitoring.coreos.com" + kind: PrometheusRule + + # Sync windows: only allow automated syncs during low-traffic hours. + # Manual syncs allowed any time. + syncWindows: + - kind: allow + schedule: "0 2-6 * * *" # 2am-6am UTC (off-peak) + duration: 4h + applications: + - "*" + manualSync: true + - kind: allow + schedule: "* * * * *" + duration: "1m" + manualSync: true # Always allow manual deploys + applications: + - "*" diff --git a/servers/legion/apps/postgres.yaml b/servers/legion/apps/postgres.yaml index 2480f96..04ed5fd 100644 --- a/servers/legion/apps/postgres.yaml +++ b/servers/legion/apps/postgres.yaml @@ -9,7 +9,7 @@ spec: # Helm chart from Docker Hub OCI registry - repoURL: registry-1.docker.io/bitnamicharts chart: postgresql - targetRevision: "*" + targetRevision: ">=16.0.0, <17.0.0" helm: values: | auth: diff --git a/servers/legion/apps/redis.yaml b/servers/legion/apps/redis.yaml index d56c5e9..6d91a50 100644 --- a/servers/legion/apps/redis.yaml +++ b/servers/legion/apps/redis.yaml @@ -8,7 +8,7 @@ spec: source: repoURL: registry-1.docker.io/bitnamicharts chart: redis - targetRevision: "*" + targetRevision: ">=20.0.0, <21.0.0" helm: values: | auth: diff --git a/servers/legion/apps/tempo.yaml b/servers/legion/apps/tempo.yaml index 6c86318..dc35229 100644 --- a/servers/legion/apps/tempo.yaml +++ b/servers/legion/apps/tempo.yaml @@ -8,7 +8,7 @@ spec: source: repoURL: https://grafana.github.io/helm-charts chart: tempo - targetRevision: "*" + targetRevision: ">=1.0.0, <2.0.0" helm: values: | tempo: diff --git a/servers/legion/k8s/backup/cronjob.yaml b/servers/legion/k8s/backup/cronjob.yaml index 283d943..9a5367e 100644 --- a/servers/legion/k8s/backup/cronjob.yaml +++ b/servers/legion/k8s/backup/cronjob.yaml @@ -43,8 +43,10 @@ spec: limits: memory: 256Mi containers: + # Pin to explicit version — never use latest for backup tooling. + # Bump this intentionally after testing; do not let it float. - name: backup - image: restic/restic:latest + image: restic/restic:0.17.3 command: - /bin/sh - -c @@ -93,3 +95,77 @@ spec: readOnly: true - name: dump emptyDir: {} +--- +# Weekly restore-verify job — proves the backup is actually restorable. +# Runs Sunday 4am. Restores latest snapshot to /tmp, validates the SQL dump. +# Alerts via Alertmanager if it fails (BackupJobFailed PrometheusRule watches git namespace). +apiVersion: batch/v1 +kind: CronJob +metadata: + name: backup-verify + namespace: git +spec: + schedule: "0 4 * * 0" + concurrencyPolicy: Forbid + failedJobsHistoryLimit: 2 + successfulJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 1800 # 30 min max + template: + spec: + restartPolicy: Never + containers: + - name: verify + image: restic/restic:0.17.3 + command: + - /bin/sh + - -c + - | + set -e + + echo "=== Backup Restore Verification ===" + echo "Listing latest snapshots..." + restic snapshots --latest 3 + + echo "Restoring latest snapshot to /tmp/restore-verify..." + mkdir -p /tmp/restore-verify + restic restore latest \ + --target /tmp/restore-verify \ + --include /all-databases.sql + + SQL_FILE="/tmp/restore-verify/all-databases.sql" + if [ ! -f "$SQL_FILE" ]; then + echo "ERROR: SQL dump not found in restored snapshot" + exit 1 + fi + + SIZE=$(du -sh "$SQL_FILE" | cut -f1) + echo "SQL dump restored: $SIZE" + + # Basic structural validation + if ! grep -q "PostgreSQL database cluster dump" "$SQL_FILE"; then + echo "ERROR: SQL dump does not look like a valid pg_dumpall output" + exit 1 + fi + + TABLE_COUNT=$(grep -c "^CREATE TABLE" "$SQL_FILE" || true) + echo "Tables found in dump: $TABLE_COUNT" + + if [ "$TABLE_COUNT" -lt 5 ]; then + echo "ERROR: Too few tables in dump (expected >= 5, got $TABLE_COUNT)" + exit 1 + fi + + echo "=== Verification PASSED: backup is restorable ===" + rm -rf /tmp/restore-verify + envFrom: + - secretRef: + name: backup-credentials + resources: + requests: + memory: 256Mi + cpu: 100m + limits: + memory: 512Mi diff --git a/servers/legion/k8s/external-secrets/cluster-secret-store.yaml b/servers/legion/k8s/external-secrets/cluster-secret-store.yaml index 4c16fdf..5047b62 100644 --- a/servers/legion/k8s/external-secrets/cluster-secret-store.yaml +++ b/servers/legion/k8s/external-secrets/cluster-secret-store.yaml @@ -1,11 +1,47 @@ --- -# ClusterSecretStore — connects ESO to Vault using the root token Secret. -# The vault-token Secret (name: vault-token, key: token) must be created -# via kubectl as a one-time bootstrap step before this store can sync: +# ClusterSecretStore — connects ESO to Vault # -# kubectl create secret generic vault-token \ -# --namespace external-secrets \ -# --from-literal=token=$(cat ~/Secrets/tokens/vault-root-token) +# CURRENT STATE: token auth (root token). This works but the root token +# has no expiry and full Vault access — it's a single credential that +# compromises everything if leaked. +# +# MIGRATION TO APPROLE (do this before GCP migration): +# ───────────────────────────────────────────────────── +# 1. Enable AppRole on Vault (if not already): +# vault auth enable approle +# +# 2. Create a policy scoped to only the paths ESO reads: +# vault policy write eso-read - < \ +# --from-literal=secretId= +# +# 6. Flip this file to the appRole stanza below, commit, push. +# ESO will hot-reload when the manifest changes. +# +# 7. Delete the old vault-token secret: +# kubectl delete secret vault-token -n external-secrets +# ───────────────────────────────────────────────────── + apiVersion: external-secrets.io/v1beta1 kind: ClusterSecretStore metadata: @@ -17,7 +53,17 @@ spec: path: "secret" version: "v2" auth: + # ── Current: root token auth ────────────────────────────────────────── + # TODO: replace with appRole stanza once AppRole is bootstrapped (see above). tokenSecretRef: name: vault-token namespace: external-secrets key: token + # ── Target: AppRole auth (uncomment after bootstrap) ────────────────── + # appRole: + # path: "approle" + # roleId: "eso" + # secretRef: + # name: vault-approle + # namespace: external-secrets + # key: secretId diff --git a/servers/legion/k8s/gitea-runner/buildkitd.yaml b/servers/legion/k8s/gitea-runner/buildkitd.yaml new file mode 100644 index 0000000..7abb889 --- /dev/null +++ b/servers/legion/k8s/gitea-runner/buildkitd.yaml @@ -0,0 +1,71 @@ +--- +# BuildKit daemon — rootless image builder for CI runners. +# Replaces the /var/run/docker.sock hostPath mount that gave containers +# full root access to the Legion host node. +# +# Runners connect via TCP: DOCKER_HOST=tcp://buildkitd.ci.svc.cluster.local:1234 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: buildkitd + namespace: ci + labels: + app: buildkitd +spec: + replicas: 1 + selector: + matchLabels: + app: buildkitd + template: + metadata: + labels: + app: buildkitd + spec: + containers: + - name: buildkitd + image: moby/buildkit:v0.19.0-rootless + args: + - --oci-worker-no-process-sandbox + - --addr + - tcp://0.0.0.0:1234 + - --addr + - unix:///run/buildkit/buildkitd.sock + ports: + - name: tcp + containerPort: 1234 + securityContext: + seccompProfile: + type: Unconfined # BuildKit rootless requires this + runAsUser: 1000 + runAsGroup: 1000 + readinessProbe: + exec: + command: ["buildctl", "debug", "workers"] + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: "4" + memory: 4Gi + volumeMounts: + - name: buildkit-storage + mountPath: /home/user/.local/share/buildkit + volumes: + - name: buildkit-storage + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: buildkitd + namespace: ci +spec: + selector: + app: buildkitd + ports: + - name: tcp + port: 1234 + targetPort: 1234 diff --git a/servers/legion/k8s/monitoring/alerting-rules.yaml b/servers/legion/k8s/monitoring/alerting-rules.yaml new file mode 100644 index 0000000..96382fd --- /dev/null +++ b/servers/legion/k8s/monitoring/alerting-rules.yaml @@ -0,0 +1,153 @@ +--- +# PrometheusRules — SLO and operational alerts for neuron-prod +# These fire into Alertmanager which routes to #infrastructure-alerts on Slack. +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: neuron-slo + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + groups: + + # ── Neuron API SLOs ──────────────────────────────────────��─────────────── + - name: neuron.slo + interval: 60s + rules: + - alert: NeuronMCPHighErrorRate + expr: | + ( + sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-mcp",status=~"5.."}[5m])) + / + sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-mcp"}[5m])) + ) > 0.01 + for: 2m + labels: + severity: critical + namespace: neuron-prod + annotations: + summary: "neuron-mcp 5xx error rate > 1%" + description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes." + + - alert: NeuronRestHighErrorRate + expr: | + ( + sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-rest",status=~"5.."}[5m])) + / + sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-rest"}[5m])) + ) > 0.01 + for: 2m + labels: + severity: critical + namespace: neuron-prod + annotations: + summary: "neuron-rest 5xx error rate > 1%" + description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes." + + - alert: NeuronMCPHighLatency + expr: | + histogram_quantile(0.99, + sum(rate(http_server_requests_seconds_bucket{namespace="neuron-prod",app="neuron-mcp"}[10m])) + by (le) + ) > 2.0 + for: 5m + labels: + severity: warning + namespace: neuron-prod + annotations: + summary: "neuron-mcp p99 latency > 2s" + description: "p99 latency is {{ $value | humanizeDuration }} over the last 10 minutes." + + - alert: NeuronRestHighLatency + expr: | + histogram_quantile(0.99, + sum(rate(http_server_requests_seconds_bucket{namespace="neuron-prod",app="neuron-rest"}[10m])) + by (le) + ) > 2.0 + for: 5m + labels: + severity: warning + namespace: neuron-prod + annotations: + summary: "neuron-rest p99 latency > 2s" + description: "p99 latency is {{ $value | humanizeDuration }} over the last 10 minutes." + + # ── Pod health ─────────────────────────────────────────────────────────── + - name: neuron.pods + rules: + - alert: NeuronPodCrashLooping + expr: | + rate(kube_pod_container_status_restarts_total{namespace="neuron-prod"}[15m]) * 60 * 15 > 3 + for: 5m + labels: + severity: critical + namespace: neuron-prod + annotations: + summary: "Pod {{ $labels.pod }} is crash looping" + description: "Container {{ $labels.container }} restarted {{ $value | humanize }} times in 15 minutes." + + - alert: NeuronPodNotReady + expr: | + sum by (pod) (kube_pod_status_ready{namespace="neuron-prod",condition="false"}) > 0 + for: 5m + labels: + severity: warning + namespace: neuron-prod + annotations: + summary: "Pod {{ $labels.pod }} not ready for 5 minutes" + + - alert: NeuronDeploymentReplicasMissing + expr: | + kube_deployment_status_replicas_available{namespace="neuron-prod"} + < kube_deployment_spec_replicas{namespace="neuron-prod"} + for: 3m + labels: + severity: warning + namespace: neuron-prod + annotations: + summary: "Deployment {{ $labels.deployment }} has fewer replicas than desired" + + # ── Postgres ─────────────────────────���─────────────────────────────────��─ + - name: postgres.health + rules: + - alert: PostgresDown + expr: pg_up{namespace="platform"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Postgres is down" + + - alert: PostgresHighConnections + expr: | + (pg_stat_database_numbackends{namespace="platform"} / pg_settings_max_connections{namespace="platform"}) + > 0.80 + for: 5m + labels: + severity: warning + annotations: + summary: "Postgres connection pool > 80%" + description: "Connection utilization is {{ $value | humanizePercentage }}." + + # ── Backup health ────────────────────────────���─────────────────────────── + - name: backup.health + rules: + - alert: BackupJobFailed + expr: | + kube_job_status_failed{namespace="git",job_name=~"gitea-backup.*"} > 0 + for: 0m + labels: + severity: critical + annotations: + summary: "Gitea backup job failed" + description: "Check: kubectl logs -n git -l job-name=gitea-backup --previous" + + - alert: BackupNotRunRecently + expr: | + (time() - kube_job_status_completion_time{namespace="git",job_name=~"gitea-backup.*"}) > 90000 + for: 0m + labels: + severity: warning + annotations: + summary: "Last successful backup was more than 25 hours ago" diff --git a/servers/legion/k8s/monitoring/servicemonitor.yaml b/servers/legion/k8s/monitoring/servicemonitor.yaml new file mode 100644 index 0000000..2ac9c2d --- /dev/null +++ b/servers/legion/k8s/monitoring/servicemonitor.yaml @@ -0,0 +1,64 @@ +--- +# ServiceMonitors for neuron-prod — scraped by kube-prometheus-stack Prometheus +# Requires: kube-prometheus-stack prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false +# OR a serviceMonitorNamespaceSelector that includes neuron-prod. +# We set this in the Prometheus stack values (see apps/kube-prometheus-stack.yaml). +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: neuron-mcp + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - neuron-prod + selector: + matchLabels: + app: neuron-mcp + endpoints: + - port: http + path: /actuator/prometheus + interval: 30s + scrapeTimeout: 10s +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: neuron-rest + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - neuron-prod + selector: + matchLabels: + app: neuron-rest + endpoints: + - port: http + path: /actuator/prometheus + interval: 30s + scrapeTimeout: 10s +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: neuron-marketing + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - neuron-prod + selector: + matchLabels: + app: neuron-marketing + endpoints: + - port: http + path: /api/metrics + interval: 60s + scrapeTimeout: 15s diff --git a/servers/legion/k8s/neuron-technologies/marketing/deployment.yaml b/servers/legion/k8s/neuron-technologies/marketing/deployment.yaml index 1c7d775..53a973c 100644 --- a/servers/legion/k8s/neuron-technologies/marketing/deployment.yaml +++ b/servers/legion/k8s/neuron-technologies/marketing/deployment.yaml @@ -17,6 +17,11 @@ spec: app: neuron-marketing env: prod spec: + securityContext: + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault containers: - name: neuron-marketing image: registry.neuralplatform.ai/neuron-technologies/marketing:1e94e8ae @@ -24,6 +29,12 @@ spec: ports: - name: http containerPort: 3000 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false # Next.js writes .next/cache at runtime + runAsNonRoot: true + capabilities: + drop: ["ALL"] env: - name: NODE_ENV value: production diff --git a/servers/legion/k8s/neuron-technologies/prod/hpa.yaml b/servers/legion/k8s/neuron-technologies/prod/hpa.yaml new file mode 100644 index 0000000..b947c72 --- /dev/null +++ b/servers/legion/k8s/neuron-technologies/prod/hpa.yaml @@ -0,0 +1,98 @@ +--- +# HorizontalPodAutoscalers for neuron-prod +# Single Legion node today — minReplicas=1 prevents unnecessary idle waste. +# When GCP GKE is live these minimums should go to 2 for HA. +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: neuron-mcp-hpa + namespace: neuron-prod +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: neuron-mcp-blue + minReplicas: 1 + maxReplicas: 6 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 65 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 # Don't thrash — wait 5m before scaling down + policies: + - type: Pods + value: 1 + periodSeconds: 120 + scaleUp: + stabilizationWindowSeconds: 30 + policies: + - type: Pods + value: 2 + periodSeconds: 60 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: neuron-rest-hpa + namespace: neuron-prod +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: neuron-rest + minReplicas: 1 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 65 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + scaleUp: + stabilizationWindowSeconds: 30 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: neuron-marketing-hpa + namespace: neuron-prod +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: neuron-marketing + minReplicas: 2 + maxReplicas: 8 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + behavior: + scaleDown: + stabilizationWindowSeconds: 120 + scaleUp: + stabilizationWindowSeconds: 15 diff --git a/servers/legion/k8s/neuron-technologies/prod/kustomization.yaml b/servers/legion/k8s/neuron-technologies/prod/kustomization.yaml index 3d1ae8b..5c73cd9 100644 --- a/servers/legion/k8s/neuron-technologies/prod/kustomization.yaml +++ b/servers/legion/k8s/neuron-technologies/prod/kustomization.yaml @@ -15,3 +15,7 @@ resources: - backup-cronjob.yaml - runpod-inference.yaml - runpod-lb-configmap.yaml + # Hardening additions + - hpa.yaml + - pdb.yaml + - network-policy.yaml diff --git a/servers/legion/k8s/neuron-technologies/prod/mcp-deployment.yaml b/servers/legion/k8s/neuron-technologies/prod/mcp-deployment.yaml index 9215924..05769ef 100644 --- a/servers/legion/k8s/neuron-technologies/prod/mcp-deployment.yaml +++ b/servers/legion/k8s/neuron-technologies/prod/mcp-deployment.yaml @@ -25,6 +25,10 @@ spec: kubectl.kubernetes.io/restartedAt: "2026-04-25T08:40:00Z" spec: terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault containers: - name: neuron-mcp image: registry.neuralplatform.ai/neuron-technologies/neuron-mcp:v0.15.3 @@ -32,6 +36,12 @@ spec: ports: - name: http containerPort: 8080 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false # JVM writes temp files; restrict paths via allowedPaths once profiled + runAsNonRoot: true + capabilities: + drop: ["ALL"] env: - name: OTEL_EXPORTER_OTLP_ENDPOINT value: "http://alloy-otlp.monitoring.svc.cluster.local:4318" @@ -57,18 +67,25 @@ spec: exec: # Give Traefik time to drain in-flight connections before pod terminates command: ["sh", "-c", "sleep 8"] + # startupProbe gives the JVM up to 150s to start before liveness takes over. + # Prevents false crash loops during slow cold starts. + startupProbe: + httpGet: + path: /actuator/health/liveness + port: http + failureThreshold: 30 + periodSeconds: 5 livenessProbe: httpGet: path: /actuator/health/liveness port: http - initialDelaySeconds: 30 periodSeconds: 30 failureThreshold: 3 readinessProbe: httpGet: path: /actuator/health/readiness port: http - initialDelaySeconds: 10 + initialDelaySeconds: 5 periodSeconds: 10 failureThreshold: 3 volumes: diff --git a/servers/legion/k8s/neuron-technologies/prod/network-policy.yaml b/servers/legion/k8s/neuron-technologies/prod/network-policy.yaml new file mode 100644 index 0000000..7f9dec9 --- /dev/null +++ b/servers/legion/k8s/neuron-technologies/prod/network-policy.yaml @@ -0,0 +1,124 @@ +--- +# NetworkPolicies for neuron-prod +# Default-deny all ingress and egress, then explicitly allow only what's needed. +# Enforced by k3s's built-in NetworkPolicy controller. + +# ── Baseline: deny all traffic in namespace ─────────────────────────────────── +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-all + namespace: neuron-prod +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress +--- +# ── neuron-mcp: accept traffic from Traefik (kube-system) only ─────────────── +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-mcp-ingress + namespace: neuron-prod +spec: + podSelector: + matchLabels: + app: neuron-mcp + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: neuron-prod +--- +# ── neuron-rest: accept from mcp, kube-system (Traefik), monitoring (Alloy) ── +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-rest-ingress + namespace: neuron-prod +spec: + podSelector: + matchLabels: + app: neuron-rest + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: neuron-prod + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring +--- +# ── neuron-marketing: accept from Traefik only ─────────────────────────────── +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-marketing-ingress + namespace: neuron-prod +spec: + podSelector: + matchLabels: + app: neuron-marketing + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system +--- +# ── Egress: all prod pods may reach platform (postgres/redis), vault, +# monitoring (alloy OTLP), kube-dns, and the internet (external APIs) ─ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-prod-egress + namespace: neuron-prod +spec: + podSelector: {} + policyTypes: + - Egress + egress: + # Kubernetes DNS + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # platform namespace (Postgres, Redis, accounts service) + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: platform + # vault namespace + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: vault + # monitoring namespace (Alloy OTLP) + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + # intra-namespace (mcp ↔ rest ↔ license) + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: neuron-prod + # external internet (Stripe, OAuth providers, Cloudflare, etc.) + - ports: + - port: 443 + protocol: TCP + - port: 80 + protocol: TCP diff --git a/servers/legion/k8s/neuron-technologies/prod/pdb.yaml b/servers/legion/k8s/neuron-technologies/prod/pdb.yaml new file mode 100644 index 0000000..a83438f --- /dev/null +++ b/servers/legion/k8s/neuron-technologies/prod/pdb.yaml @@ -0,0 +1,39 @@ +--- +# PodDisruptionBudgets for neuron-prod +# Ensures at least 1 replica of each service survives node drains and upgrades. +# With replicas=1 today, minAvailable=1 means drain will block until HPA scales up +# to 2+ — this is intentional and provides a natural safety gate. +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: neuron-mcp-pdb + namespace: neuron-prod +spec: + minAvailable: 1 + selector: + matchLabels: + app: neuron-mcp + slot: blue +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: neuron-rest-pdb + namespace: neuron-prod +spec: + minAvailable: 1 + selector: + matchLabels: + app: neuron-rest +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: neuron-marketing-pdb + namespace: neuron-prod +spec: + # Marketing runs 2 replicas — keep at least 1 alive during drains. + minAvailable: 1 + selector: + matchLabels: + app: neuron-marketing diff --git a/servers/legion/k8s/neuron-technologies/prod/rest-deployment.yaml b/servers/legion/k8s/neuron-technologies/prod/rest-deployment.yaml index 59c3508..4cdf765 100644 --- a/servers/legion/k8s/neuron-technologies/prod/rest-deployment.yaml +++ b/servers/legion/k8s/neuron-technologies/prod/rest-deployment.yaml @@ -19,6 +19,10 @@ spec: annotations: kubectl.kubernetes.io/restartedAt: "2026-04-24T10:00:00Z" spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault containers: - name: neuron-rest image: registry.neuralplatform.ai/neuron-technologies/neuron-rest:v0.15.3 @@ -26,6 +30,12 @@ spec: ports: - name: http containerPort: 8081 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: true + capabilities: + drop: ["ALL"] envFrom: - configMapRef: name: neuron-prod-config @@ -41,18 +51,23 @@ spec: limits: cpu: 1000m memory: 1Gi + startupProbe: + httpGet: + path: /actuator/health/liveness + port: http + failureThreshold: 30 + periodSeconds: 5 livenessProbe: httpGet: path: /actuator/health/liveness port: http - initialDelaySeconds: 30 periodSeconds: 30 failureThreshold: 3 readinessProbe: httpGet: path: /actuator/health/readiness port: http - initialDelaySeconds: 15 + initialDelaySeconds: 5 periodSeconds: 10 failureThreshold: 3 volumes: