Harden prod: security, autoscaling, observability, BuildKit CI
Security: - Drop ALL capabilities, enforce non-root, RuntimeDefault seccomp on neuron-mcp, neuron-rest, neuron-marketing pods - Add startup probes (150s window for JVM) so liveness doesn't fire early - Replace docker-sock hostPath with BuildKit rootless TCP endpoint (moby/buildkit:v0.19.0-rootless) — removes node root access from CI - Document full ESO AppRole migration path in cluster-secret-store.yaml Autoscaling & availability: - HPAs on mcp (1–6), rest (1–4), marketing (2–8) at 65–70% CPU - PodDisruptionBudgets (minAvailable: 1) on all three services - NetworkPolicy: default-deny-all in neuron-prod, explicit allow rules for Traefik ingress, intra-namespace, and egress to DNS/platform/vault Observability: - ServiceMonitors for mcp, rest, marketing (cross-namespace enabled in kube-prometheus-stack with serviceMonitorSelectorNilUsesHelmValues:false) - PrometheusRules: high error rate, high latency, crash loops, replica shortage, Postgres down/connections, backup failure, backup staleness Chart version pinning: - kube-prometheus-stack, loki, tempo, redis, alloy, postgres — all pinned to major-version ranges to block silent breaking upgrades Backup hardening: - restic:latest → restic:0.17.3 (deterministic image) - Weekly backup-verify CronJob: restores latest snapshot and validates SQL dump structure (≥5 CREATE TABLE, pg_dump header check) ArgoCD: - neuron-prod AppProject: scopes deploys to neuron-prod + platform ns, blacklists ClusterRole/ClusterRoleBinding/Namespace creation, automated sync window 2–6am UTC, manual always allowed
This commit is contained in:
Will Anderson
@@ -0,0 +1,153 @@
|
||||
---
|
||||
# PrometheusRules — SLO and operational alerts for neuron-prod
|
||||
# These fire into Alertmanager which routes to #infrastructure-alerts on Slack.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: neuron-slo
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
|
||||
# ── Neuron API SLOs ──────────────────────────────────────��───────────────
|
||||
- name: neuron.slo
|
||||
interval: 60s
|
||||
rules:
|
||||
- alert: NeuronMCPHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-mcp",status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-mcp"}[5m]))
|
||||
) > 0.01
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
namespace: neuron-prod
|
||||
annotations:
|
||||
summary: "neuron-mcp 5xx error rate > 1%"
|
||||
description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes."
|
||||
|
||||
- alert: NeuronRestHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-rest",status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-rest"}[5m]))
|
||||
) > 0.01
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
namespace: neuron-prod
|
||||
annotations:
|
||||
summary: "neuron-rest 5xx error rate > 1%"
|
||||
description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes."
|
||||
|
||||
- alert: NeuronMCPHighLatency
|
||||
expr: |
|
||||
histogram_quantile(0.99,
|
||||
sum(rate(http_server_requests_seconds_bucket{namespace="neuron-prod",app="neuron-mcp"}[10m]))
|
||||
by (le)
|
||||
) > 2.0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
namespace: neuron-prod
|
||||
annotations:
|
||||
summary: "neuron-mcp p99 latency > 2s"
|
||||
description: "p99 latency is {{ $value | humanizeDuration }} over the last 10 minutes."
|
||||
|
||||
- alert: NeuronRestHighLatency
|
||||
expr: |
|
||||
histogram_quantile(0.99,
|
||||
sum(rate(http_server_requests_seconds_bucket{namespace="neuron-prod",app="neuron-rest"}[10m]))
|
||||
by (le)
|
||||
) > 2.0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
namespace: neuron-prod
|
||||
annotations:
|
||||
summary: "neuron-rest p99 latency > 2s"
|
||||
description: "p99 latency is {{ $value | humanizeDuration }} over the last 10 minutes."
|
||||
|
||||
# ── Pod health ───────────────────────────────────────────────────────────
|
||||
- name: neuron.pods
|
||||
rules:
|
||||
- alert: NeuronPodCrashLooping
|
||||
expr: |
|
||||
rate(kube_pod_container_status_restarts_total{namespace="neuron-prod"}[15m]) * 60 * 15 > 3
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
namespace: neuron-prod
|
||||
annotations:
|
||||
summary: "Pod {{ $labels.pod }} is crash looping"
|
||||
description: "Container {{ $labels.container }} restarted {{ $value | humanize }} times in 15 minutes."
|
||||
|
||||
- alert: NeuronPodNotReady
|
||||
expr: |
|
||||
sum by (pod) (kube_pod_status_ready{namespace="neuron-prod",condition="false"}) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
namespace: neuron-prod
|
||||
annotations:
|
||||
summary: "Pod {{ $labels.pod }} not ready for 5 minutes"
|
||||
|
||||
- alert: NeuronDeploymentReplicasMissing
|
||||
expr: |
|
||||
kube_deployment_status_replicas_available{namespace="neuron-prod"}
|
||||
< kube_deployment_spec_replicas{namespace="neuron-prod"}
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
namespace: neuron-prod
|
||||
annotations:
|
||||
summary: "Deployment {{ $labels.deployment }} has fewer replicas than desired"
|
||||
|
||||
# ── Postgres ─────────────────────────���─────────────────────────────────��─
|
||||
- name: postgres.health
|
||||
rules:
|
||||
- alert: PostgresDown
|
||||
expr: pg_up{namespace="platform"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Postgres is down"
|
||||
|
||||
- alert: PostgresHighConnections
|
||||
expr: |
|
||||
(pg_stat_database_numbackends{namespace="platform"} / pg_settings_max_connections{namespace="platform"})
|
||||
> 0.80
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Postgres connection pool > 80%"
|
||||
description: "Connection utilization is {{ $value | humanizePercentage }}."
|
||||
|
||||
# ── Backup health ────────────────────────────���───────────────────────────
|
||||
- name: backup.health
|
||||
rules:
|
||||
- alert: BackupJobFailed
|
||||
expr: |
|
||||
kube_job_status_failed{namespace="git",job_name=~"gitea-backup.*"} > 0
|
||||
for: 0m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Gitea backup job failed"
|
||||
description: "Check: kubectl logs -n git -l job-name=gitea-backup --previous"
|
||||
|
||||
- alert: BackupNotRunRecently
|
||||
expr: |
|
||||
(time() - kube_job_status_completion_time{namespace="git",job_name=~"gitea-backup.*"}) > 90000
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Last successful backup was more than 25 hours ago"
|
||||
@@ -0,0 +1,64 @@
|
||||
---
|
||||
# ServiceMonitors for neuron-prod — scraped by kube-prometheus-stack Prometheus
|
||||
# Requires: kube-prometheus-stack prometheusSpec.serviceMonitorSelectorNilUsesHelmValues=false
|
||||
# OR a serviceMonitorNamespaceSelector that includes neuron-prod.
|
||||
# We set this in the Prometheus stack values (see apps/kube-prometheus-stack.yaml).
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: neuron-mcp
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- neuron-prod
|
||||
selector:
|
||||
matchLabels:
|
||||
app: neuron-mcp
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /actuator/prometheus
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: neuron-rest
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- neuron-prod
|
||||
selector:
|
||||
matchLabels:
|
||||
app: neuron-rest
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /actuator/prometheus
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: neuron-marketing
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- neuron-prod
|
||||
selector:
|
||||
matchLabels:
|
||||
app: neuron-marketing
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /api/metrics
|
||||
interval: 60s
|
||||
scrapeTimeout: 15s
|
||||
Reference in New Issue
Block a user