infrastructure/servers/legion/k8s/monitoring/alerting-rules.yaml

---
# PrometheusRules — SLO and operational alerts for neuron-prod
# These fire into Alertmanager which routes to #infrastructure-alerts on Slack.
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: neuron-slo
  namespace: monitoring
  labels:
    release: kube-prometheus-stack
spec:
  groups:

    # ── Neuron API SLOs ──────────────────────────────────────��───────────────
    - name: neuron.slo
      interval: 60s
      rules:
        - alert: NeuronMCPHighErrorRate
          expr: |
            (
              sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-mcp",status=~"5.."}[5m]))
              /
              sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-mcp"}[5m]))
            ) > 0.01
          for: 2m
          labels:
            severity: critical
            namespace: neuron-prod
          annotations:
            summary: "neuron-mcp 5xx error rate > 1%"
            description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes."

        - alert: NeuronRestHighErrorRate
          expr: |
            (
              sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-rest",status=~"5.."}[5m]))
              /
              sum(rate(http_server_requests_seconds_count{namespace="neuron-prod",app="neuron-rest"}[5m]))
            ) > 0.01
          for: 2m
          labels:
            severity: critical
            namespace: neuron-prod
          annotations:
            summary: "neuron-rest 5xx error rate > 1%"
            description: "Error rate is {{ $value | humanizePercentage }} over the last 5 minutes."

        - alert: NeuronMCPHighLatency
          expr: |
            histogram_quantile(0.99,
              sum(rate(http_server_requests_seconds_bucket{namespace="neuron-prod",app="neuron-mcp"}[10m]))
              by (le)
            ) > 2.0
          for: 5m
          labels:
            severity: warning
            namespace: neuron-prod
          annotations:
            summary: "neuron-mcp p99 latency > 2s"
            description: "p99 latency is {{ $value | humanizeDuration }} over the last 10 minutes."

        - alert: NeuronRestHighLatency
          expr: |
            histogram_quantile(0.99,
              sum(rate(http_server_requests_seconds_bucket{namespace="neuron-prod",app="neuron-rest"}[10m]))
              by (le)
            ) > 2.0
          for: 5m
          labels:
            severity: warning
            namespace: neuron-prod
          annotations:
            summary: "neuron-rest p99 latency > 2s"
            description: "p99 latency is {{ $value | humanizeDuration }} over the last 10 minutes."

    # ── Pod health ───────────────────────────────────────────────────────────
    - name: neuron.pods
      rules:
        - alert: NeuronPodCrashLooping
          expr: |
            rate(kube_pod_container_status_restarts_total{namespace="neuron-prod"}[15m]) * 60 * 15 > 3
          for: 5m
          labels:
            severity: critical
            namespace: neuron-prod
          annotations:
            summary: "Pod {{ $labels.pod }} is crash looping"
            description: "Container {{ $labels.container }} restarted {{ $value | humanize }} times in 15 minutes."

        - alert: NeuronPodNotReady
          expr: |
            sum by (pod) (kube_pod_status_ready{namespace="neuron-prod",condition="false"}) > 0
          for: 5m
          labels:
            severity: warning
            namespace: neuron-prod
          annotations:
            summary: "Pod {{ $labels.pod }} not ready for 5 minutes"

        - alert: NeuronDeploymentReplicasMissing
          expr: |
            kube_deployment_status_replicas_available{namespace="neuron-prod"}
            < kube_deployment_spec_replicas{namespace="neuron-prod"}
          for: 3m
          labels:
            severity: warning
            namespace: neuron-prod
          annotations:
            summary: "Deployment {{ $labels.deployment }} has fewer replicas than desired"

    # ── Postgres ─────────────────────────���─────────────────────────────────��─
    - name: postgres.health
      rules:
        - alert: PostgresDown
          expr: pg_up{namespace="platform"} == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "Postgres is down"

        - alert: PostgresHighConnections
          expr: |
            (pg_stat_database_numbackends{namespace="platform"} / pg_settings_max_connections{namespace="platform"})
            > 0.80
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Postgres connection pool > 80%"
            description: "Connection utilization is {{ $value | humanizePercentage }}."

    # ── Backup health ────────────────────────────���───────────────────────────
    - name: backup.health
      rules:
        - alert: BackupJobFailed
          expr: |
            kube_job_status_failed{namespace="git",job_name=~"gitea-backup.*"} > 0
          for: 0m
          labels:
            severity: critical
          annotations:
            summary: "Gitea backup job failed"
            description: "Check: kubectl logs -n git -l job-name=gitea-backup --previous"

        - alert: BackupNotRunRecently
          expr: |
            (time() - kube_job_status_completion_time{namespace="git",job_name=~"gitea-backup.*"}) > 90000
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "Last successful backup was more than 25 hours ago"