From 1c8d7cecd84250845562344d507f20a9ecc11c57 Mon Sep 17 00:00:00 2001
From: Will Anderson <andersonwilliam85@gmail.com>
Date: Tue, 24 Mar 2026 01:45:06 -0500
Subject: [PATCH] monitoring: wire Alertmanager to #infrastructure-alerts in
 Slack

Routes all warning/critical alerts to the Neural Platform Slack workspace.
Suppresses Watchdog and info-level noise. Groups by namespace+alertname,
repeats every 4h. Uses bot token auth via chat.postMessage API.
---
 servers/legion/monitoring.tf | 45 ++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/servers/legion/monitoring.tf b/servers/legion/monitoring.tf
index 53b171e..500e7c1 100644
--- a/servers/legion/monitoring.tf
+++ b/servers/legion/monitoring.tf
@@ -96,6 +96,51 @@ resource "helm_release" "kube_prometheus_stack" {
     value = "false"
   }
 
+  # Alertmanager — route alerts to #infrastructure-alerts in Slack
+  values = [<<-EOT
+    alertmanager:
+      config:
+        global:
+          resolve_timeout: 5m
+        inhibit_rules:
+          - source_matchers: [severity="critical"]
+            target_matchers: [severity=~"warning|info"]
+            equal: [namespace, alertname]
+          - source_matchers: [severity="warning"]
+            target_matchers: [severity="info"]
+            equal: [namespace, alertname]
+        receivers:
+          - name: "null"
+          - name: slack-alerts
+            slack_configs:
+              - channel: "#infrastructure-alerts"
+                api_url: "https://slack.com/api/chat.postMessage"
+                http_config:
+                  authorization:
+                    type: Bearer
+                    credentials: "${var.slack_bot_token}"
+                send_resolved: true
+                title: '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
+                text: |
+                  {{ range .Alerts }}
+                  *{{ if .Annotations.summary }}{{ .Annotations.summary }}{{ else }}{{ .Labels.alertname }}{{ end }}*{{ if .Labels.severity }} · {{ .Labels.severity }}{{ end }}
+                  {{ if .Annotations.description }}{{ .Annotations.description }}{{ end }}
+                  {{ end }}
+                footer: "Legion · neuralplatform.ai"
+        route:
+          group_by: [namespace, alertname]
+          group_wait: 30s
+          group_interval: 5m
+          repeat_interval: 4h
+          receiver: slack-alerts
+          routes:
+            - matchers: [alertname="Watchdog"]
+              receiver: "null"
+            - matchers: [severity="info"]
+              receiver: "null"
+  EOT
+  ]
+
   depends_on = [helm_release.cert_manager]
 }