diff --git a/servers/legion/monitoring.tf b/servers/legion/monitoring.tf index 53b171e..500e7c1 100644 --- a/servers/legion/monitoring.tf +++ b/servers/legion/monitoring.tf @@ -96,6 +96,51 @@ resource "helm_release" "kube_prometheus_stack" { value = "false" } + # Alertmanager — route alerts to #infrastructure-alerts in Slack + values = [<<-EOT + alertmanager: + config: + global: + resolve_timeout: 5m + inhibit_rules: + - source_matchers: [severity="critical"] + target_matchers: [severity=~"warning|info"] + equal: [namespace, alertname] + - source_matchers: [severity="warning"] + target_matchers: [severity="info"] + equal: [namespace, alertname] + receivers: + - name: "null" + - name: slack-alerts + slack_configs: + - channel: "#infrastructure-alerts" + api_url: "https://slack.com/api/chat.postMessage" + http_config: + authorization: + type: Bearer + credentials: "${var.slack_bot_token}" + send_resolved: true + title: '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})' + text: | + {{ range .Alerts }} + *{{ if .Annotations.summary }}{{ .Annotations.summary }}{{ else }}{{ .Labels.alertname }}{{ end }}*{{ if .Labels.severity }} · {{ .Labels.severity }}{{ end }} + {{ if .Annotations.description }}{{ .Annotations.description }}{{ end }} + {{ end }} + footer: "Legion · neuralplatform.ai" + route: + group_by: [namespace, alertname] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + receiver: slack-alerts + routes: + - matchers: [alertname="Watchdog"] + receiver: "null" + - matchers: [severity="info"] + receiver: "null" + EOT + ] + depends_on = [helm_release.cert_manager] }