monitoring: wire Alertmanager to #infrastructure-alerts in Slack
Routes all warning/critical alerts to the Neural Platform Slack workspace. Suppresses Watchdog and info-level noise. Groups by namespace+alertname, repeats every 4h. Uses bot token auth via chat.postMessage API.
This commit is contained in:
@@ -96,6 +96,51 @@ resource "helm_release" "kube_prometheus_stack" {
|
|||||||
value = "false"
|
value = "false"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Alertmanager — route alerts to #infrastructure-alerts in Slack
|
||||||
|
values = [<<-EOT
|
||||||
|
alertmanager:
|
||||||
|
config:
|
||||||
|
global:
|
||||||
|
resolve_timeout: 5m
|
||||||
|
inhibit_rules:
|
||||||
|
- source_matchers: [severity="critical"]
|
||||||
|
target_matchers: [severity=~"warning|info"]
|
||||||
|
equal: [namespace, alertname]
|
||||||
|
- source_matchers: [severity="warning"]
|
||||||
|
target_matchers: [severity="info"]
|
||||||
|
equal: [namespace, alertname]
|
||||||
|
receivers:
|
||||||
|
- name: "null"
|
||||||
|
- name: slack-alerts
|
||||||
|
slack_configs:
|
||||||
|
- channel: "#infrastructure-alerts"
|
||||||
|
api_url: "https://slack.com/api/chat.postMessage"
|
||||||
|
http_config:
|
||||||
|
authorization:
|
||||||
|
type: Bearer
|
||||||
|
credentials: "${var.slack_bot_token}"
|
||||||
|
send_resolved: true
|
||||||
|
title: '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
|
||||||
|
text: |
|
||||||
|
{{ range .Alerts }}
|
||||||
|
*{{ if .Annotations.summary }}{{ .Annotations.summary }}{{ else }}{{ .Labels.alertname }}{{ end }}*{{ if .Labels.severity }} · {{ .Labels.severity }}{{ end }}
|
||||||
|
{{ if .Annotations.description }}{{ .Annotations.description }}{{ end }}
|
||||||
|
{{ end }}
|
||||||
|
footer: "Legion · neuralplatform.ai"
|
||||||
|
route:
|
||||||
|
group_by: [namespace, alertname]
|
||||||
|
group_wait: 30s
|
||||||
|
group_interval: 5m
|
||||||
|
repeat_interval: 4h
|
||||||
|
receiver: slack-alerts
|
||||||
|
routes:
|
||||||
|
- matchers: [alertname="Watchdog"]
|
||||||
|
receiver: "null"
|
||||||
|
- matchers: [severity="info"]
|
||||||
|
receiver: "null"
|
||||||
|
EOT
|
||||||
|
]
|
||||||
|
|
||||||
depends_on = [helm_release.cert_manager]
|
depends_on = [helm_release.cert_manager]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user