infrastructure/servers/legion/monitoring.tf

# Monitoring stack — Prometheus, Grafana, Loki, Tempo, Alloy, Alertmanager
# Grafana UI: https://grafana.neuralplatform.ai (admin / var.grafana_admin_password)
#
# TODO: Migrate Tempo to tempo-distributed once R2 is wired as trace storage backend
# TODO: Replace loki.useTestSchema=true with proper schema_config for production

resource "kubernetes_namespace" "monitoring" {
  metadata {
    name = "monitoring"
    labels = {
      managed-by = "terraform"
      tier       = "observability"
    }
  }
}

# kube-prometheus-stack — Prometheus, Grafana, Alertmanager, node exporters
resource "helm_release" "kube_prometheus_stack" {
  name       = "kube-prometheus-stack"
  namespace  = kubernetes_namespace.monitoring.metadata[0].name
  repository = "https://prometheus-community.github.io/helm-charts"
  chart      = "kube-prometheus-stack"
  timeout    = 600

  # Grafana
  set {
    name  = "grafana.adminPassword"
    value = var.grafana_admin_password
  }
  set {
    name  = "grafana.persistence.enabled"
    value = "true"
  }
  set {
    name  = "grafana.persistence.size"
    value = "5Gi"
  }
  set {
    name  = "grafana.sidecar.datasources.defaultDatasourceEnabled"
    value = "false"
  }
  set {
    name  = "grafana.initChownData.enabled"
    value = "false"
  }
  set {
    name  = "grafana.ingress.enabled"
    value = "true"
  }
  set {
    name  = "grafana.ingress.ingressClassName"
    value = "traefik"
  }
  set {
    name  = "grafana.ingress.hosts[0]"
    value = "grafana.${var.infra_domain}"
  }
  set {
    name  = "grafana.ingress.tls[0].secretName"
    value = "grafana-tls"
  }
  set {
    name  = "grafana.ingress.tls[0].hosts[0]"
    value = "grafana.${var.infra_domain}"
  }
  set {
    name  = "grafana.ingress.annotations.traefik\\.ingress\\.kubernetes\\.io/router\\.entrypoints"
    value = "websecure"
  }
  set {
    name  = "grafana.ingress.annotations.cert-manager\\.io/cluster-issuer"
    value = "letsencrypt-prod"
  }

  # Prometheus persistent storage
  set {
    name  = "prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage"
    value = "20Gi"
  }

  # Disable k3s-incompatible scrapers
  set {
    name  = "kubeEtcd.enabled"
    value = "false"
  }
  set {
    name  = "kubeControllerManager.enabled"
    value = "false"
  }
  set {
    name  = "kubeScheduler.enabled"
    value = "false"
  }
  set {
    name  = "kubeProxy.enabled"
    value = "false"
  }

  # Alertmanager — route alerts to #infrastructure-alerts in Slack
  values = [<<-EOT
    alertmanager:
      config:
        global:
          resolve_timeout: 5m
        inhibit_rules:
          - source_matchers: [severity="critical"]
            target_matchers: [severity=~"warning|info"]
            equal: [namespace, alertname]
          - source_matchers: [severity="warning"]
            target_matchers: [severity="info"]
            equal: [namespace, alertname]
        receivers:
          - name: "null"
          - name: slack-alerts
            slack_configs:
              - channel: "#infrastructure-alerts"
                api_url: "https://slack.com/api/chat.postMessage"
                http_config:
                  authorization:
                    type: Bearer
                    credentials: "${var.slack_bot_token}"
                send_resolved: true
                title: '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
                text: |
                  {{ range .Alerts }}
                  *{{ if .Annotations.summary }}{{ .Annotations.summary }}{{ else }}{{ .Labels.alertname }}{{ end }}*{{ if .Labels.severity }} · {{ .Labels.severity }}{{ end }}
                  {{ if .Annotations.description }}{{ .Annotations.description }}{{ end }}
                  {{ end }}
                footer: "Legion · neuralplatform.ai"
        route:
          group_by: [namespace, alertname]
          group_wait: 30s
          group_interval: 5m
          repeat_interval: 4h
          receiver: slack-alerts
          routes:
            - matchers: [alertname="Watchdog"]
              receiver: "null"
            - matchers: [severity="info"]
              receiver: "null"
  EOT
  ]

  depends_on = [helm_release.cert_manager]
}

# Grafana extra datasources — Loki and Tempo wired up with cross-linking
resource "kubernetes_config_map" "grafana_extra_datasources" {
  metadata {
    name      = "grafana-extra-datasources"
    namespace = kubernetes_namespace.monitoring.metadata[0].name
    labels    = { grafana_datasource = "1" }
  }

  data = {
    "datasources.yaml" = <<-EOT
      apiVersion: 1
      datasources:
      - name: Loki
        type: loki
        uid: loki
        url: http://loki-gateway.monitoring.svc.cluster.local
        access: proxy
        jsonData:
          httpHeaderName1: X-Scope-OrgID
        secureJsonData:
          httpHeaderValue1: legion
      - name: Tempo
        type: tempo
        uid: tempo
        url: http://tempo.monitoring.svc.cluster.local:3100
        access: proxy
        jsonData:
          lokiSearch:
            datasourceUid: loki
          tracesToLogsV2:
            datasourceUid: loki
            filterByTraceID: true
            filterBySpanID: false
          tracesToMetrics:
            datasourceUid: prometheus
    EOT
  }

  depends_on = [helm_release.kube_prometheus_stack]
}

# Loki — log aggregation (single binary mode)
# TODO: Replace useTestSchema with proper schema_config before storing long-term logs
resource "helm_release" "loki" {
  name       = "loki"
  namespace  = kubernetes_namespace.monitoring.metadata[0].name
  repository = "https://grafana.github.io/helm-charts"
  chart      = "loki"
  timeout    = 300

  set {
    name  = "loki.commonConfig.replication_factor"
    value = "1"
  }
  set {
    name  = "loki.storage.type"
    value = "filesystem"
  }
  set {
    name  = "loki.useTestSchema"
    value = "true"
  }
  set {
    name  = "singleBinary.replicas"
    value = "1"
  }
  set {
    name  = "singleBinary.persistence.enabled"
    value = "true"
  }
  set {
    name  = "singleBinary.persistence.size"
    value = "10Gi"
  }
  set {
    name  = "deploymentMode"
    value = "SingleBinary"
  }
  set {
    name  = "backend.replicas"
    value = "0"
  }
  set {
    name  = "read.replicas"
    value = "0"
  }
  set {
    name  = "write.replicas"
    value = "0"
  }
  set {
    name  = "chunksCache.enabled"
    value = "false"
  }
  set {
    name  = "resultsCache.enabled"
    value = "false"
  }
}

# Tempo — distributed tracing backend (local storage)
# TODO: Migrate to tempo-distributed with R2 storage once R2 trace bucket is created
resource "helm_release" "tempo" {
  name       = "tempo"
  namespace  = kubernetes_namespace.monitoring.metadata[0].name
  repository = "https://grafana.github.io/helm-charts"
  chart      = "tempo"
  timeout    = 300

  set {
    name  = "tempo.storage.trace.backend"
    value = "local"
  }
  set {
    name  = "tempo.storage.trace.local.path"
    value = "/var/tempo"
  }
  set {
    name  = "persistence.enabled"
    value = "true"
  }
  set {
    name  = "persistence.size"
    value = "10Gi"
  }
}

# Alloy — log shipping + OTLP trace forwarding (replaces deprecated Promtail)
resource "helm_release" "alloy" {
  name       = "alloy"
  namespace  = kubernetes_namespace.monitoring.metadata[0].name
  repository = "https://grafana.github.io/helm-charts"
  chart      = "alloy"
  timeout    = 300

  set {
    name  = "controller.type"
    value = "daemonset"
  }

  values = [<<-EOT
    alloy:
      configMap:
        content: |
          discovery.kubernetes "pods" {
            role = "pod"
          }

          discovery.relabel "pods" {
            targets = discovery.kubernetes.pods.targets
            rule { source_labels = ["__meta_kubernetes_namespace"];          target_label = "namespace" }
            rule { source_labels = ["__meta_kubernetes_pod_name"];           target_label = "pod" }
            rule { source_labels = ["__meta_kubernetes_pod_container_name"]; target_label = "container" }
            rule { source_labels = ["__meta_kubernetes_pod_label_app"];      target_label = "app" }
          }

          loki.source.kubernetes "pods" {
            targets    = discovery.relabel.pods.output
            forward_to = [loki.write.default.receiver]
          }

          loki.write "default" {
            endpoint {
              url       = "http://loki-gateway.monitoring.svc.cluster.local/loki/api/v1/push"
              tenant_id = "legion"
            }
          }

          otelcol.receiver.otlp "default" {
            grpc { endpoint = "0.0.0.0:4317" }
            http { endpoint = "0.0.0.0:4318" }
            output {
              traces = [otelcol.exporter.otlp.tempo.input]
            }
          }

          otelcol.exporter.otlp "tempo" {
            client {
              endpoint = "tempo.monitoring.svc.cluster.local:4317"
              tls { insecure = true }
            }
          }
  EOT
  ]

  depends_on = [helm_release.loki, helm_release.tempo]
}