Migrate k8s config from Terraform to Argo CD + ESO
Phase 1: Install External Secrets Operator via Argo CD app - apps/external-secrets.yaml — ESO Helm chart install - apps/external-secrets-config.yaml — ClusterSecretStore deployment - k8s/external-secrets/cluster-secret-store.yaml — Vault backend using vault-token Secret Phase 2: Create k8s manifests for all services - k8s/neuron/ — PVC, ConfigMap, ExternalSecrets (neuron-secrets, cloudflared-secret), Ingress - k8s/gitea/ — PVC, ConfigMap (custom CSS), ExternalSecret (gitea-db), Ingress - k8s/github-runner/ — ExternalSecret (github-runner-secret) - k8s/gitea-runner/ — ExternalSecret (gitea-runner-secret) - k8s/monitoring/ — ExternalSecrets (grafana, slack), Alloy OTLP service+middleware, datasources ConfigMap, Ingress - k8s/postgres/ — ExternalSecret (postgres-passwords) - k8s/vault/ — ExternalSecret (vault-gcp-sa from Vault) - k8s/adguard/ — PVCs, ConfigMap, Certificate, Ingress, ddclient Deployment+ExternalSecret - k8s/ollama/ — PVC, Ingress - k8s/headscale/ — PVC - k8s/packages/ — PVCs, ConfigMap, Ingresses - k8s/registry/ — PVC, Ingresses - k8s/backup/ — CronJob, ExternalSecret (backup-credentials) New Argo CD apps for Helm releases: - apps/kube-prometheus-stack.yaml, loki.yaml, tempo.yaml, alloy.yaml - apps/postgres.yaml, redis.yaml, vault.yaml New Argo CD apps for k8s config paths: - apps/neuron-config, gitea-config, ci-config, gitea-runner-config - apps/monitoring-config, adguard-config, ollama-config, headscale-config - apps/packages-config, registry-config, postgres-config, vault-config, backup Phase 3: Strip Terraform to infrastructure-only - All kubernetes_* and helm_release resources removed from service .tf files - Each service .tf now contains only kubernetes_namespace (bootstrap dependency) - variables.tf stripped to only cloudflare_api_key, cloudflare_email, gitea_api_token - namespaces.tf gains external-secrets namespace - ingress.tf, backup.tf, ddclient.tf emptied (resources in k8s/) - cert-manager.tf, argocd.tf, traefik.tf unchanged (bootstrap)
This commit is contained in:
Will Anderson
@@ -1,8 +1,7 @@
|
||||
# Monitoring stack — Prometheus, Grafana, Loki, Tempo, Alloy, Alertmanager
|
||||
# Grafana UI: https://grafana.neuralplatform.ai (admin / var.grafana_admin_password)
|
||||
#
|
||||
# TODO: Migrate Tempo to tempo-distributed once R2 is wired as trace storage backend
|
||||
# TODO: Replace loki.useTestSchema=true with proper schema_config for production
|
||||
# Monitoring stack — Prometheus, Grafana, Loki, Tempo, Alloy
|
||||
# Namespace only — all Helm releases and config managed by Argo CD + ESO
|
||||
# See: apps/kube-prometheus-stack.yaml, apps/loki.yaml, apps/tempo.yaml, apps/alloy.yaml
|
||||
# See: apps/monitoring-config.yaml, k8s/monitoring/
|
||||
|
||||
resource "kubernetes_namespace" "monitoring" {
|
||||
metadata {
|
||||
@@ -13,372 +12,3 @@ resource "kubernetes_namespace" "monitoring" {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# kube-prometheus-stack — Prometheus, Grafana, Alertmanager, node exporters
|
||||
resource "helm_release" "kube_prometheus_stack" {
|
||||
name = "kube-prometheus-stack"
|
||||
namespace = kubernetes_namespace.monitoring.metadata[0].name
|
||||
repository = "https://prometheus-community.github.io/helm-charts"
|
||||
chart = "kube-prometheus-stack"
|
||||
timeout = 600
|
||||
|
||||
# Grafana
|
||||
set {
|
||||
name = "grafana.adminPassword"
|
||||
value = var.grafana_admin_password
|
||||
}
|
||||
set {
|
||||
name = "grafana.persistence.enabled"
|
||||
value = "true"
|
||||
}
|
||||
set {
|
||||
name = "grafana.persistence.size"
|
||||
value = "5Gi"
|
||||
}
|
||||
set {
|
||||
name = "grafana.sidecar.datasources.defaultDatasourceEnabled"
|
||||
value = "false"
|
||||
}
|
||||
set {
|
||||
name = "grafana.initChownData.enabled"
|
||||
value = "false"
|
||||
}
|
||||
set {
|
||||
name = "grafana.ingress.enabled"
|
||||
value = "true"
|
||||
}
|
||||
set {
|
||||
name = "grafana.ingress.ingressClassName"
|
||||
value = "traefik"
|
||||
}
|
||||
set {
|
||||
name = "grafana.ingress.hosts[0]"
|
||||
value = "grafana.${var.infra_domain}"
|
||||
}
|
||||
set {
|
||||
name = "grafana.ingress.tls[0].secretName"
|
||||
value = "grafana-tls"
|
||||
}
|
||||
set {
|
||||
name = "grafana.ingress.tls[0].hosts[0]"
|
||||
value = "grafana.${var.infra_domain}"
|
||||
}
|
||||
set {
|
||||
name = "grafana.ingress.annotations.traefik\\.ingress\\.kubernetes\\.io/router\\.entrypoints"
|
||||
value = "websecure"
|
||||
}
|
||||
set {
|
||||
name = "grafana.ingress.annotations.cert-manager\\.io/cluster-issuer"
|
||||
value = "letsencrypt-prod"
|
||||
}
|
||||
|
||||
# Prometheus persistent storage
|
||||
set {
|
||||
name = "prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage"
|
||||
value = "20Gi"
|
||||
}
|
||||
|
||||
# Disable k3s-incompatible scrapers
|
||||
set {
|
||||
name = "kubeEtcd.enabled"
|
||||
value = "false"
|
||||
}
|
||||
set {
|
||||
name = "kubeControllerManager.enabled"
|
||||
value = "false"
|
||||
}
|
||||
set {
|
||||
name = "kubeScheduler.enabled"
|
||||
value = "false"
|
||||
}
|
||||
set {
|
||||
name = "kubeProxy.enabled"
|
||||
value = "false"
|
||||
}
|
||||
|
||||
# Alertmanager — route alerts to #infrastructure-alerts in Slack
|
||||
values = [<<-EOT
|
||||
alertmanager:
|
||||
config:
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
inhibit_rules:
|
||||
- source_matchers: [severity="critical"]
|
||||
target_matchers: [severity=~"warning|info"]
|
||||
equal: [namespace, alertname]
|
||||
- source_matchers: [severity="warning"]
|
||||
target_matchers: [severity="info"]
|
||||
equal: [namespace, alertname]
|
||||
receivers:
|
||||
- name: "null"
|
||||
- name: slack-alerts
|
||||
slack_configs:
|
||||
- channel: "#infrastructure-alerts"
|
||||
api_url: "https://slack.com/api/chat.postMessage"
|
||||
http_config:
|
||||
authorization:
|
||||
type: Bearer
|
||||
credentials: "${var.slack_bot_token}"
|
||||
send_resolved: true
|
||||
title: '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} ({{ .CommonLabels.namespace }})'
|
||||
text: |
|
||||
{{ range .Alerts }}
|
||||
*{{ if .Annotations.summary }}{{ .Annotations.summary }}{{ else }}{{ .Labels.alertname }}{{ end }}*{{ if .Labels.severity }} · {{ .Labels.severity }}{{ end }}
|
||||
{{ if .Annotations.description }}{{ .Annotations.description }}{{ end }}
|
||||
{{ end }}
|
||||
footer: "Legion · neuralplatform.ai"
|
||||
route:
|
||||
group_by: [namespace, alertname]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
receiver: slack-alerts
|
||||
routes:
|
||||
- matchers: [alertname="Watchdog"]
|
||||
receiver: "null"
|
||||
- matchers: [severity="info"]
|
||||
receiver: "null"
|
||||
EOT
|
||||
]
|
||||
|
||||
depends_on = [helm_release.cert_manager]
|
||||
}
|
||||
|
||||
# Grafana extra datasources — Loki and Tempo wired up with cross-linking
|
||||
resource "kubernetes_config_map" "grafana_extra_datasources" {
|
||||
metadata {
|
||||
name = "grafana-extra-datasources"
|
||||
namespace = kubernetes_namespace.monitoring.metadata[0].name
|
||||
labels = { grafana_datasource = "1" }
|
||||
}
|
||||
|
||||
data = {
|
||||
"datasources.yaml" = <<-EOT
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Loki
|
||||
type: loki
|
||||
uid: loki
|
||||
url: http://loki-gateway.monitoring.svc.cluster.local
|
||||
access: proxy
|
||||
jsonData:
|
||||
httpHeaderName1: X-Scope-OrgID
|
||||
secureJsonData:
|
||||
httpHeaderValue1: legion
|
||||
- name: Tempo
|
||||
type: tempo
|
||||
uid: tempo
|
||||
url: http://tempo.monitoring.svc.cluster.local:3100
|
||||
access: proxy
|
||||
jsonData:
|
||||
lokiSearch:
|
||||
datasourceUid: loki
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
EOT
|
||||
}
|
||||
|
||||
depends_on = [helm_release.kube_prometheus_stack]
|
||||
}
|
||||
|
||||
# Loki — log aggregation (single binary mode)
|
||||
# TODO: Replace useTestSchema with proper schema_config before storing long-term logs
|
||||
resource "helm_release" "loki" {
|
||||
name = "loki"
|
||||
namespace = kubernetes_namespace.monitoring.metadata[0].name
|
||||
repository = "https://grafana.github.io/helm-charts"
|
||||
chart = "loki"
|
||||
timeout = 300
|
||||
|
||||
set {
|
||||
name = "loki.commonConfig.replication_factor"
|
||||
value = "1"
|
||||
}
|
||||
set {
|
||||
name = "loki.storage.type"
|
||||
value = "filesystem"
|
||||
}
|
||||
set {
|
||||
name = "loki.useTestSchema"
|
||||
value = "true"
|
||||
}
|
||||
set {
|
||||
name = "singleBinary.replicas"
|
||||
value = "1"
|
||||
}
|
||||
set {
|
||||
name = "singleBinary.persistence.enabled"
|
||||
value = "true"
|
||||
}
|
||||
set {
|
||||
name = "singleBinary.persistence.size"
|
||||
value = "10Gi"
|
||||
}
|
||||
set {
|
||||
name = "deploymentMode"
|
||||
value = "SingleBinary"
|
||||
}
|
||||
set {
|
||||
name = "backend.replicas"
|
||||
value = "0"
|
||||
}
|
||||
set {
|
||||
name = "read.replicas"
|
||||
value = "0"
|
||||
}
|
||||
set {
|
||||
name = "write.replicas"
|
||||
value = "0"
|
||||
}
|
||||
set {
|
||||
name = "chunksCache.enabled"
|
||||
value = "false"
|
||||
}
|
||||
set {
|
||||
name = "resultsCache.enabled"
|
||||
value = "false"
|
||||
}
|
||||
}
|
||||
|
||||
# Tempo — distributed tracing backend (local storage)
|
||||
# TODO: Migrate to tempo-distributed with R2 storage once R2 trace bucket is created
|
||||
resource "helm_release" "tempo" {
|
||||
name = "tempo"
|
||||
namespace = kubernetes_namespace.monitoring.metadata[0].name
|
||||
repository = "https://grafana.github.io/helm-charts"
|
||||
chart = "tempo"
|
||||
timeout = 300
|
||||
|
||||
set {
|
||||
name = "tempo.storage.trace.backend"
|
||||
value = "local"
|
||||
}
|
||||
set {
|
||||
name = "tempo.storage.trace.local.path"
|
||||
value = "/var/tempo"
|
||||
}
|
||||
set {
|
||||
name = "persistence.enabled"
|
||||
value = "true"
|
||||
}
|
||||
set {
|
||||
name = "persistence.size"
|
||||
value = "10Gi"
|
||||
}
|
||||
}
|
||||
|
||||
# Alloy — log shipping + OTLP trace forwarding (replaces deprecated Promtail)
|
||||
resource "helm_release" "alloy" {
|
||||
name = "alloy"
|
||||
namespace = kubernetes_namespace.monitoring.metadata[0].name
|
||||
repository = "https://grafana.github.io/helm-charts"
|
||||
chart = "alloy"
|
||||
timeout = 300
|
||||
|
||||
set {
|
||||
name = "controller.type"
|
||||
value = "daemonset"
|
||||
}
|
||||
|
||||
values = [<<-EOT
|
||||
alloy:
|
||||
configMap:
|
||||
content: |
|
||||
discovery.kubernetes "pods" {
|
||||
role = "pod"
|
||||
}
|
||||
|
||||
discovery.relabel "pods" {
|
||||
targets = discovery.kubernetes.pods.targets
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace"]
|
||||
target_label = "namespace"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_name"]
|
||||
target_label = "pod"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_container_name"]
|
||||
target_label = "container"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_label_app"]
|
||||
target_label = "app"
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.kubernetes "pods" {
|
||||
targets = discovery.relabel.pods.output
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
url = "http://loki-gateway.monitoring.svc.cluster.local/loki/api/v1/push"
|
||||
tenant_id = "legion"
|
||||
}
|
||||
}
|
||||
|
||||
otelcol.receiver.otlp "default" {
|
||||
grpc { endpoint = "0.0.0.0:4317" }
|
||||
http { endpoint = "0.0.0.0:4318" }
|
||||
output {
|
||||
traces = [otelcol.exporter.otlp.tempo.input]
|
||||
logs = [otelcol.exporter.loki.pantheon.input]
|
||||
}
|
||||
}
|
||||
|
||||
otelcol.exporter.otlp "tempo" {
|
||||
client {
|
||||
endpoint = "tempo.monitoring.svc.cluster.local:4317"
|
||||
tls { insecure = true }
|
||||
}
|
||||
}
|
||||
|
||||
otelcol.exporter.loki "pantheon" {
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
EOT
|
||||
]
|
||||
|
||||
depends_on = [helm_release.loki, helm_release.tempo]
|
||||
}
|
||||
|
||||
# Expose Alloy's OTLP HTTP port (4318) for external log/trace ingestion from RunPod pods
|
||||
resource "kubernetes_service" "alloy_otlp" {
|
||||
metadata {
|
||||
name = "alloy-otlp"
|
||||
namespace = kubernetes_namespace.monitoring.metadata[0].name
|
||||
labels = { app = "alloy-otlp" }
|
||||
}
|
||||
spec {
|
||||
selector = { "app.kubernetes.io/name" = "alloy" }
|
||||
port {
|
||||
name = "otlp-http"
|
||||
port = 4318
|
||||
target_port = 4318
|
||||
protocol = "TCP"
|
||||
}
|
||||
type = "ClusterIP"
|
||||
}
|
||||
depends_on = [helm_release.alloy]
|
||||
}
|
||||
|
||||
# BasicAuth secret for Traefik middleware — pods authenticate with
|
||||
# Authorization: Basic base64(pantheon:<ALLOY_PUSH_TOKEN>)
|
||||
resource "kubernetes_secret" "alloy_otlp_auth" {
|
||||
metadata {
|
||||
name = "alloy-otlp-auth"
|
||||
namespace = kubernetes_namespace.monitoring.metadata[0].name
|
||||
}
|
||||
data = {
|
||||
# htpasswd entry — bcrypt hash of the push token, username "pantheon"
|
||||
users = "pantheon:${bcrypt(var.alloy_push_token)}"
|
||||
}
|
||||
depends_on = [kubernetes_namespace.monitoring]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user