From dd0f8a49a3194a66eaf8e0c3a10a691d25fcc9a4 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Wed, 25 Mar 2026 10:15:41 -0500 Subject: [PATCH] monitoring: expose Alloy OTLP for external log ingestion from RunPod pods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Alloy config: route OTLP logs → Loki (was traces-only) - Alloy config: fix River syntax (semicolons in rule{} blocks are invalid) - Add alloy-otlp Service (ClusterIP:4318) - Add alloy-otlp-auth Secret (bcrypt htpasswd for Traefik BasicAuth) - Add alloy-otlp-basicauth-middleware Traefik Middleware - Add alloy.neuralplatform.ai Ingress (TLS via cert-manager) Auth: Authorization: Basic base64(pantheon:) Token stored in Vault at secret/alloy push_token RunPod pods push OTLP logs to https://alloy.neuralplatform.ai/v1/logs Alloy routes → Loki → queryable in Grafana --- servers/legion/ingress.tf | 59 ++++++++++++++++++++++++++++++++++++ servers/legion/monitoring.tf | 59 +++++++++++++++++++++++++++++++++--- servers/legion/variables.tf | 6 ++++ 3 files changed, 120 insertions(+), 4 deletions(-) diff --git a/servers/legion/ingress.tf b/servers/legion/ingress.tf index c491e84..480be40 100644 --- a/servers/legion/ingress.tf +++ b/servers/legion/ingress.tf @@ -147,6 +147,65 @@ resource "kubernetes_ingress_v1" "npm_registry" { depends_on = [helm_release.cert_manager] } +# Alloy OTLP HTTP ingress — external log/trace ingestion from RunPod pods +# Auth: HTTP Basic Auth (username: pantheon, password: ALLOY_PUSH_TOKEN from Vault) +# Pods set: Authorization: Basic base64(pantheon:) +# Traefik middleware strips auth before forwarding to Alloy. +resource "kubernetes_manifest" "alloy_otlp_basicauth_middleware" { + manifest = { + apiVersion = "traefik.io/v1alpha1" + kind = "Middleware" + metadata = { + name = "alloy-otlp-auth" + namespace = "monitoring" + } + spec = { + basicAuth = { + secret = "alloy-otlp-auth" + } + } + } + depends_on = [kubernetes_secret.alloy_otlp_auth] +} + +resource "kubernetes_ingress_v1" "alloy_otlp" { + metadata { + name = "alloy-otlp" + namespace = "monitoring" + annotations = { + "traefik.ingress.kubernetes.io/router.entrypoints" = "websecure" + "traefik.ingress.kubernetes.io/router.middlewares" = "monitoring-alloy-otlp-auth@kubernetescrd" + "cert-manager.io/cluster-issuer" = "letsencrypt-prod" + } + } + spec { + tls { + hosts = ["alloy.${var.infra_domain}"] + secret_name = "alloy-otlp-tls" + } + rule { + host = "alloy.${var.infra_domain}" + http { + path { + path = "/" + path_type = "Prefix" + backend { + service { + name = "alloy-otlp" + port { number = 4318 } + } + } + } + } + } + } + depends_on = [ + kubernetes_manifest.alloy_otlp_basicauth_middleware, + kubernetes_service.alloy_otlp, + helm_release.cert_manager, + ] +} + # PyPI registry — devpi (proxies pypi.org, caches locally) resource "kubernetes_ingress_v1" "pypi_registry" { metadata { diff --git a/servers/legion/monitoring.tf b/servers/legion/monitoring.tf index 017439a..5db4f83 100644 --- a/servers/legion/monitoring.tf +++ b/servers/legion/monitoring.tf @@ -294,10 +294,22 @@ resource "helm_release" "alloy" { discovery.relabel "pods" { targets = discovery.kubernetes.pods.targets - rule { source_labels = ["__meta_kubernetes_namespace"]; target_label = "namespace" } - rule { source_labels = ["__meta_kubernetes_pod_name"]; target_label = "pod" } - rule { source_labels = ["__meta_kubernetes_pod_container_name"]; target_label = "container" } - rule { source_labels = ["__meta_kubernetes_pod_label_app"]; target_label = "app" } + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app"] + target_label = "app" + } } loki.source.kubernetes "pods" { @@ -317,6 +329,7 @@ resource "helm_release" "alloy" { http { endpoint = "0.0.0.0:4318" } output { traces = [otelcol.exporter.otlp.tempo.input] + logs = [otelcol.exporter.loki.pantheon.input] } } @@ -326,8 +339,46 @@ resource "helm_release" "alloy" { tls { insecure = true } } } + + otelcol.exporter.loki "pantheon" { + forward_to = [loki.write.default.receiver] + } EOT ] depends_on = [helm_release.loki, helm_release.tempo] } + +# Expose Alloy's OTLP HTTP port (4318) for external log/trace ingestion from RunPod pods +resource "kubernetes_service" "alloy_otlp" { + metadata { + name = "alloy-otlp" + namespace = kubernetes_namespace.monitoring.metadata[0].name + labels = { app = "alloy-otlp" } + } + spec { + selector = { "app.kubernetes.io/name" = "alloy" } + port { + name = "otlp-http" + port = 4318 + target_port = 4318 + protocol = "TCP" + } + type = "ClusterIP" + } + depends_on = [helm_release.alloy] +} + +# BasicAuth secret for Traefik middleware — pods authenticate with +# Authorization: Basic base64(pantheon:) +resource "kubernetes_secret" "alloy_otlp_auth" { + metadata { + name = "alloy-otlp-auth" + namespace = kubernetes_namespace.monitoring.metadata[0].name + } + data = { + # htpasswd entry — bcrypt hash of the push token, username "pantheon" + users = "pantheon:${bcrypt(var.alloy_push_token)}" + } + depends_on = [kubernetes_namespace.monitoring] +} diff --git a/servers/legion/variables.tf b/servers/legion/variables.tf index b01fd5d..6c5531f 100644 --- a/servers/legion/variables.tf +++ b/servers/legion/variables.tf @@ -141,3 +141,9 @@ variable "gitea_runner_token" { type = string sensitive = true } + +variable "alloy_push_token" { + description = "Bearer token required to push OTLP logs/traces to alloy.neuralplatform.ai" + type = string + sensitive = true +}