monitoring: expose Alloy OTLP for external log ingestion from RunPod pods

- Alloy config: route OTLP logs → Loki (was traces-only)
- Alloy config: fix River syntax (semicolons in rule{} blocks are invalid)
- Add alloy-otlp Service (ClusterIP:4318)
- Add alloy-otlp-auth Secret (bcrypt htpasswd for Traefik BasicAuth)
- Add alloy-otlp-basicauth-middleware Traefik Middleware
- Add alloy.neuralplatform.ai Ingress (TLS via cert-manager)
  Auth: Authorization: Basic base64(pantheon:<push_token>)
  Token stored in Vault at secret/alloy push_token

RunPod pods push OTLP logs to https://alloy.neuralplatform.ai/v1/logs
Alloy routes → Loki → queryable in Grafana
This commit is contained in:
Will Anderson
2026-03-25 10:15:41 -05:00
parent 6ebf8a9b3e
commit dd0f8a49a3
3 changed files with 120 additions and 4 deletions
+59
View File
@@ -147,6 +147,65 @@ resource "kubernetes_ingress_v1" "npm_registry" {
depends_on = [helm_release.cert_manager]
}
# Alloy OTLP HTTP ingress — external log/trace ingestion from RunPod pods
# Auth: HTTP Basic Auth (username: pantheon, password: ALLOY_PUSH_TOKEN from Vault)
# Pods set: Authorization: Basic base64(pantheon:<token>)
# Traefik middleware strips auth before forwarding to Alloy.
resource "kubernetes_manifest" "alloy_otlp_basicauth_middleware" {
manifest = {
apiVersion = "traefik.io/v1alpha1"
kind = "Middleware"
metadata = {
name = "alloy-otlp-auth"
namespace = "monitoring"
}
spec = {
basicAuth = {
secret = "alloy-otlp-auth"
}
}
}
depends_on = [kubernetes_secret.alloy_otlp_auth]
}
resource "kubernetes_ingress_v1" "alloy_otlp" {
metadata {
name = "alloy-otlp"
namespace = "monitoring"
annotations = {
"traefik.ingress.kubernetes.io/router.entrypoints" = "websecure"
"traefik.ingress.kubernetes.io/router.middlewares" = "monitoring-alloy-otlp-auth@kubernetescrd"
"cert-manager.io/cluster-issuer" = "letsencrypt-prod"
}
}
spec {
tls {
hosts = ["alloy.${var.infra_domain}"]
secret_name = "alloy-otlp-tls"
}
rule {
host = "alloy.${var.infra_domain}"
http {
path {
path = "/"
path_type = "Prefix"
backend {
service {
name = "alloy-otlp"
port { number = 4318 }
}
}
}
}
}
}
depends_on = [
kubernetes_manifest.alloy_otlp_basicauth_middleware,
kubernetes_service.alloy_otlp,
helm_release.cert_manager,
]
}
# PyPI registry — devpi (proxies pypi.org, caches locally)
resource "kubernetes_ingress_v1" "pypi_registry" {
metadata {