will.anderson/infrastructure
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

monitoring: expose Alloy OTLP for external log ingestion from RunPod pods

Browse Source 
- Alloy config: route OTLP logs → Loki (was traces-only)
- Alloy config: fix River syntax (semicolons in rule{} blocks are invalid)
- Add alloy-otlp Service (ClusterIP:4318)
- Add alloy-otlp-auth Secret (bcrypt htpasswd for Traefik BasicAuth)
- Add alloy-otlp-basicauth-middleware Traefik Middleware
- Add alloy.neuralplatform.ai Ingress (TLS via cert-manager)
  Auth: Authorization: Basic base64(pantheon:<push_token>)
  Token stored in Vault at secret/alloy push_token

RunPod pods push OTLP logs to https://alloy.neuralplatform.ai/v1/logs
Alloy routes → Loki → queryable in Grafana
This commit is contained in:
Will Anderson
2026-03-25 10:15:41 -05:00
parent 6ebf8a9b3e
commit dd0f8a49a3
3 changed files with 120 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
servers/legion/monitoring.tf
+55 -4
View File
@@ -294,10 +294,22 @@ resource "helm_release" "alloy" {
discovery.relabel "pods" {
targets = discovery.kubernetes.pods.targets
rule { source_labels = ["__meta_kubernetes_namespace"]; target_label = "namespace" }
rule { source_labels = ["__meta_kubernetes_pod_name"]; target_label = "pod" }
rule { source_labels = ["__meta_kubernetes_pod_container_name"]; target_label = "container" }
rule { source_labels = ["__meta_kubernetes_pod_label_app"]; target_label = "app" }
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_name"]
target_label = "container"
}
rule {
source_labels = ["__meta_kubernetes_pod_label_app"]
target_label = "app"
}
}
loki.source.kubernetes "pods" {
@@ -317,6 +329,7 @@ resource "helm_release" "alloy" {
http { endpoint = "0.0.0.0:4318" }
output {
traces = [otelcol.exporter.otlp.tempo.input]
logs = [otelcol.exporter.loki.pantheon.input]
}
}
@@ -326,8 +339,46 @@ resource "helm_release" "alloy" {
tls { insecure = true }
}
}
otelcol.exporter.loki "pantheon" {
forward_to = [loki.write.default.receiver]
}
EOT
]
depends_on = [helm_release.loki, helm_release.tempo]
}
# Expose Alloy's OTLP HTTP port (4318) for external log/trace ingestion from RunPod pods
resource "kubernetes_service" "alloy_otlp" {
metadata {
name = "alloy-otlp"
namespace = kubernetes_namespace.monitoring.metadata[0].name
labels = { app = "alloy-otlp" }
}
spec {
selector = { "app.kubernetes.io/name" = "alloy" }
port {
name = "otlp-http"
port = 4318
target_port = 4318
protocol = "TCP"
}
type = "ClusterIP"
}
depends_on = [helm_release.alloy]
}
# BasicAuth secret for Traefik middleware — pods authenticate with
# Authorization: Basic base64(pantheon:<ALLOY_PUSH_TOKEN>)
resource "kubernetes_secret" "alloy_otlp_auth" {
metadata {
name = "alloy-otlp-auth"
namespace = kubernetes_namespace.monitoring.metadata[0].name
}
data = {
# htpasswd entry — bcrypt hash of the push token, username "pantheon"
users = "pantheon:${bcrypt(var.alloy_push_token)}"
}
depends_on = [kubernetes_namespace.monitoring]
}