monitoring: expose Alloy OTLP for external log ingestion from RunPod pods

- Alloy config: route OTLP logs → Loki (was traces-only)
- Alloy config: fix River syntax (semicolons in rule{} blocks are invalid)
- Add alloy-otlp Service (ClusterIP:4318)
- Add alloy-otlp-auth Secret (bcrypt htpasswd for Traefik BasicAuth)
- Add alloy-otlp-basicauth-middleware Traefik Middleware
- Add alloy.neuralplatform.ai Ingress (TLS via cert-manager)
  Auth: Authorization: Basic base64(pantheon:<push_token>)
  Token stored in Vault at secret/alloy push_token

RunPod pods push OTLP logs to https://alloy.neuralplatform.ai/v1/logs
Alloy routes → Loki → queryable in Grafana

servers/legion/ingress.tf

@@ -147,6 +147,65 @@ resource "kubernetes_ingress_v1" "npm_registry" {
   depends_on = [helm_release.cert_manager]
 }
 
+# Alloy OTLP HTTP ingress — external log/trace ingestion from RunPod pods
+# Auth: HTTP Basic Auth (username: pantheon, password: ALLOY_PUSH_TOKEN from Vault)
+# Pods set: Authorization: Basic base64(pantheon:<token>)
+# Traefik middleware strips auth before forwarding to Alloy.
+resource "kubernetes_manifest" "alloy_otlp_basicauth_middleware" {
+  manifest = {
+    apiVersion = "traefik.io/v1alpha1"
+    kind       = "Middleware"
+    metadata = {
+      name      = "alloy-otlp-auth"
+      namespace = "monitoring"
+    }
+    spec = {
+      basicAuth = {
+        secret = "alloy-otlp-auth"
+      }
+    }
+  }
+  depends_on = [kubernetes_secret.alloy_otlp_auth]
+}
+
+resource "kubernetes_ingress_v1" "alloy_otlp" {
+  metadata {
+    name      = "alloy-otlp"
+    namespace = "monitoring"
+    annotations = {
+      "traefik.ingress.kubernetes.io/router.entrypoints" = "websecure"
+      "traefik.ingress.kubernetes.io/router.middlewares" = "monitoring-alloy-otlp-auth@kubernetescrd"
+      "cert-manager.io/cluster-issuer"                   = "letsencrypt-prod"
+    }
+  }
+  spec {
+    tls {
+      hosts       = ["alloy.${var.infra_domain}"]
+      secret_name = "alloy-otlp-tls"
+    }
+    rule {
+      host = "alloy.${var.infra_domain}"
+      http {
+        path {
+          path      = "/"
+          path_type = "Prefix"
+          backend {
+            service {
+              name = "alloy-otlp"
+              port { number = 4318 }
+            }
+          }
+        }
+      }
+    }
+  }
+  depends_on = [
+    kubernetes_manifest.alloy_otlp_basicauth_middleware,
+    kubernetes_service.alloy_otlp,
+    helm_release.cert_manager,
+  ]
+}
+
 # PyPI registry — devpi (proxies pypi.org, caches locally)
 resource "kubernetes_ingress_v1" "pypi_registry" {
   metadata {

servers/legion/monitoring.tf

@@ -294,10 +294,22 @@ resource "helm_release" "alloy" {
 
           discovery.relabel "pods" {
             targets = discovery.kubernetes.pods.targets
-            rule { source_labels = ["__meta_kubernetes_namespace"];          target_label = "namespace" }
+            rule {
-            rule { source_labels = ["__meta_kubernetes_pod_name"];           target_label = "pod" }
+              source_labels = ["__meta_kubernetes_namespace"]
-            rule { source_labels = ["__meta_kubernetes_pod_container_name"]; target_label = "container" }
+              target_label  = "namespace"
-            rule { source_labels = ["__meta_kubernetes_pod_label_app"];      target_label = "app" }
+            }
+            rule {
+              source_labels = ["__meta_kubernetes_pod_name"]
+              target_label  = "pod"
+            }
+            rule {
+              source_labels = ["__meta_kubernetes_pod_container_name"]
+              target_label  = "container"
+            }
+            rule {
+              source_labels = ["__meta_kubernetes_pod_label_app"]
+              target_label  = "app"
+            }
           }
 
           loki.source.kubernetes "pods" {
@@ -317,6 +329,7 @@ resource "helm_release" "alloy" {
             http { endpoint = "0.0.0.0:4318" }
             output {
               traces = [otelcol.exporter.otlp.tempo.input]
+              logs   = [otelcol.exporter.loki.pantheon.input]
             }
           }
 
@@ -326,8 +339,46 @@ resource "helm_release" "alloy" {
               tls { insecure = true }
             }
           }
+
+          otelcol.exporter.loki "pantheon" {
+            forward_to = [loki.write.default.receiver]
+          }
   EOT
   ]
 
   depends_on = [helm_release.loki, helm_release.tempo]
 }
+
+# Expose Alloy's OTLP HTTP port (4318) for external log/trace ingestion from RunPod pods
+resource "kubernetes_service" "alloy_otlp" {
+  metadata {
+    name      = "alloy-otlp"
+    namespace = kubernetes_namespace.monitoring.metadata[0].name
+    labels    = { app = "alloy-otlp" }
+  }
+  spec {
+    selector = { "app.kubernetes.io/name" = "alloy" }
+    port {
+      name        = "otlp-http"
+      port        = 4318
+      target_port = 4318
+      protocol    = "TCP"
+    }
+    type = "ClusterIP"
+  }
+  depends_on = [helm_release.alloy]
+}
+
+# BasicAuth secret for Traefik middleware — pods authenticate with
+# Authorization: Basic base64(pantheon:<ALLOY_PUSH_TOKEN>)
+resource "kubernetes_secret" "alloy_otlp_auth" {
+  metadata {
+    name      = "alloy-otlp-auth"
+    namespace = kubernetes_namespace.monitoring.metadata[0].name
+  }
+  data = {
+    # htpasswd entry — bcrypt hash of the push token, username "pantheon"
+    users = "pantheon:${bcrypt(var.alloy_push_token)}"
+  }
+  depends_on = [kubernetes_namespace.monitoring]
+}

servers/legion/variables.tf

@@ -141,3 +141,9 @@ variable "gitea_runner_token" {
   type        = string
   sensitive   = true
 }
+
+variable "alloy_push_token" {
+  description = "Bearer token required to push OTLP logs/traces to alloy.neuralplatform.ai"
+  type        = string
+  sensitive   = true
+}