infrastructure/servers/legion/k8s/neuron-technologies/prod/runpod-lb-configmap.yaml

---
# RunPod Inference Load Balancer — Nginx ConfigMap
#
# Deploys a lightweight nginx reverse proxy inside the cluster that fans out
# requests to multiple RunPod inference endpoints.  Update the upstream block
# with real pod proxy URLs after running runpod-provision.sh.
#
# Reload nginx after updating endpoints:
#   kubectl rollout restart deployment/runpod-lb -n neuron-prod
apiVersion: v1
kind: ConfigMap
metadata:
  name: runpod-lb-nginx-config
  namespace: neuron-prod
  labels:
    app.kubernetes.io/name: runpod-lb
    app.kubernetes.io/component: load-balancer
    app.kubernetes.io/managed-by: argocd
data:
  nginx.conf: |
    worker_processes auto;
    error_log /dev/stderr warn;
    pid /tmp/nginx.pid;

    events {
        worker_connections 1024;
        use epoll;
    }

    http {
        access_log /dev/stdout combined;

        # Aggressive upstream keepalive — RunPod pods are remote HTTPS
        upstream runpod_inference {
            least_conn;

            # ----------------------------------------------------------------
            # Add one line per active RunPod pod endpoint.
            # Format:  server <pod-id>-8000.proxy.runpod.net:443;
            # Example:
            #   server abc123def456-8000.proxy.runpod.net:443;
            #   server xyz789uvw012-8000.proxy.runpod.net:443;
            # ----------------------------------------------------------------

            keepalive 32;
            keepalive_requests 1000;
            keepalive_timeout 75s;
        }

        # Health-check upstream (one known-good pod for active probing)
        # Replace with real pod ID.
        upstream runpod_healthcheck {
            server REPLACE_POD_ID-8000.proxy.runpod.net:443;
        }

        server {
            listen 8080;
            server_name _;

            # Proxy timeouts generous for LLM inference
            proxy_read_timeout    300s;
            proxy_send_timeout     60s;
            proxy_connect_timeout  10s;

            # ---- OpenAI-compatible inference passthrough ----
            location /v1/ {
                proxy_pass          https://runpod_inference;
                proxy_http_version  1.1;
                proxy_set_header    Connection      "";
                proxy_set_header    Host            $proxy_host;
                proxy_set_header    X-Real-IP       $remote_addr;
                proxy_set_header    X-Forwarded-For $proxy_add_x_forwarded_for;

                # Stream tokens back to caller without buffering
                proxy_buffering         off;
                proxy_cache             off;
                chunked_transfer_encoding on;
            }

            # ---- Cluster-internal health probe ----
            location /health {
                return 200 'ok\n';
                add_header Content-Type text/plain;
            }

            # ---- Upstream model info (routed to first healthy pod) ----
            location /v1/models {
                proxy_pass          https://runpod_inference;
                proxy_http_version  1.1;
                proxy_set_header    Connection "";
                proxy_set_header    Host       $proxy_host;
            }
        }
    }

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: runpod-lb
  namespace: neuron-prod
  labels:
    app.kubernetes.io/name: runpod-lb
    app.kubernetes.io/component: load-balancer
    app.kubernetes.io/managed-by: argocd
spec:
  # Start at 0 — scale up after runpod-provision.sh creates pods and upstream
  # servers are populated in the nginx ConfigMap.
  replicas: 0
  selector:
    matchLabels:
      app: runpod-lb
  template:
    metadata:
      labels:
        app: runpod-lb
    spec:
      containers:
        - name: nginx
          image: nginx:1.27-alpine
          ports:
            - containerPort: 8080
              name: http
          volumeMounts:
            - name: nginx-config
              mountPath: /etc/nginx/nginx.conf
              subPath: nginx.conf
              readOnly: true
          livenessProbe:
            httpGet:
              path: /health
              port: 8080
            initialDelaySeconds: 5
            periodSeconds: 15
          readinessProbe:
            httpGet:
              path: /health
              port: 8080
            initialDelaySeconds: 3
            periodSeconds: 10
          resources:
            requests:
              cpu: "50m"
              memory: "64Mi"
            limits:
              cpu: "200m"
              memory: "128Mi"
          securityContext:
            runAsNonRoot: true
            runAsUser: 101
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: false
      volumes:
        - name: nginx-config
          configMap:
            name: runpod-lb-nginx-config

---
apiVersion: v1
kind: Service
metadata:
  name: runpod-lb
  namespace: neuron-prod
  labels:
    app.kubernetes.io/name: runpod-lb
    app.kubernetes.io/component: load-balancer
spec:
  selector:
    app: runpod-lb
  ports:
    - name: http
      port: 80
      targetPort: 8080
  type: ClusterIP