infrastructure/servers/legion/k8s/neuron-technologies/prod/runpod-inference.yaml

---
# RunPod Inference ConfigMap
# Holds pod creation payloads for on-demand vLLM inference pods.
# Used by the runpod-provision.sh script and any k8s Job that provisions RunPod pods.
#
# GPU IDs sourced from RunPod GraphQL { gpuTypes { id } } — verified 2026-04-25.
# RTX 4090:  "NVIDIA GeForce RTX 4090"  (24 GB VRAM, secure+community cloud)
# A40:       "NVIDIA A40"               (48 GB VRAM, secure cloud only)
# L40S:      "NVIDIA L40S"              (48 GB VRAM, secure+community cloud)
apiVersion: v1
kind: ConfigMap
metadata:
  name: runpod-inference-templates
  namespace: neuron-prod
  labels:
    app.kubernetes.io/name: runpod-inference
    app.kubernetes.io/component: inference-provisioner
    app.kubernetes.io/managed-by: argocd
data:
  # --- Llama-3-8B on RTX 4090 (24 GB) ---
  # Fits quantised (GPTQ/AWQ) or full fp16 8B models.
  pod-template-rtx4090-llama3-8b.json: |
    {
      "name": "neuron-llama3-8b-rtx4090",
      "imageName": "vllm/vllm-openai:latest",
      "gpuTypeId": "NVIDIA GeForce RTX 4090",
      "cloudType": "SECURE",
      "gpuCount": 1,
      "volumeInGb": 80,
      "containerDiskInGb": 30,
      "minVcpuCount": 8,
      "minMemoryInGb": 32,
      "ports": "8000/http",
      "env": [
        { "key": "MODEL_ID",            "value": "meta-llama/Meta-Llama-3-8B-Instruct" },
        { "key": "QUANTIZATION",        "value": "awq" },
        { "key": "MAX_MODEL_LEN",       "value": "8192" },
        { "key": "TENSOR_PARALLEL_SIZE","value": "1" },
        { "key": "GPU_MEMORY_UTILIZATION", "value": "0.92" },
        { "key": "SERVED_MODEL_NAME",   "value": "llama3-8b" },
        { "key": "HF_TOKEN",            "value": "REPLACE_WITH_HF_TOKEN" }
      ],
      "dockerArgs": "--host 0.0.0.0 --port 8000 --model $MODEL_ID --quantization $QUANTIZATION --max-model-len $MAX_MODEL_LEN --tensor-parallel-size $TENSOR_PARALLEL_SIZE --gpu-memory-utilization $GPU_MEMORY_UTILIZATION --served-model-name $SERVED_MODEL_NAME"
    }

  # --- Llama-3-70B on A40 (48 GB) — single GPU with AWQ quantisation ---
  pod-template-a40-llama3-70b.json: |
    {
      "name": "neuron-llama3-70b-a40",
      "imageName": "vllm/vllm-openai:latest",
      "gpuTypeId": "NVIDIA A40",
      "cloudType": "SECURE",
      "gpuCount": 1,
      "volumeInGb": 150,
      "containerDiskInGb": 50,
      "minVcpuCount": 16,
      "minMemoryInGb": 64,
      "ports": "8000/http",
      "env": [
        { "key": "MODEL_ID",            "value": "meta-llama/Meta-Llama-3-70B-Instruct" },
        { "key": "QUANTIZATION",        "value": "awq" },
        { "key": "MAX_MODEL_LEN",       "value": "4096" },
        { "key": "TENSOR_PARALLEL_SIZE","value": "1" },
        { "key": "GPU_MEMORY_UTILIZATION", "value": "0.90" },
        { "key": "SERVED_MODEL_NAME",   "value": "llama3-70b" },
        { "key": "HF_TOKEN",            "value": "REPLACE_WITH_HF_TOKEN" }
      ],
      "dockerArgs": "--host 0.0.0.0 --port 8000 --model $MODEL_ID --quantization $QUANTIZATION --max-model-len $MAX_MODEL_LEN --tensor-parallel-size $TENSOR_PARALLEL_SIZE --gpu-memory-utilization $GPU_MEMORY_UTILIZATION --served-model-name $SERVED_MODEL_NAME"
    }

  # --- Llama-3-70B on 2x A40 (tensor parallel) ---
  pod-template-2xa40-llama3-70b-fp16.json: |
    {
      "name": "neuron-llama3-70b-2xa40-fp16",
      "imageName": "vllm/vllm-openai:latest",
      "gpuTypeId": "NVIDIA A40",
      "cloudType": "SECURE",
      "gpuCount": 2,
      "volumeInGb": 150,
      "containerDiskInGb": 50,
      "minVcpuCount": 16,
      "minMemoryInGb": 64,
      "ports": "8000/http",
      "env": [
        { "key": "MODEL_ID",            "value": "meta-llama/Meta-Llama-3-70B-Instruct" },
        { "key": "QUANTIZATION",        "value": "" },
        { "key": "MAX_MODEL_LEN",       "value": "8192" },
        { "key": "TENSOR_PARALLEL_SIZE","value": "2" },
        { "key": "GPU_MEMORY_UTILIZATION", "value": "0.90" },
        { "key": "SERVED_MODEL_NAME",   "value": "llama3-70b" },
        { "key": "HF_TOKEN",            "value": "REPLACE_WITH_HF_TOKEN" }
      ],
      "dockerArgs": "--host 0.0.0.0 --port 8000 --model $MODEL_ID --max-model-len $MAX_MODEL_LEN --tensor-parallel-size $TENSOR_PARALLEL_SIZE --gpu-memory-utilization $GPU_MEMORY_UTILIZATION --served-model-name $SERVED_MODEL_NAME"
    }

  # --- Generic custom model template ---
  pod-template-custom.json: |
    {
      "name": "neuron-inference-custom",
      "imageName": "vllm/vllm-openai:latest",
      "gpuTypeId": "NVIDIA GeForce RTX 4090",
      "cloudType": "SECURE",
      "gpuCount": 1,
      "volumeInGb": 100,
      "containerDiskInGb": 30,
      "minVcpuCount": 8,
      "minMemoryInGb": 32,
      "ports": "8000/http",
      "env": [
        { "key": "MODEL_ID",            "value": "REPLACE_WITH_MODEL_ID" },
        { "key": "QUANTIZATION",        "value": "awq" },
        { "key": "MAX_MODEL_LEN",       "value": "4096" },
        { "key": "TENSOR_PARALLEL_SIZE","value": "1" },
        { "key": "GPU_MEMORY_UTILIZATION", "value": "0.90" },
        { "key": "SERVED_MODEL_NAME",   "value": "model" },
        { "key": "HF_TOKEN",            "value": "REPLACE_WITH_HF_TOKEN" }
      ],
      "dockerArgs": "--host 0.0.0.0 --port 8000 --model $MODEL_ID --quantization $QUANTIZATION --max-model-len $MAX_MODEL_LEN --tensor-parallel-size $TENSOR_PARALLEL_SIZE --gpu-memory-utilization $GPU_MEMORY_UTILIZATION --served-model-name $SERVED_MODEL_NAME"
    }

  # --- Nginx load balancer config (updated dynamically by runpod-provision.sh) ---
  # Paste pod endpoint URLs under upstream block when pods are running.
  nginx-lb.conf: |
    upstream runpod_inference {
        least_conn;
        # Populated dynamically — add lines like:
        #   server <pod-id>-8000.proxy.runpod.net:443;
        keepalive 32;
    }

    server {
        listen 80;
        server_name inference.neuralplatform.ai;

        location /v1/ {
            proxy_pass https://runpod_inference;
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            proxy_set_header Host $proxy_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_read_timeout 300s;
            proxy_send_timeout 60s;
        }

        location /health {
            return 200 'ok';
            add_header Content-Type text/plain;
        }
    }