feat(runpod): add inference pod templates, nginx LB, and provisioner script
Infrastructure readiness for RunPod inference workloads: - runpod-inference.yaml: ConfigMap with pod creation payloads for RTX 4090, A40 (single+dual), and custom templates - runpod-lb-configmap.yaml: nginx least-conn load balancer for inference endpoint distribution (Deployment + ClusterIP Service) - runpod-provision.sh: bash provisioner script — reads RUNPOD_API_KEY/HF_TOKEN, creates pods via GraphQL, polls until RUNNING, outputs endpoint URLs. Does NOT spin up any pods (dry-run flag available).
This commit is contained in:
Will Anderson
@@ -0,0 +1,149 @@
|
||||
---
|
||||
# RunPod Inference ConfigMap
|
||||
# Holds pod creation payloads for on-demand vLLM inference pods.
|
||||
# Used by the runpod-provision.sh script and any k8s Job that provisions RunPod pods.
|
||||
#
|
||||
# GPU IDs sourced from RunPod GraphQL { gpuTypes { id } } — verified 2026-04-25.
|
||||
# RTX 4090: "NVIDIA GeForce RTX 4090" (24 GB VRAM, secure+community cloud)
|
||||
# A40: "NVIDIA A40" (48 GB VRAM, secure cloud only)
|
||||
# L40S: "NVIDIA L40S" (48 GB VRAM, secure+community cloud)
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: runpod-inference-templates
|
||||
namespace: neuron-prod
|
||||
labels:
|
||||
app.kubernetes.io/name: runpod-inference
|
||||
app.kubernetes.io/component: inference-provisioner
|
||||
app.kubernetes.io/managed-by: argocd
|
||||
data:
|
||||
# --- Llama-3-8B on RTX 4090 (24 GB) ---
|
||||
# Fits quantised (GPTQ/AWQ) or full fp16 8B models.
|
||||
pod-template-rtx4090-llama3-8b.json: |
|
||||
{
|
||||
"name": "neuron-llama3-8b-rtx4090",
|
||||
"imageName": "vllm/vllm-openai:latest",
|
||||
"gpuTypeId": "NVIDIA GeForce RTX 4090",
|
||||
"cloudType": "SECURE",
|
||||
"gpuCount": 1,
|
||||
"volumeInGb": 80,
|
||||
"containerDiskInGb": 30,
|
||||
"minVcpuCount": 8,
|
||||
"minMemoryInGb": 32,
|
||||
"ports": "8000/http",
|
||||
"env": [
|
||||
{ "key": "MODEL_ID", "value": "meta-llama/Meta-Llama-3-8B-Instruct" },
|
||||
{ "key": "QUANTIZATION", "value": "awq" },
|
||||
{ "key": "MAX_MODEL_LEN", "value": "8192" },
|
||||
{ "key": "TENSOR_PARALLEL_SIZE","value": "1" },
|
||||
{ "key": "GPU_MEMORY_UTILIZATION", "value": "0.92" },
|
||||
{ "key": "SERVED_MODEL_NAME", "value": "llama3-8b" },
|
||||
{ "key": "HF_TOKEN", "value": "REPLACE_WITH_HF_TOKEN" }
|
||||
],
|
||||
"dockerArgs": "--host 0.0.0.0 --port 8000 --model $MODEL_ID --quantization $QUANTIZATION --max-model-len $MAX_MODEL_LEN --tensor-parallel-size $TENSOR_PARALLEL_SIZE --gpu-memory-utilization $GPU_MEMORY_UTILIZATION --served-model-name $SERVED_MODEL_NAME"
|
||||
}
|
||||
|
||||
# --- Llama-3-70B on A40 (48 GB) — single GPU with AWQ quantisation ---
|
||||
pod-template-a40-llama3-70b.json: |
|
||||
{
|
||||
"name": "neuron-llama3-70b-a40",
|
||||
"imageName": "vllm/vllm-openai:latest",
|
||||
"gpuTypeId": "NVIDIA A40",
|
||||
"cloudType": "SECURE",
|
||||
"gpuCount": 1,
|
||||
"volumeInGb": 150,
|
||||
"containerDiskInGb": 50,
|
||||
"minVcpuCount": 16,
|
||||
"minMemoryInGb": 64,
|
||||
"ports": "8000/http",
|
||||
"env": [
|
||||
{ "key": "MODEL_ID", "value": "meta-llama/Meta-Llama-3-70B-Instruct" },
|
||||
{ "key": "QUANTIZATION", "value": "awq" },
|
||||
{ "key": "MAX_MODEL_LEN", "value": "4096" },
|
||||
{ "key": "TENSOR_PARALLEL_SIZE","value": "1" },
|
||||
{ "key": "GPU_MEMORY_UTILIZATION", "value": "0.90" },
|
||||
{ "key": "SERVED_MODEL_NAME", "value": "llama3-70b" },
|
||||
{ "key": "HF_TOKEN", "value": "REPLACE_WITH_HF_TOKEN" }
|
||||
],
|
||||
"dockerArgs": "--host 0.0.0.0 --port 8000 --model $MODEL_ID --quantization $QUANTIZATION --max-model-len $MAX_MODEL_LEN --tensor-parallel-size $TENSOR_PARALLEL_SIZE --gpu-memory-utilization $GPU_MEMORY_UTILIZATION --served-model-name $SERVED_MODEL_NAME"
|
||||
}
|
||||
|
||||
# --- Llama-3-70B on 2x A40 (tensor parallel) ---
|
||||
pod-template-2xa40-llama3-70b-fp16.json: |
|
||||
{
|
||||
"name": "neuron-llama3-70b-2xa40-fp16",
|
||||
"imageName": "vllm/vllm-openai:latest",
|
||||
"gpuTypeId": "NVIDIA A40",
|
||||
"cloudType": "SECURE",
|
||||
"gpuCount": 2,
|
||||
"volumeInGb": 150,
|
||||
"containerDiskInGb": 50,
|
||||
"minVcpuCount": 16,
|
||||
"minMemoryInGb": 64,
|
||||
"ports": "8000/http",
|
||||
"env": [
|
||||
{ "key": "MODEL_ID", "value": "meta-llama/Meta-Llama-3-70B-Instruct" },
|
||||
{ "key": "QUANTIZATION", "value": "" },
|
||||
{ "key": "MAX_MODEL_LEN", "value": "8192" },
|
||||
{ "key": "TENSOR_PARALLEL_SIZE","value": "2" },
|
||||
{ "key": "GPU_MEMORY_UTILIZATION", "value": "0.90" },
|
||||
{ "key": "SERVED_MODEL_NAME", "value": "llama3-70b" },
|
||||
{ "key": "HF_TOKEN", "value": "REPLACE_WITH_HF_TOKEN" }
|
||||
],
|
||||
"dockerArgs": "--host 0.0.0.0 --port 8000 --model $MODEL_ID --max-model-len $MAX_MODEL_LEN --tensor-parallel-size $TENSOR_PARALLEL_SIZE --gpu-memory-utilization $GPU_MEMORY_UTILIZATION --served-model-name $SERVED_MODEL_NAME"
|
||||
}
|
||||
|
||||
# --- Generic custom model template ---
|
||||
pod-template-custom.json: |
|
||||
{
|
||||
"name": "neuron-inference-custom",
|
||||
"imageName": "vllm/vllm-openai:latest",
|
||||
"gpuTypeId": "NVIDIA GeForce RTX 4090",
|
||||
"cloudType": "SECURE",
|
||||
"gpuCount": 1,
|
||||
"volumeInGb": 100,
|
||||
"containerDiskInGb": 30,
|
||||
"minVcpuCount": 8,
|
||||
"minMemoryInGb": 32,
|
||||
"ports": "8000/http",
|
||||
"env": [
|
||||
{ "key": "MODEL_ID", "value": "REPLACE_WITH_MODEL_ID" },
|
||||
{ "key": "QUANTIZATION", "value": "awq" },
|
||||
{ "key": "MAX_MODEL_LEN", "value": "4096" },
|
||||
{ "key": "TENSOR_PARALLEL_SIZE","value": "1" },
|
||||
{ "key": "GPU_MEMORY_UTILIZATION", "value": "0.90" },
|
||||
{ "key": "SERVED_MODEL_NAME", "value": "model" },
|
||||
{ "key": "HF_TOKEN", "value": "REPLACE_WITH_HF_TOKEN" }
|
||||
],
|
||||
"dockerArgs": "--host 0.0.0.0 --port 8000 --model $MODEL_ID --quantization $QUANTIZATION --max-model-len $MAX_MODEL_LEN --tensor-parallel-size $TENSOR_PARALLEL_SIZE --gpu-memory-utilization $GPU_MEMORY_UTILIZATION --served-model-name $SERVED_MODEL_NAME"
|
||||
}
|
||||
|
||||
# --- Nginx load balancer config (updated dynamically by runpod-provision.sh) ---
|
||||
# Paste pod endpoint URLs under upstream block when pods are running.
|
||||
nginx-lb.conf: |
|
||||
upstream runpod_inference {
|
||||
least_conn;
|
||||
# Populated dynamically — add lines like:
|
||||
# server <pod-id>-8000.proxy.runpod.net:443;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name inference.neuralplatform.ai;
|
||||
|
||||
location /v1/ {
|
||||
proxy_pass https://runpod_inference;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $proxy_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_read_timeout 300s;
|
||||
proxy_send_timeout 60s;
|
||||
}
|
||||
|
||||
location /health {
|
||||
return 200 'ok';
|
||||
add_header Content-Type text/plain;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user