will.anderson/infrastructure
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(runpod): add inference pod templates, nginx LB, and provisioner script

Browse Source 
Infrastructure readiness for RunPod inference workloads:
- runpod-inference.yaml: ConfigMap with pod creation payloads for RTX 4090,
  A40 (single+dual), and custom templates
- runpod-lb-configmap.yaml: nginx least-conn load balancer for inference
  endpoint distribution (Deployment + ClusterIP Service)
- runpod-provision.sh: bash provisioner script — reads RUNPOD_API_KEY/HF_TOKEN,
  creates pods via GraphQL, polls until RUNNING, outputs endpoint URLs.
  Does NOT spin up any pods (dry-run flag available).
This commit is contained in:
Will Anderson
2026-04-25 01:20:55 -05:00
parent c847b22014
commit 8eb88a3116
4 changed files with 566 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
servers/legion/k8s/neuron-technologies/prod/kustomization.yaml
+2
View File
@@ -12,3 +12,5 @@ resources:
- ingress.yaml
- license-ingressroute.yaml
- backup-cronjob.yaml
- runpod-inference.yaml
- runpod-lb-configmap.yaml
servers/legion/k8s/neuron-technologies/prod/runpod-inference.yaml
+149
View File
@@ -0,0 +1,149 @@
---
# RunPod Inference ConfigMap
# Holds pod creation payloads for on-demand vLLM inference pods.
# Used by the runpod-provision.sh script and any k8s Job that provisions RunPod pods.
#
# GPU IDs sourced from RunPod GraphQL { gpuTypes { id } } — verified 2026-04-25.
# RTX 4090: "NVIDIA GeForce RTX 4090" (24 GB VRAM, secure+community cloud)
# A40: "NVIDIA A40" (48 GB VRAM, secure cloud only)
# L40S: "NVIDIA L40S" (48 GB VRAM, secure+community cloud)
apiVersion: v1
kind: ConfigMap
metadata:
name: runpod-inference-templates
namespace: neuron-prod
labels:
app.kubernetes.io/name: runpod-inference
app.kubernetes.io/component: inference-provisioner
app.kubernetes.io/managed-by: argocd
data:
# --- Llama-3-8B on RTX 4090 (24 GB) ---
# Fits quantised (GPTQ/AWQ) or full fp16 8B models.
pod-template-rtx4090-llama3-8b.json: |
{
"name": "neuron-llama3-8b-rtx4090",
"imageName": "vllm/vllm-openai:latest",
"gpuTypeId": "NVIDIA GeForce RTX 4090",
"cloudType": "SECURE",
"gpuCount": 1,
"volumeInGb": 80,
"containerDiskInGb": 30,
"minVcpuCount": 8,
"minMemoryInGb": 32,
"ports": "8000/http",
"env": [
{ "key": "MODEL_ID", "value": "meta-llama/Meta-Llama-3-8B-Instruct" },
{ "key": "QUANTIZATION", "value": "awq" },
{ "key": "MAX_MODEL_LEN", "value": "8192" },
{ "key": "TENSOR_PARALLEL_SIZE","value": "1" },
{ "key": "GPU_MEMORY_UTILIZATION", "value": "0.92" },
{ "key": "SERVED_MODEL_NAME", "value": "llama3-8b" },
{ "key": "HF_TOKEN", "value": "REPLACE_WITH_HF_TOKEN" }
],
"dockerArgs": "--host 0.0.0.0 --port 8000 --model $MODEL_ID --quantization $QUANTIZATION --max-model-len $MAX_MODEL_LEN --tensor-parallel-size $TENSOR_PARALLEL_SIZE --gpu-memory-utilization $GPU_MEMORY_UTILIZATION --served-model-name $SERVED_MODEL_NAME"
}
# --- Llama-3-70B on A40 (48 GB) — single GPU with AWQ quantisation ---
pod-template-a40-llama3-70b.json: |
{
"name": "neuron-llama3-70b-a40",
"imageName": "vllm/vllm-openai:latest",
"gpuTypeId": "NVIDIA A40",
"cloudType": "SECURE",
"gpuCount": 1,
"volumeInGb": 150,
"containerDiskInGb": 50,
"minVcpuCount": 16,
"minMemoryInGb": 64,
"ports": "8000/http",
"env": [
{ "key": "MODEL_ID", "value": "meta-llama/Meta-Llama-3-70B-Instruct" },
{ "key": "QUANTIZATION", "value": "awq" },
{ "key": "MAX_MODEL_LEN", "value": "4096" },
{ "key": "TENSOR_PARALLEL_SIZE","value": "1" },
{ "key": "GPU_MEMORY_UTILIZATION", "value": "0.90" },
{ "key": "SERVED_MODEL_NAME", "value": "llama3-70b" },
{ "key": "HF_TOKEN", "value": "REPLACE_WITH_HF_TOKEN" }
],
"dockerArgs": "--host 0.0.0.0 --port 8000 --model $MODEL_ID --quantization $QUANTIZATION --max-model-len $MAX_MODEL_LEN --tensor-parallel-size $TENSOR_PARALLEL_SIZE --gpu-memory-utilization $GPU_MEMORY_UTILIZATION --served-model-name $SERVED_MODEL_NAME"
}
# --- Llama-3-70B on 2x A40 (tensor parallel) ---
pod-template-2xa40-llama3-70b-fp16.json: |
{
"name": "neuron-llama3-70b-2xa40-fp16",
"imageName": "vllm/vllm-openai:latest",
"gpuTypeId": "NVIDIA A40",
"cloudType": "SECURE",
"gpuCount": 2,
"volumeInGb": 150,
"containerDiskInGb": 50,
"minVcpuCount": 16,
"minMemoryInGb": 64,
"ports": "8000/http",
"env": [
{ "key": "MODEL_ID", "value": "meta-llama/Meta-Llama-3-70B-Instruct" },
{ "key": "QUANTIZATION", "value": "" },
{ "key": "MAX_MODEL_LEN", "value": "8192" },
{ "key": "TENSOR_PARALLEL_SIZE","value": "2" },
{ "key": "GPU_MEMORY_UTILIZATION", "value": "0.90" },
{ "key": "SERVED_MODEL_NAME", "value": "llama3-70b" },
{ "key": "HF_TOKEN", "value": "REPLACE_WITH_HF_TOKEN" }
],
"dockerArgs": "--host 0.0.0.0 --port 8000 --model $MODEL_ID --max-model-len $MAX_MODEL_LEN --tensor-parallel-size $TENSOR_PARALLEL_SIZE --gpu-memory-utilization $GPU_MEMORY_UTILIZATION --served-model-name $SERVED_MODEL_NAME"
}
# --- Generic custom model template ---
pod-template-custom.json: |
{
"name": "neuron-inference-custom",
"imageName": "vllm/vllm-openai:latest",
"gpuTypeId": "NVIDIA GeForce RTX 4090",
"cloudType": "SECURE",
"gpuCount": 1,
"volumeInGb": 100,
"containerDiskInGb": 30,
"minVcpuCount": 8,
"minMemoryInGb": 32,
"ports": "8000/http",
"env": [
{ "key": "MODEL_ID", "value": "REPLACE_WITH_MODEL_ID" },
{ "key": "QUANTIZATION", "value": "awq" },
{ "key": "MAX_MODEL_LEN", "value": "4096" },
{ "key": "TENSOR_PARALLEL_SIZE","value": "1" },
{ "key": "GPU_MEMORY_UTILIZATION", "value": "0.90" },
{ "key": "SERVED_MODEL_NAME", "value": "model" },
{ "key": "HF_TOKEN", "value": "REPLACE_WITH_HF_TOKEN" }
],
"dockerArgs": "--host 0.0.0.0 --port 8000 --model $MODEL_ID --quantization $QUANTIZATION --max-model-len $MAX_MODEL_LEN --tensor-parallel-size $TENSOR_PARALLEL_SIZE --gpu-memory-utilization $GPU_MEMORY_UTILIZATION --served-model-name $SERVED_MODEL_NAME"
}
# --- Nginx load balancer config (updated dynamically by runpod-provision.sh) ---
# Paste pod endpoint URLs under upstream block when pods are running.
nginx-lb.conf: |
upstream runpod_inference {
least_conn;
# Populated dynamically — add lines like:
# server <pod-id>-8000.proxy.runpod.net:443;
keepalive 32;
}
server {
listen 80;
server_name inference.neuralplatform.ai;
location /v1/ {
proxy_pass https://runpod_inference;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $proxy_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_read_timeout 300s;
proxy_send_timeout 60s;
}
location /health {
return 200 'ok';
add_header Content-Type text/plain;
}
}
servers/legion/k8s/neuron-technologies/prod/runpod-lb-configmap.yaml
+172
View File
@@ -0,0 +1,172 @@
---
# RunPod Inference Load Balancer — Nginx ConfigMap
#
# Deploys a lightweight nginx reverse proxy inside the cluster that fans out
# requests to multiple RunPod inference endpoints. Update the upstream block
# with real pod proxy URLs after running runpod-provision.sh.
#
# Reload nginx after updating endpoints:
# kubectl rollout restart deployment/runpod-lb -n neuron-prod
apiVersion: v1
kind: ConfigMap
metadata:
name: runpod-lb-nginx-config
namespace: neuron-prod
labels:
app.kubernetes.io/name: runpod-lb
app.kubernetes.io/component: load-balancer
app.kubernetes.io/managed-by: argocd
data:
nginx.conf: |
worker_processes auto;
error_log /dev/stderr warn;
pid /tmp/nginx.pid;
events {
worker_connections 1024;
use epoll;
}
http {
access_log /dev/stdout combined;
# Aggressive upstream keepalive — RunPod pods are remote HTTPS
upstream runpod_inference {
least_conn;
# ----------------------------------------------------------------
# Add one line per active RunPod pod endpoint.
# Format: server <pod-id>-8000.proxy.runpod.net:443;
# Example:
# server abc123def456-8000.proxy.runpod.net:443;
# server xyz789uvw012-8000.proxy.runpod.net:443;
# ----------------------------------------------------------------
keepalive 32;
keepalive_requests 1000;
keepalive_timeout 75s;
}
# Health-check upstream (one known-good pod for active probing)
# Replace with real pod ID.
upstream runpod_healthcheck {
server REPLACE_POD_ID-8000.proxy.runpod.net:443;
}
server {
listen 8080;
server_name _;
# Proxy timeouts generous for LLM inference
proxy_read_timeout 300s;
proxy_send_timeout 60s;
proxy_connect_timeout 10s;
# ---- OpenAI-compatible inference passthrough ----
location /v1/ {
proxy_pass https://runpod_inference;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $proxy_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# Stream tokens back to caller without buffering
proxy_buffering off;
proxy_cache off;
chunked_transfer_encoding on;
}
# ---- Cluster-internal health probe ----
location /health {
return 200 'ok\n';
add_header Content-Type text/plain;
}
# ---- Upstream model info (routed to first healthy pod) ----
location /v1/models {
proxy_pass https://runpod_inference;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $proxy_host;
}
}
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: runpod-lb
namespace: neuron-prod
labels:
app.kubernetes.io/name: runpod-lb
app.kubernetes.io/component: load-balancer
app.kubernetes.io/managed-by: argocd
spec:
replicas: 1
selector:
matchLabels:
app: runpod-lb
template:
metadata:
labels:
app: runpod-lb
spec:
containers:
- name: nginx
image: nginx:1.27-alpine
ports:
- containerPort: 8080
name: http
volumeMounts:
- name: nginx-config
mountPath: /etc/nginx/nginx.conf
subPath: nginx.conf
readOnly: true
livenessProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 5
periodSeconds: 15
readinessProbe:
httpGet:
path: /health
port: 8080
initialDelaySeconds: 3
periodSeconds: 10
resources:
requests:
cpu: "50m"
memory: "64Mi"
limits:
cpu: "200m"
memory: "128Mi"
securityContext:
runAsNonRoot: true
runAsUser: 101
allowPrivilegeEscalation: false
readOnlyRootFilesystem: false
volumes:
- name: nginx-config
configMap:
name: runpod-lb-nginx-config
---
apiVersion: v1
kind: Service
metadata:
name: runpod-lb
namespace: neuron-prod
labels:
app.kubernetes.io/name: runpod-lb
app.kubernetes.io/component: load-balancer
spec:
selector:
app: runpod-lb
ports:
- name: http
port: 80
targetPort: 8080
type: ClusterIP