From e12fd9781a1e04347c2259aaac1a260b402bdc17 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Wed, 15 Apr 2026 01:40:02 -0500 Subject: [PATCH] Fix VPN routing: give portforward-helper NET_ADMIN to delete gluetun rule 100 gluetun adds 'ip rule priority 100 from lookup 200' which routes all pod traffic via eth0. The iptables OUTPUT DROP policy then blocks it. Rule 101 (not fwmark 0xca6c -> tun0) never fires because rule 100 matches first. Result: qBittorrent has no internet access through the VPN. Fix: portforward-helper waits for gluetun to be ready (polls :9999), then deletes rule 100. Application traffic then falls through to rule 101 and routes correctly via tun0. --- .../k8s/fornax/worker/base/deployment.yaml | 16 ++++-- servers/legion/k8s/media/fornax-workers.yaml | 55 ++++++++++++++++--- 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/servers/legion/k8s/fornax/worker/base/deployment.yaml b/servers/legion/k8s/fornax/worker/base/deployment.yaml index c14b7b9..23a383d 100644 --- a/servers/legion/k8s/fornax/worker/base/deployment.yaml +++ b/servers/legion/k8s/fornax/worker/base/deployment.yaml @@ -107,6 +107,9 @@ spec: - name: portforward-helper image: alpine:latest imagePullPolicy: Always + securityContext: + capabilities: + add: ["NET_ADMIN"] env: - name: COORDINATOR_URL value: "https://fornax.neuralplatform.ai" @@ -116,6 +119,14 @@ spec: - sh - -c - | + echo "Waiting for gluetun health endpoint..." + while ! wget -q -T 3 -O- http://127.0.0.1:9999 > /dev/null 2>&1; do sleep 3; done + echo "gluetun ready" + # gluetun adds 'ip rule priority 100 from lookup 200' which routes + # all pod traffic via eth0, bypassing tun0. The iptables OUTPUT DROP policy + # then blocks it. Removing rule 100 lets traffic fall through to priority 101 + # (not fwmark 0xca6c -> table 51820 -> tun0). + ip rule del priority 100 2>/dev/null && echo "Fixed VPN routing (removed rule 100)" || echo "Rule 100 not present" until apk add -q curl 2>/dev/null; do sleep 5; done echo "Watching /tmp/gluetun/forwarded_port for assigned port..." TTL=300 @@ -129,18 +140,13 @@ spec: curl -s -b /tmp/qbt.txt -X POST http://localhost:8080/api/v2/app/setPreferences \ -d "json={\"listen_port\":$PORT,\"random_port\":false}" >/dev/null 2>&1 echo "$(date): qBittorrent listen port set to $PORT" - # Notify coordinator of port lease if [ -n "$COORDINATOR_URL" ] && [ -n "$WORKER_ID" ]; then curl -s -X POST "$COORDINATOR_URL/api/v2/workers/$WORKER_ID/port-lease" \ -H "Content-Type: application/json" \ -d "{\"port\": $PORT, \"ttlSeconds\": $TTL}" >/dev/null 2>&1 || true echo "$(date): Coordinator notified of port $PORT (TTL: ${TTL}s)" fi - else - echo "$(date): Port file empty, waiting..." fi - else - echo "$(date): Waiting for gluetun to write port file..." fi sleep 45 done diff --git a/servers/legion/k8s/media/fornax-workers.yaml b/servers/legion/k8s/media/fornax-workers.yaml index faa58fc..39850ce 100644 --- a/servers/legion/k8s/media/fornax-workers.yaml +++ b/servers/legion/k8s/media/fornax-workers.yaml @@ -133,15 +133,32 @@ spec: memory: 512Mi cpu: 200m - # Port forwarding helper — reads port from file gluetun writes to /tmp/gluetun/forwarded_port + # Port forwarding helper — fixes gluetun routing rule then watches for VPN port assignment - name: portforward-helper image: alpine:latest + securityContext: + capabilities: + add: ["NET_ADMIN"] command: - sh - -c - | + # Wait for gluetun to finish setting up its routing rules + echo "Waiting for gluetun health endpoint..." + while ! wget -q -T 3 -O- http://127.0.0.1:9999 > /dev/null 2>&1; do sleep 3; done + echo "gluetun ready" + + # gluetun adds 'ip rule priority 100 from lookup 200' which routes + # ALL pod traffic through eth0, bypassing tun0. The iptables OUTPUT DROP policy + # then blocks non-VPN, non-cluster traffic. Removing rule 100 allows traffic to + # fall through to 'priority 101 not fwmark 0xca6c lookup 51820' (tun0/VPN). + ip rule del priority 100 2>/dev/null && echo "Fixed VPN routing (removed rule 100)" || echo "Rule 100 not present" + + # Install curl (succeeds now that VPN traffic routes correctly) until apk add -q curl 2>/dev/null; do sleep 5; done + echo "Watching /tmp/gluetun/forwarded_port for assigned port..." + TTL=300 while true; do if [ -f /tmp/gluetun/forwarded_port ]; then PORT=$(cat /tmp/gluetun/forwarded_port) @@ -152,14 +169,20 @@ spec: curl -s -b /tmp/qbt.txt -X POST http://localhost:8080/api/v2/app/setPreferences \ -d "json={\"listen_port\":$PORT,\"random_port\":false}" >/dev/null 2>&1 echo "$(date): qBittorrent listen port set to $PORT" - else - echo "$(date): Port file empty, waiting..." + if [ -n "$COORDINATOR_URL" ] && [ -n "$WORKER_ID" ]; then + curl -s -X POST "$COORDINATOR_URL/api/v2/workers/$WORKER_ID/port-lease" \ + -H "Content-Type: application/json" \ + -d "{\"port\": $PORT, \"ttlSeconds\": $TTL}" >/dev/null 2>&1 || true + fi fi - else - echo "$(date): Waiting for gluetun to write port file..." fi sleep 45 done + env: + - name: COORDINATOR_URL + value: "https://fornax.neuralplatform.ai" + - name: WORKER_ID + value: "tx253" volumeMounts: - name: gluetun-data mountPath: /tmp/gluetun @@ -355,12 +378,20 @@ spec: - name: portforward-helper image: alpine:latest + securityContext: + capabilities: + add: ["NET_ADMIN"] command: - sh - -c - | + echo "Waiting for gluetun health endpoint..." + while ! wget -q -T 3 -O- http://127.0.0.1:9999 > /dev/null 2>&1; do sleep 3; done + echo "gluetun ready" + ip rule del priority 100 2>/dev/null && echo "Fixed VPN routing (removed rule 100)" || echo "Rule 100 not present" until apk add -q curl 2>/dev/null; do sleep 5; done echo "Watching /tmp/gluetun/forwarded_port for assigned port..." + TTL=300 while true; do if [ -f /tmp/gluetun/forwarded_port ]; then PORT=$(cat /tmp/gluetun/forwarded_port) @@ -371,14 +402,20 @@ spec: curl -s -b /tmp/qbt.txt -X POST http://localhost:8080/api/v2/app/setPreferences \ -d "json={\"listen_port\":$PORT,\"random_port\":false}" >/dev/null 2>&1 echo "$(date): qBittorrent listen port set to $PORT" - else - echo "$(date): Port file empty, waiting..." + if [ -n "$COORDINATOR_URL" ] && [ -n "$WORKER_ID" ]; then + curl -s -X POST "$COORDINATOR_URL/api/v2/workers/$WORKER_ID/port-lease" \ + -H "Content-Type: application/json" \ + -d "{\"port\": $PORT, \"ttlSeconds\": $TTL}" >/dev/null 2>&1 || true + fi fi - else - echo "$(date): Waiting for gluetun to write port file..." fi sleep 45 done + env: + - name: COORDINATOR_URL + value: "https://fornax.neuralplatform.ai" + - name: WORKER_ID + value: "tx34" volumeMounts: - name: gluetun-data mountPath: /tmp/gluetun