diff --git a/servers/legion/k8s/fornax/worker/base/deployment.yaml b/servers/legion/k8s/fornax/worker/base/deployment.yaml index c14b7b9..23a383d 100644 --- a/servers/legion/k8s/fornax/worker/base/deployment.yaml +++ b/servers/legion/k8s/fornax/worker/base/deployment.yaml @@ -107,6 +107,9 @@ spec: - name: portforward-helper image: alpine:latest imagePullPolicy: Always + securityContext: + capabilities: + add: ["NET_ADMIN"] env: - name: COORDINATOR_URL value: "https://fornax.neuralplatform.ai" @@ -116,6 +119,14 @@ spec: - sh - -c - | + echo "Waiting for gluetun health endpoint..." + while ! wget -q -T 3 -O- http://127.0.0.1:9999 > /dev/null 2>&1; do sleep 3; done + echo "gluetun ready" + # gluetun adds 'ip rule priority 100 from lookup 200' which routes + # all pod traffic via eth0, bypassing tun0. The iptables OUTPUT DROP policy + # then blocks it. Removing rule 100 lets traffic fall through to priority 101 + # (not fwmark 0xca6c -> table 51820 -> tun0). + ip rule del priority 100 2>/dev/null && echo "Fixed VPN routing (removed rule 100)" || echo "Rule 100 not present" until apk add -q curl 2>/dev/null; do sleep 5; done echo "Watching /tmp/gluetun/forwarded_port for assigned port..." TTL=300 @@ -129,18 +140,13 @@ spec: curl -s -b /tmp/qbt.txt -X POST http://localhost:8080/api/v2/app/setPreferences \ -d "json={\"listen_port\":$PORT,\"random_port\":false}" >/dev/null 2>&1 echo "$(date): qBittorrent listen port set to $PORT" - # Notify coordinator of port lease if [ -n "$COORDINATOR_URL" ] && [ -n "$WORKER_ID" ]; then curl -s -X POST "$COORDINATOR_URL/api/v2/workers/$WORKER_ID/port-lease" \ -H "Content-Type: application/json" \ -d "{\"port\": $PORT, \"ttlSeconds\": $TTL}" >/dev/null 2>&1 || true echo "$(date): Coordinator notified of port $PORT (TTL: ${TTL}s)" fi - else - echo "$(date): Port file empty, waiting..." fi - else - echo "$(date): Waiting for gluetun to write port file..." fi sleep 45 done diff --git a/servers/legion/k8s/media/fornax-workers.yaml b/servers/legion/k8s/media/fornax-workers.yaml index faa58fc..39850ce 100644 --- a/servers/legion/k8s/media/fornax-workers.yaml +++ b/servers/legion/k8s/media/fornax-workers.yaml @@ -133,15 +133,32 @@ spec: memory: 512Mi cpu: 200m - # Port forwarding helper — reads port from file gluetun writes to /tmp/gluetun/forwarded_port + # Port forwarding helper — fixes gluetun routing rule then watches for VPN port assignment - name: portforward-helper image: alpine:latest + securityContext: + capabilities: + add: ["NET_ADMIN"] command: - sh - -c - | + # Wait for gluetun to finish setting up its routing rules + echo "Waiting for gluetun health endpoint..." + while ! wget -q -T 3 -O- http://127.0.0.1:9999 > /dev/null 2>&1; do sleep 3; done + echo "gluetun ready" + + # gluetun adds 'ip rule priority 100 from lookup 200' which routes + # ALL pod traffic through eth0, bypassing tun0. The iptables OUTPUT DROP policy + # then blocks non-VPN, non-cluster traffic. Removing rule 100 allows traffic to + # fall through to 'priority 101 not fwmark 0xca6c lookup 51820' (tun0/VPN). + ip rule del priority 100 2>/dev/null && echo "Fixed VPN routing (removed rule 100)" || echo "Rule 100 not present" + + # Install curl (succeeds now that VPN traffic routes correctly) until apk add -q curl 2>/dev/null; do sleep 5; done + echo "Watching /tmp/gluetun/forwarded_port for assigned port..." + TTL=300 while true; do if [ -f /tmp/gluetun/forwarded_port ]; then PORT=$(cat /tmp/gluetun/forwarded_port) @@ -152,14 +169,20 @@ spec: curl -s -b /tmp/qbt.txt -X POST http://localhost:8080/api/v2/app/setPreferences \ -d "json={\"listen_port\":$PORT,\"random_port\":false}" >/dev/null 2>&1 echo "$(date): qBittorrent listen port set to $PORT" - else - echo "$(date): Port file empty, waiting..." + if [ -n "$COORDINATOR_URL" ] && [ -n "$WORKER_ID" ]; then + curl -s -X POST "$COORDINATOR_URL/api/v2/workers/$WORKER_ID/port-lease" \ + -H "Content-Type: application/json" \ + -d "{\"port\": $PORT, \"ttlSeconds\": $TTL}" >/dev/null 2>&1 || true + fi fi - else - echo "$(date): Waiting for gluetun to write port file..." fi sleep 45 done + env: + - name: COORDINATOR_URL + value: "https://fornax.neuralplatform.ai" + - name: WORKER_ID + value: "tx253" volumeMounts: - name: gluetun-data mountPath: /tmp/gluetun @@ -355,12 +378,20 @@ spec: - name: portforward-helper image: alpine:latest + securityContext: + capabilities: + add: ["NET_ADMIN"] command: - sh - -c - | + echo "Waiting for gluetun health endpoint..." + while ! wget -q -T 3 -O- http://127.0.0.1:9999 > /dev/null 2>&1; do sleep 3; done + echo "gluetun ready" + ip rule del priority 100 2>/dev/null && echo "Fixed VPN routing (removed rule 100)" || echo "Rule 100 not present" until apk add -q curl 2>/dev/null; do sleep 5; done echo "Watching /tmp/gluetun/forwarded_port for assigned port..." + TTL=300 while true; do if [ -f /tmp/gluetun/forwarded_port ]; then PORT=$(cat /tmp/gluetun/forwarded_port) @@ -371,14 +402,20 @@ spec: curl -s -b /tmp/qbt.txt -X POST http://localhost:8080/api/v2/app/setPreferences \ -d "json={\"listen_port\":$PORT,\"random_port\":false}" >/dev/null 2>&1 echo "$(date): qBittorrent listen port set to $PORT" - else - echo "$(date): Port file empty, waiting..." + if [ -n "$COORDINATOR_URL" ] && [ -n "$WORKER_ID" ]; then + curl -s -X POST "$COORDINATOR_URL/api/v2/workers/$WORKER_ID/port-lease" \ + -H "Content-Type: application/json" \ + -d "{\"port\": $PORT, \"ttlSeconds\": $TTL}" >/dev/null 2>&1 || true + fi fi - else - echo "$(date): Waiting for gluetun to write port file..." fi sleep 45 done + env: + - name: COORDINATOR_URL + value: "https://fornax.neuralplatform.ai" + - name: WORKER_ID + value: "tx34" volumeMounts: - name: gluetun-data mountPath: /tmp/gluetun