From bdba692db2067cf6f10aa1364dfc3513de65f9f9 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Tue, 14 Apr 2026 05:30:01 +0000 Subject: [PATCH] Fornax: COORDINATOR_URL env, port-lease webhook, Grafana dashboard - Worker deployment: add COORDINATOR_URL and WORKER_ID env vars to portforward-helper; webhook calls coordinator /api/v2/workers/:id/port-lease after each NAT-PMP renewal so coordinator tracks current forwarded port - Worker overlays: patch WORKER_ID env var to actual worker ID (tx253/tx34) - Grafana: fornax-grafana Argo CD app pointing to k8s/fornax/grafana/ ConfigMap with grafana_dashboard=1 label for auto-discovery by sidecar; 7 panels covering bandwidth, active torrents, worker health, request rate, and error rate --- servers/legion/apps/fornax.yaml | 22 ++ .../legion/k8s/fornax/grafana/configmap.yaml | 304 ++++++++++++++++++ .../k8s/fornax/grafana/kustomization.yaml | 4 + .../k8s/fornax/worker/base/deployment.yaml | 13 + .../overlays/worker-tx253/kustomization.yaml | 3 + .../overlays/worker-tx34/kustomization.yaml | 3 + 6 files changed, 349 insertions(+) create mode 100644 servers/legion/k8s/fornax/grafana/configmap.yaml create mode 100644 servers/legion/k8s/fornax/grafana/kustomization.yaml diff --git a/servers/legion/apps/fornax.yaml b/servers/legion/apps/fornax.yaml index 9f8cfd8..09161ed 100644 --- a/servers/legion/apps/fornax.yaml +++ b/servers/legion/apps/fornax.yaml @@ -63,3 +63,25 @@ spec: selfHeal: true syncOptions: - CreateNamespace=true +--- +# Fornax Grafana dashboard — auto-discovered by Grafana sidecar (grafana_dashboard=1) +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: fornax-grafana + namespace: argocd +spec: + project: default + source: + repoURL: http://10.43.1.53:3000/will/infrastructure.git + targetRevision: main + path: servers/legion/k8s/fornax/grafana + destination: + server: https://kubernetes.default.svc + namespace: monitoring + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/servers/legion/k8s/fornax/grafana/configmap.yaml b/servers/legion/k8s/fornax/grafana/configmap.yaml new file mode 100644 index 0000000..fa84783 --- /dev/null +++ b/servers/legion/k8s/fornax/grafana/configmap.yaml @@ -0,0 +1,304 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: fornax-grafana-dashboard + namespace: monitoring + labels: + grafana_dashboard: "1" +data: + fornax.json: | + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + }, + { + "name": "DS_LOKI", + "label": "Loki", + "description": "", + "type": "datasource", + "pluginId": "loki", + "pluginName": "Loki" + } + ], + "__requires": [ + { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "9.0.0" }, + { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, + { "type": "datasource", "id": "loki", "name": "Loki", "version": "1.0.0" } + ], + "annotations": { "list": [] }, + "description": "Fornax torrent distribution coordinator — fleet bandwidth, worker health, and distribution metrics", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "title": "Fleet Bandwidth", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, + "id": 1, + "options": { "tooltip": { "mode": "multi" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum(fornax_worker_download_bytes_per_second)", + "legendFormat": "Total Download", + "refId": "A" + } + ], + "title": "Total Download Bandwidth", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, + "id": 2, + "options": { "tooltip": { "mode": "multi" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum(fornax_worker_upload_bytes_per_second)", + "legendFormat": "Total Upload", + "refId": "A" + } + ], + "title": "Total Upload Bandwidth", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 9 }, + "id": 3, + "options": { "tooltip": { "mode": "multi" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "fornax_worker_download_bytes_per_second", + "legendFormat": "Worker {{ worker_id }}", + "refId": "A" + } + ], + "title": "Per-Worker Download Bandwidth", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }, + "id": 101, + "title": "Torrent Distribution", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, + "id": 4, + "options": { + "orientation": "horizontal", + "reduceOptions": { "calcs": ["lastNotNull"] }, + "text": {} + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "fornax_worker_active_torrents", + "legendFormat": "{{ worker_id }}", + "refId": "A" + } + ], + "title": "Active Torrent Count per Worker", + "type": "bargauge" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [ + { "options": { "0": { "color": "red", "text": "DOWN" }, "1": { "color": "green", "text": "UP" } }, "type": "value" } + ] + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, + "id": 5, + "options": { + "footer": { "show": false }, + "showHeader": true + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "fornax_worker_healthy", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Worker Health Status", + "transformations": [ + { + "id": "organize", + "options": { + "renameByName": { + "worker_id": "Worker ID", + "Value": "Status", + "Time": "Last Seen" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 }, + "id": 102, + "title": "Coordinator Logs & Errors", + "type": "row" + }, + { + "datasource": { "type": "loki", "uid": "${loki}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 }, + "id": 6, + "options": { "tooltip": { "mode": "multi" } }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "${loki}" }, + "expr": "rate({app=\"fornax-coordinator\"} |= \"POST /api/v2\" [1m])", + "legendFormat": "Request Rate", + "refId": "A" + } + ], + "title": "Coordinator Request Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "loki", "uid": "${loki}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "unit": "reqps", + "color": { "fixedColor": "red", "mode": "fixed" } + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 27 }, + "id": 7, + "options": { "tooltip": { "mode": "multi" } }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "${loki}" }, + "expr": "rate({app=\"fornax-coordinator\"} |= \"level=error\" [5m])", + "legendFormat": "Error Rate", + "refId": "A" + } + ], + "title": "Error Rate", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 37, + "style": "dark", + "tags": ["fornax", "torrents", "monitoring"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Loki", + "multi": false, + "name": "loki", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { "from": "now-3h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Fornax — Fleet Overview", + "uid": "fornax-fleet-overview", + "version": 1, + "weekStart": "" + } diff --git a/servers/legion/k8s/fornax/grafana/kustomization.yaml b/servers/legion/k8s/fornax/grafana/kustomization.yaml new file mode 100644 index 0000000..9fb65fa --- /dev/null +++ b/servers/legion/k8s/fornax/grafana/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - configmap.yaml diff --git a/servers/legion/k8s/fornax/worker/base/deployment.yaml b/servers/legion/k8s/fornax/worker/base/deployment.yaml index 0b9bc09..99954b6 100644 --- a/servers/legion/k8s/fornax/worker/base/deployment.yaml +++ b/servers/legion/k8s/fornax/worker/base/deployment.yaml @@ -96,12 +96,18 @@ spec: - name: portforward-helper image: alpine:latest imagePullPolicy: Always + env: + - name: COORDINATOR_URL + value: "https://fornax.neuralplatform.ai" + - name: WORKER_ID + value: "WORKER_ID" command: - sh - -c - | until apk add -q curl 2>/dev/null; do sleep 5; done echo "Watching /tmp/gluetun/forwarded_port for assigned port..." + TTL=300 while true; do if [ -f /tmp/gluetun/forwarded_port ]; then PORT=$(cat /tmp/gluetun/forwarded_port) @@ -112,6 +118,13 @@ spec: curl -s -b /tmp/qbt.txt -X POST http://localhost:8080/api/v2/app/setPreferences \ -d "json={\"listen_port\":$PORT,\"random_port\":false}" >/dev/null 2>&1 echo "$(date): qBittorrent listen port set to $PORT" + # Notify coordinator of port lease + if [ -n "$COORDINATOR_URL" ] && [ -n "$WORKER_ID" ]; then + curl -s -X POST "$COORDINATOR_URL/api/v2/workers/$WORKER_ID/port-lease" \ + -H "Content-Type: application/json" \ + -d "{\"port\": $PORT, \"ttlSeconds\": $TTL}" >/dev/null 2>&1 || true + echo "$(date): Coordinator notified of port $PORT (TTL: ${TTL}s)" + fi else echo "$(date): Port file empty, waiting..." fi diff --git a/servers/legion/k8s/fornax/worker/overlays/worker-tx253/kustomization.yaml b/servers/legion/k8s/fornax/worker/overlays/worker-tx253/kustomization.yaml index 081aeda..f9a2d90 100644 --- a/servers/legion/k8s/fornax/worker/overlays/worker-tx253/kustomization.yaml +++ b/servers/legion/k8s/fornax/worker/overlays/worker-tx253/kustomization.yaml @@ -26,6 +26,9 @@ patches: - op: replace path: /spec/template/spec/containers/0/env/3/valueFrom/secretKeyRef/name value: fornax-worker-tx253-secrets + - op: replace + path: /spec/template/spec/containers/1/env/1/value + value: tx253 - op: replace path: /spec/template/spec/volumes/1/persistentVolumeClaim/claimName value: fornax-worker-tx253-config diff --git a/servers/legion/k8s/fornax/worker/overlays/worker-tx34/kustomization.yaml b/servers/legion/k8s/fornax/worker/overlays/worker-tx34/kustomization.yaml index 319f49f..444f5b7 100644 --- a/servers/legion/k8s/fornax/worker/overlays/worker-tx34/kustomization.yaml +++ b/servers/legion/k8s/fornax/worker/overlays/worker-tx34/kustomization.yaml @@ -26,6 +26,9 @@ patches: - op: replace path: /spec/template/spec/containers/0/env/3/valueFrom/secretKeyRef/name value: fornax-worker-tx34-secrets + - op: replace + path: /spec/template/spec/containers/1/env/1/value + value: tx34 - op: replace path: /spec/template/spec/volumes/1/persistentVolumeClaim/claimName value: fornax-worker-tx34-config