Fornax: COORDINATOR_URL env, port-lease webhook, Grafana dashboard
- Worker deployment: add COORDINATOR_URL and WORKER_ID env vars to portforward-helper; webhook calls coordinator /api/v2/workers/:id/port-lease after each NAT-PMP renewal so coordinator tracks current forwarded port - Worker overlays: patch WORKER_ID env var to actual worker ID (tx253/tx34) - Grafana: fornax-grafana Argo CD app pointing to k8s/fornax/grafana/ ConfigMap with grafana_dashboard=1 label for auto-discovery by sidecar; 7 panels covering bandwidth, active torrents, worker health, request rate, and error rate
This commit is contained in:
Will Anderson
@@ -63,3 +63,25 @@ spec:
|
|||||||
selfHeal: true
|
selfHeal: true
|
||||||
syncOptions:
|
syncOptions:
|
||||||
- CreateNamespace=true
|
- CreateNamespace=true
|
||||||
|
---
|
||||||
|
# Fornax Grafana dashboard — auto-discovered by Grafana sidecar (grafana_dashboard=1)
|
||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: fornax-grafana
|
||||||
|
namespace: argocd
|
||||||
|
spec:
|
||||||
|
project: default
|
||||||
|
source:
|
||||||
|
repoURL: http://10.43.1.53:3000/will/infrastructure.git
|
||||||
|
targetRevision: main
|
||||||
|
path: servers/legion/k8s/fornax/grafana
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: monitoring
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
|
syncOptions:
|
||||||
|
- CreateNamespace=true
|
||||||
|
|||||||
@@ -0,0 +1,304 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: fornax-grafana-dashboard
|
||||||
|
namespace: monitoring
|
||||||
|
labels:
|
||||||
|
grafana_dashboard: "1"
|
||||||
|
data:
|
||||||
|
fornax.json: |
|
||||||
|
{
|
||||||
|
"__inputs": [
|
||||||
|
{
|
||||||
|
"name": "DS_PROMETHEUS",
|
||||||
|
"label": "Prometheus",
|
||||||
|
"description": "",
|
||||||
|
"type": "datasource",
|
||||||
|
"pluginId": "prometheus",
|
||||||
|
"pluginName": "Prometheus"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "DS_LOKI",
|
||||||
|
"label": "Loki",
|
||||||
|
"description": "",
|
||||||
|
"type": "datasource",
|
||||||
|
"pluginId": "loki",
|
||||||
|
"pluginName": "Loki"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"__requires": [
|
||||||
|
{ "type": "grafana", "id": "grafana", "name": "Grafana", "version": "9.0.0" },
|
||||||
|
{ "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" },
|
||||||
|
{ "type": "datasource", "id": "loki", "name": "Loki", "version": "1.0.0" }
|
||||||
|
],
|
||||||
|
"annotations": { "list": [] },
|
||||||
|
"description": "Fornax torrent distribution coordinator — fleet bandwidth, worker health, and distribution metrics",
|
||||||
|
"editable": true,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 1,
|
||||||
|
"id": null,
|
||||||
|
"links": [],
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"collapsed": false,
|
||||||
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||||
|
"id": 100,
|
||||||
|
"title": "Fleet Bandwidth",
|
||||||
|
"type": "row"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": { "lineWidth": 2, "fillOpacity": 10 },
|
||||||
|
"unit": "Bps"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
|
||||||
|
"id": 1,
|
||||||
|
"options": { "tooltip": { "mode": "multi" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "sum(fornax_worker_download_bytes_per_second)",
|
||||||
|
"legendFormat": "Total Download",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Total Download Bandwidth",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": { "lineWidth": 2, "fillOpacity": 10 },
|
||||||
|
"unit": "Bps"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
|
||||||
|
"id": 2,
|
||||||
|
"options": { "tooltip": { "mode": "multi" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "sum(fornax_worker_upload_bytes_per_second)",
|
||||||
|
"legendFormat": "Total Upload",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Total Upload Bandwidth",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": { "lineWidth": 2, "fillOpacity": 10 },
|
||||||
|
"unit": "Bps"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 9 },
|
||||||
|
"id": 3,
|
||||||
|
"options": { "tooltip": { "mode": "multi" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "fornax_worker_download_bytes_per_second",
|
||||||
|
"legendFormat": "Worker {{ worker_id }}",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Per-Worker Download Bandwidth",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"collapsed": false,
|
||||||
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 },
|
||||||
|
"id": 101,
|
||||||
|
"title": "Torrent Distribution",
|
||||||
|
"type": "row"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"unit": "short",
|
||||||
|
"min": 0
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 },
|
||||||
|
"id": 4,
|
||||||
|
"options": {
|
||||||
|
"orientation": "horizontal",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"] },
|
||||||
|
"text": {}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "fornax_worker_active_torrents",
|
||||||
|
"legendFormat": "{{ worker_id }}",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Active Torrent Count per Worker",
|
||||||
|
"type": "bargauge"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "red", "value": null },
|
||||||
|
{ "color": "yellow", "value": 1 },
|
||||||
|
{ "color": "green", "value": 1 }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"mappings": [
|
||||||
|
{ "options": { "0": { "color": "red", "text": "DOWN" }, "1": { "color": "green", "text": "UP" } }, "type": "value" }
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 },
|
||||||
|
"id": 5,
|
||||||
|
"options": {
|
||||||
|
"footer": { "show": false },
|
||||||
|
"showHeader": true
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||||||
|
"expr": "fornax_worker_healthy",
|
||||||
|
"format": "table",
|
||||||
|
"instant": true,
|
||||||
|
"legendFormat": "",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Worker Health Status",
|
||||||
|
"transformations": [
|
||||||
|
{
|
||||||
|
"id": "organize",
|
||||||
|
"options": {
|
||||||
|
"renameByName": {
|
||||||
|
"worker_id": "Worker ID",
|
||||||
|
"Value": "Status",
|
||||||
|
"Time": "Last Seen"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"type": "table"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"collapsed": false,
|
||||||
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 },
|
||||||
|
"id": 102,
|
||||||
|
"title": "Coordinator Logs & Errors",
|
||||||
|
"type": "row"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "${loki}" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": { "lineWidth": 2, "fillOpacity": 10 },
|
||||||
|
"unit": "reqps"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 },
|
||||||
|
"id": 6,
|
||||||
|
"options": { "tooltip": { "mode": "multi" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "${loki}" },
|
||||||
|
"expr": "rate({app=\"fornax-coordinator\"} |= \"POST /api/v2\" [1m])",
|
||||||
|
"legendFormat": "Request Rate",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Coordinator Request Rate",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "${loki}" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": { "lineWidth": 2, "fillOpacity": 10 },
|
||||||
|
"unit": "reqps",
|
||||||
|
"color": { "fixedColor": "red", "mode": "fixed" }
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 27 },
|
||||||
|
"id": 7,
|
||||||
|
"options": { "tooltip": { "mode": "multi" } },
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "${loki}" },
|
||||||
|
"expr": "rate({app=\"fornax-coordinator\"} |= \"level=error\" [5m])",
|
||||||
|
"legendFormat": "Error Rate",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Error Rate",
|
||||||
|
"type": "timeseries"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "30s",
|
||||||
|
"schemaVersion": 37,
|
||||||
|
"style": "dark",
|
||||||
|
"tags": ["fornax", "torrents", "monitoring"],
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"current": {},
|
||||||
|
"hide": 0,
|
||||||
|
"includeAll": false,
|
||||||
|
"label": "Prometheus",
|
||||||
|
"multi": false,
|
||||||
|
"name": "datasource",
|
||||||
|
"options": [],
|
||||||
|
"query": "prometheus",
|
||||||
|
"refresh": 1,
|
||||||
|
"regex": "",
|
||||||
|
"type": "datasource"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"current": {},
|
||||||
|
"hide": 0,
|
||||||
|
"includeAll": false,
|
||||||
|
"label": "Loki",
|
||||||
|
"multi": false,
|
||||||
|
"name": "loki",
|
||||||
|
"options": [],
|
||||||
|
"query": "loki",
|
||||||
|
"refresh": 1,
|
||||||
|
"regex": "",
|
||||||
|
"type": "datasource"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"time": { "from": "now-3h", "to": "now" },
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "browser",
|
||||||
|
"title": "Fornax — Fleet Overview",
|
||||||
|
"uid": "fornax-fleet-overview",
|
||||||
|
"version": 1,
|
||||||
|
"weekStart": ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
resources:
|
||||||
|
- configmap.yaml
|
||||||
@@ -96,12 +96,18 @@ spec:
|
|||||||
- name: portforward-helper
|
- name: portforward-helper
|
||||||
image: alpine:latest
|
image: alpine:latest
|
||||||
imagePullPolicy: Always
|
imagePullPolicy: Always
|
||||||
|
env:
|
||||||
|
- name: COORDINATOR_URL
|
||||||
|
value: "https://fornax.neuralplatform.ai"
|
||||||
|
- name: WORKER_ID
|
||||||
|
value: "WORKER_ID"
|
||||||
command:
|
command:
|
||||||
- sh
|
- sh
|
||||||
- -c
|
- -c
|
||||||
- |
|
- |
|
||||||
until apk add -q curl 2>/dev/null; do sleep 5; done
|
until apk add -q curl 2>/dev/null; do sleep 5; done
|
||||||
echo "Watching /tmp/gluetun/forwarded_port for assigned port..."
|
echo "Watching /tmp/gluetun/forwarded_port for assigned port..."
|
||||||
|
TTL=300
|
||||||
while true; do
|
while true; do
|
||||||
if [ -f /tmp/gluetun/forwarded_port ]; then
|
if [ -f /tmp/gluetun/forwarded_port ]; then
|
||||||
PORT=$(cat /tmp/gluetun/forwarded_port)
|
PORT=$(cat /tmp/gluetun/forwarded_port)
|
||||||
@@ -112,6 +118,13 @@ spec:
|
|||||||
curl -s -b /tmp/qbt.txt -X POST http://localhost:8080/api/v2/app/setPreferences \
|
curl -s -b /tmp/qbt.txt -X POST http://localhost:8080/api/v2/app/setPreferences \
|
||||||
-d "json={\"listen_port\":$PORT,\"random_port\":false}" >/dev/null 2>&1
|
-d "json={\"listen_port\":$PORT,\"random_port\":false}" >/dev/null 2>&1
|
||||||
echo "$(date): qBittorrent listen port set to $PORT"
|
echo "$(date): qBittorrent listen port set to $PORT"
|
||||||
|
# Notify coordinator of port lease
|
||||||
|
if [ -n "$COORDINATOR_URL" ] && [ -n "$WORKER_ID" ]; then
|
||||||
|
curl -s -X POST "$COORDINATOR_URL/api/v2/workers/$WORKER_ID/port-lease" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "{\"port\": $PORT, \"ttlSeconds\": $TTL}" >/dev/null 2>&1 || true
|
||||||
|
echo "$(date): Coordinator notified of port $PORT (TTL: ${TTL}s)"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
echo "$(date): Port file empty, waiting..."
|
echo "$(date): Port file empty, waiting..."
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -26,6 +26,9 @@ patches:
|
|||||||
- op: replace
|
- op: replace
|
||||||
path: /spec/template/spec/containers/0/env/3/valueFrom/secretKeyRef/name
|
path: /spec/template/spec/containers/0/env/3/valueFrom/secretKeyRef/name
|
||||||
value: fornax-worker-tx253-secrets
|
value: fornax-worker-tx253-secrets
|
||||||
|
- op: replace
|
||||||
|
path: /spec/template/spec/containers/1/env/1/value
|
||||||
|
value: tx253
|
||||||
- op: replace
|
- op: replace
|
||||||
path: /spec/template/spec/volumes/1/persistentVolumeClaim/claimName
|
path: /spec/template/spec/volumes/1/persistentVolumeClaim/claimName
|
||||||
value: fornax-worker-tx253-config
|
value: fornax-worker-tx253-config
|
||||||
|
|||||||
@@ -26,6 +26,9 @@ patches:
|
|||||||
- op: replace
|
- op: replace
|
||||||
path: /spec/template/spec/containers/0/env/3/valueFrom/secretKeyRef/name
|
path: /spec/template/spec/containers/0/env/3/valueFrom/secretKeyRef/name
|
||||||
value: fornax-worker-tx34-secrets
|
value: fornax-worker-tx34-secrets
|
||||||
|
- op: replace
|
||||||
|
path: /spec/template/spec/containers/1/env/1/value
|
||||||
|
value: tx34
|
||||||
- op: replace
|
- op: replace
|
||||||
path: /spec/template/spec/volumes/1/persistentVolumeClaim/claimName
|
path: /spec/template/spec/volumes/1/persistentVolumeClaim/claimName
|
||||||
value: fornax-worker-tx34-config
|
value: fornax-worker-tx34-config
|
||||||
|
|||||||
Reference in New Issue
Block a user