From 0341b570abea82e3775b46a4f7d142d8e34e4893 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Tue, 24 Mar 2026 12:55:29 -0500 Subject: [PATCH] fix: gitea runner workingDir, README updates, monitoring domain - Fix act-runner workingDir: /data on both init and main containers so .runner registration file is found at daemon startup - Add servers/legion/README.md with full service catalog and deploy docs - Update root README.md to be an index pointing to key docs - Fix monitoring.tf comment: grafana.neuralplatform.ai (not nook.family) - Fix AGENTS.md router model to TP-Link Deco BE5000 --- README.md | 16 ++-- servers/legion/README.md | 121 ++++++++++++++++++++++++++ servers/legion/apps/gitea-runner.yaml | 2 + servers/legion/monitoring.tf | 2 +- 4 files changed, 134 insertions(+), 7 deletions(-) create mode 100644 servers/legion/README.md diff --git a/README.md b/README.md index f93831e..e293f7f 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,20 @@ # infrastructure -Personal infrastructure-as-code — servers, networking, DNS, and config. +Personal infrastructure-as-code for Legion (k3s home server) and supporting tooling. -## Structure +| Doc | What it covers | +|-----|---------------| +| [`AGENTS.md`](AGENTS.md) | Full reference: machines, secrets, services, domains, namespaces, common operations | +| [`servers/legion/README.md`](servers/legion/README.md) | What's running on Legion and how deployment works | +| [`servers/legion/RUNBOOK.md`](servers/legion/RUNBOOK.md) | Disaster recovery — full restore from scratch | -- `servers/legion/` — Legion k3s cluster (Terraform) - -## Usage +## Quick start ```bash cd servers/legion -direnv allow # loads credentials from ~/Secrets/credentials/infrastructure.env +direnv allow terraform plan terraform apply ``` + +Push to `servers/` to deploy app changes via Argo CD. diff --git a/servers/legion/README.md b/servers/legion/README.md new file mode 100644 index 0000000..7d1b25a --- /dev/null +++ b/servers/legion/README.md @@ -0,0 +1,121 @@ +# Legion + +Ubuntu 24.04 home server running k3s. Hosts all personal and platform infrastructure. + +**LAN IP:** `192.168.68.77` +**SSH:** `ssh legion` +**Kubeconfig:** `~/.kube/legion-config` + +--- + +## Services + +### Git & Code +| Service | URL | What it does | +|---------|-----|--------------| +| Gitea | https://git.neuralplatform.ai | Self-hosted Git — all repos live here | +| GitHub Actions Runner | — | Runs CI jobs from harmonic-framework org | +| Verdaccio | https://npm.neuralplatform.ai | Private npm registry | + +### Platform +| Service | URL | What it does | +|---------|-----|--------------| +| Argo CD | https://argocd.neuralplatform.ai | Deploys everything in `apps/` from Gitea | +| Vault | https://vault.neuralplatform.ai | Secret storage | +| Docker Registry | https://registry.neuralplatform.ai | Private container image registry | + +### AI & Apps +| Service | URL | What it does | +|---------|-----|--------------| +| Neuron | internal | MCP server + webhook handler for Axon events | +| Ollama | http://ollama.legion.local | Local LLM inference (GPU) | + +### Networking & DNS +| Service | URL | What it does | +|---------|-----|--------------| +| AdGuard Home | https://dns.nook.family | LAN DNS with ad blocking | +| DoT | `dot.nook.family:853` | DNS-over-TLS for mobile devices (Android Private DNS) | +| Traefik | — | Ingress controller, terminates TLS for all services | +| Cloudflare Tunnel | — | Exposes public services without opening ports | + +### Observability +| Service | URL | What it does | +|---------|-----|--------------| +| Grafana | https://grafana.neuralplatform.ai | Dashboards, log search, trace viewer | +| Prometheus | internal | Metrics collection and alerting | +| Alertmanager | internal | Routes alerts to #infrastructure-alerts in Slack | +| Loki | internal | Log aggregation (fed by Alloy) | +| Tempo | internal | Distributed tracing backend | +| Alloy | internal | Collects pod logs → Loki, forwards OTLP traces → Tempo | + +### Data +| Service | What it does | +|---------|--------------| +| Postgres | Shared database (Gitea, Kong, others) | +| Redis | Shared cache | + +--- + +## How Deployment Works + +- **Infrastructure** (namespaces, secrets, Helm charts, PVCs): managed by Terraform in this directory +- **Apps** (Deployments, Services, ConfigMaps): live in `apps/`, deployed by Argo CD watching this repo +- Never apply changes with `kubectl` directly — use Terraform or push to `apps/` + +```bash +# Apply infra changes +cd servers/legion/ +direnv exec . terraform apply + +# Deploy app changes — just push to Gitea +git push # Argo CD auto-syncs within ~30s +``` + +--- + +## Networking + +**Router:** TP-Link Deco BE5000 mesh network, router mode. Subnet `192.168.68.x`, DHCP DNS points to AdGuard (`192.168.68.77`). + +External traffic reaches Legion two ways: + +1. **Cloudflare Tunnel** — public services (`*.neuralplatform.ai`) route through a tunnel pod in k8s. No ports open on the router. +2. **Direct port forward** — DoT on port 853 (TCP) is forwarded from the router to `192.168.68.77:853`. Required because Cloudflare can't proxy raw TCP. + +LAN devices use AdGuard as their DNS server (`192.168.68.77`). AdGuard rewrites `*.nook.family` and `*.neuralplatform.ai` to the LAN IP so local traffic stays local. + +ddclient keeps the `dot.nook.family` A record updated if the home IP changes. + +--- + +## GPU + +Legion has an NVIDIA GPU. k3s is configured to use it as the default container runtime. Ollama uses it for local inference. + +```bash +kubectl describe node legion | grep nvidia.com/gpu +``` + +--- + +## Backups + +Nightly at 2am (cron job in `git` namespace): +- Dumps all Postgres databases +- Backs up Gitea data volume + DB dump via restic → R2 bucket `legion-gitea-backup` +- Retains 30 daily / 12 weekly / 6 monthly snapshots + +`~/Secrets` on the Mac is encrypted with age and uploaded to R2 bucket `legion-secrets-backup` nightly. + +See `RUNBOOK.md` for full disaster recovery procedure. + +--- + +## Key Files + +| File | What it does | +|------|--------------| +| `bootstrap.sh` | Provisions a fresh Ubuntu 24.04 machine from scratch | +| `RUNBOOK.md` | Step-by-step disaster recovery | +| `variables.tf` | All configurable values and secrets | +| `apps/` | Kubernetes manifests deployed by Argo CD | diff --git a/servers/legion/apps/gitea-runner.yaml b/servers/legion/apps/gitea-runner.yaml index c90466b..122baf5 100644 --- a/servers/legion/apps/gitea-runner.yaml +++ b/servers/legion/apps/gitea-runner.yaml @@ -18,6 +18,7 @@ spec: initContainers: - name: register image: gitea/act_runner:latest + workingDir: /data command: ["/bin/sh", "-c"] args: - | @@ -36,6 +37,7 @@ spec: containers: - name: runner image: gitea/act_runner:latest + workingDir: /data command: ["act_runner", "daemon"] envFrom: - secretRef: diff --git a/servers/legion/monitoring.tf b/servers/legion/monitoring.tf index 500e7c1..017439a 100644 --- a/servers/legion/monitoring.tf +++ b/servers/legion/monitoring.tf @@ -1,5 +1,5 @@ # Monitoring stack — Prometheus, Grafana, Loki, Tempo, Alloy, Alertmanager -# Grafana UI: https://grafana.nook.family (admin / var.grafana_admin_password) +# Grafana UI: https://grafana.neuralplatform.ai (admin / var.grafana_admin_password) # # TODO: Migrate Tempo to tempo-distributed once R2 is wired as trace storage backend # TODO: Replace loki.useTestSchema=true with proper schema_config for production