From f8bc853136ba1c32fbcbe27964c1d1277c6c9f3d Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Tue, 24 Mar 2026 00:46:36 -0500 Subject: [PATCH] DR: full bootstrap script + recovery runbook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bootstrap.sh now covers a complete fresh Ubuntu 24.04 setup: - NVIDIA driver 580 + container toolkit + k3s GPU config - k3s install with nvidia as default runtime + device plugin - Host cloudflared (Gitea SSH tunnel) - Terraform + Helm - systemd-resolved disabled, resolv.conf pinned RUNBOOK.md documents the full 4-step recovery: bootstrap → terraform apply → Gitea/Postgres restore → verify Target: under 1 hour on a new machine. --- servers/legion/RUNBOOK.md | 176 ++++++++++++++++++++++++++++++++++++ servers/legion/bootstrap.sh | 147 +++++++++++++++++++++++++++--- 2 files changed, 308 insertions(+), 15 deletions(-) create mode 100644 servers/legion/RUNBOOK.md diff --git a/servers/legion/RUNBOOK.md b/servers/legion/RUNBOOK.md new file mode 100644 index 0000000..782eba7 --- /dev/null +++ b/servers/legion/RUNBOOK.md @@ -0,0 +1,176 @@ +# Legion Disaster Recovery Runbook + +Complete recovery from total loss of the Legion server. +Target: running cluster in under 2 hours on a fresh machine. + +--- + +## Prerequisites (stored offsite) + +| What | Where | +|------|-------| +| `~/Secrets` (encrypted) | R2 bucket `legion-secrets-backup` | +| Terraform state | R2 bucket `legion-terraform-state` | +| Gitea repos + DB dump | R2 bucket `legion-gitea-backup` (restic) | +| age decryption key | Printed copy / separate secure storage | + +> **Note:** The age private key (`~/Secrets/age-backup.key`) is inside the encrypted +> archive — which means you need the key to decrypt the archive that contains the key. +> Store the age private key somewhere independent (password manager, printed sheet, etc.). + +--- + +## Step 1 — Recover Secrets (on Mac or any machine with `age` + `aws`) + +```bash +# Install age if needed +brew install age + +# Configure R2 access (you'll need the R2 access key from your password manager) +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +R2_ENDPOINT="https://651161e0a3d321561b4c90b5bcd5f15b.r2.cloudflarestorage.com" + +# Download the latest encrypted backup +aws s3 cp s3://legion-secrets-backup/secrets-latest.tar.gz.age /tmp/secrets.tar.gz.age \ + --endpoint-url "$R2_ENDPOINT" + +# Decrypt and extract (you need the age private key for this) +age --decrypt -i /path/to/age-backup.key /tmp/secrets.tar.gz.age | tar -xz -C ~/ + +# Verify +ls ~/Secrets/credentials/ +source ~/Secrets/credentials/infrastructure.env +``` + +--- + +## Step 2 — Provision Fresh Machine + +Fresh Ubuntu 24.04 LTS, same LAN IP (`192.168.68.77`). + +```bash +# From Mac — copy SSH key and cloudflared credentials +ssh-copy-id will@192.168.68.77 +scp ~/Secrets/certs/cloudflared-gitea.json legion:/tmp/cloudflared-gitea.json + +# Run bootstrap +ssh legion "bash -s" < ~/Development/infrastructure/servers/legion/bootstrap.sh + +# Copy kubeconfig back to Mac +scp legion:/home/will/.kube/config ~/.kube/legion-config +export KUBECONFIG=~/.kube/legion-config +kubectl get nodes # should show legion as Ready +``` + +**Bootstrap installs:** +- NVIDIA driver 580 + container toolkit +- k3s (latest stable) with NVIDIA as default runtime +- NVIDIA device plugin +- Cloudflared (host-level, Gitea SSH tunnel) +- Terraform + Helm + +--- + +## Step 3 — Apply Terraform + +Terraform state lives in R2 — it will re-create all k8s resources from scratch. + +```bash +cd ~/Development/infrastructure/servers/legion/ +direnv exec . terraform init # pulls state from R2 +direnv exec . terraform plan # review what will be created +direnv exec . terraform apply # recreates everything: namespaces, PVCs, secrets, Helm releases, Argo CD +``` + +Wait for Argo CD to sync (~2 minutes after apply). Check: + +```bash +kubectl get pods -A # all pods should reach Running +``` + +--- + +## Step 4 — Restore Gitea Data + +Gitea repos and the Postgres DB are backed up nightly via restic to `legion-gitea-backup`. + +```bash +# Set up restic env +set -a; source ~/Secrets/credentials/infrastructure.env; set +a +export RESTIC_REPOSITORY="s3:$(secret r2-endpoint)/legion-gitea-backup" +export RESTIC_PASSWORD="gitea-restic-2026" +export AWS_ACCESS_KEY_ID=$(secret r2-access-key) +export AWS_SECRET_ACCESS_KEY=$(secret r2-secret-key) + +# List available snapshots +restic snapshots + +# Restore Gitea data volume to a local dir +restic restore latest --target /tmp/gitea-restore --include /gitea-data + +# Copy into the PVC via kubectl +GITEA_POD=$(kubectl get pod -n git -l app=gitea -o jsonpath='{.items[0].metadata.name}') +kubectl cp /tmp/gitea-restore/gitea-data/. git/$GITEA_POD:/data/gitea/ + +# Restore Postgres DB dump +restic restore latest --target /tmp/gitea-restore --include /dump/all-databases.sql +kubectl exec -n platform deploy/postgres-postgresql -- \ + psql -U postgres < /tmp/gitea-restore/dump/all-databases.sql + +# Restart Gitea to pick up restored data +kubectl rollout restart deployment/gitea -n git +``` + +--- + +## Step 5 — Verify + +```bash +# k8s cluster healthy +kubectl get nodes +kubectl get pods -A | grep -v Running | grep -v Completed + +# Argo CD synced +kubectl get applications -n argocd + +# DNS working (AdGuard) +dig @192.168.68.77 git.neuralplatform.ai + +# Services reachable +curl -sk https://git.neuralplatform.ai/api/v1/version +curl -sk https://argocd.neuralplatform.ai/healthz + +# GPU available +kubectl describe node legion | grep nvidia.com/gpu +``` + +--- + +## Key Reference + +| Secret | Vault path | Field | +|--------|------------|-------| +| R2 access key | `secret/r2` | `access_key_id` | +| R2 secret | `secret/r2` | `secret_access_key` | +| Cloudflare API key | `secret/cloudflare` | `api_key` | +| Gitea admin token | `secret/gitea` | `api_token` | +| Postgres password | `secret/postgres` | `password` | +| Vault root token | `~/Secrets/tokens/vault-root-token` | — | + +Tunnel IDs: +- Platform tunnel (k8s pod): `54bc9b05-3953-47a2-9c3e-adecdcc53d51` +- Gitea SSH tunnel (host cloudflared): `b1bf80bf-5448-4c33-8667-d6fce8a82c7a` + +--- + +## Estimated Recovery Time + +| Step | Time | +|------|------| +| Recover secrets | 5 min | +| Provision OS + bootstrap | 20-30 min (driver install) | +| Terraform apply | 10-15 min | +| Gitea data restore | 10 min | +| Verify | 5 min | +| **Total** | **~1 hour** | diff --git a/servers/legion/bootstrap.sh b/servers/legion/bootstrap.sh index fe9fbb9..03a204f 100644 --- a/servers/legion/bootstrap.sh +++ b/servers/legion/bootstrap.sh @@ -1,18 +1,70 @@ #!/usr/bin/env bash -# Legion host bootstrap — run once on a fresh install before Terraform -# Usage: ssh legion "bash -s" < bootstrap.sh +# Legion host bootstrap — run ONCE on a fresh Ubuntu 24.04 install, before Terraform. +# +# Usage (from Mac after restoring ~/Secrets): +# ssh-copy-id legion +# scp ~/Secrets/cloudflared-gitea.json legion:/tmp/cloudflared-gitea.json +# ssh legion "bash -s" < bootstrap.sh +# +# After this script: run `direnv exec . terraform apply` from ~/Development/infrastructure/servers/legion/ + set -euo pipefail LEGION_IP="${1:-192.168.68.77}" +NVIDIA_DRIVER="nvidia-driver-580" -echo "==> Disabling systemd-resolved (conflicts with AdGuard on port 53)" -sudo systemctl disable --now systemd-resolved || true -sudo rm -f /etc/resolv.conf -echo "nameserver 1.1.1.1" | sudo tee /etc/resolv.conf +log() { echo ""; echo "==> $*"; } -echo "==> Configuring k3s tls-san for LAN IP ${LEGION_IP}" -sudo mkdir -p /etc/rancher/k3s -sudo tee /etc/rancher/k3s/config.yaml > /dev/null << EOF +# ── 0. Sanity check ────────────────────────────────────────────────────────── +if [ "$(id -u)" != "0" ]; then + echo "Run as root or via sudo. Re-running with sudo..." + exec sudo -E bash "$0" "$@" +fi + +log "Starting Legion bootstrap (IP: ${LEGION_IP})" + +# ── 1. Base packages ────────────────────────────────────────────────────────── +log "Installing base prerequisites" +apt-get update -qq +apt-get install -y --no-install-recommends \ + curl wget gnupg ca-certificates apt-transport-https \ + nfs-common open-iscsi \ + jq unzip git + +# Enable open-iscsi (required for some k3s storage backends) +systemctl enable --now iscsid || true + +# ── 2. Disable systemd-resolved (conflicts with AdGuard on port 53) ─────────── +log "Disabling systemd-resolved" +systemctl disable --now systemd-resolved || true +rm -f /etc/resolv.conf +echo "nameserver 1.1.1.1" > /etc/resolv.conf +echo "nameserver 8.8.8.8" >> /etc/resolv.conf +chattr +i /etc/resolv.conf # prevent NetworkManager from overwriting it + +# ── 3. NVIDIA drivers ───────────────────────────────────────────────────────── +log "Installing NVIDIA drivers (${NVIDIA_DRIVER})" +add-apt-repository -y ppa:graphics-drivers/ppa +apt-get update -qq +apt-get install -y "${NVIDIA_DRIVER}" + +# NVIDIA container toolkit (required for k3s GPU support) +log "Installing NVIDIA container toolkit" +curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \ + | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg +curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \ + | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \ + > /etc/apt/sources.list.d/nvidia-container-toolkit.list +apt-get update -qq +apt-get install -y nvidia-container-toolkit + +# Configure NVIDIA runtime for containerd (k3s uses containerd) +nvidia-ctk runtime configure --runtime=containerd + +# ── 4. k3s ──────────────────────────────────────────────────────────────────── +log "Configuring k3s" +mkdir -p /etc/rancher/k3s +cat > /etc/rancher/k3s/config.yaml << EOF write-kubeconfig-mode: "644" default-runtime: nvidia tls-san: @@ -20,12 +72,77 @@ tls-san: - 127.0.0.1 EOF -echo "==> Rotating k3s certificates" -sudo k3s certificate rotate -sudo systemctl restart k3s +log "Installing k3s" +curl -sfL https://get.k3s.io | sh - -echo "==> Waiting for k3s to come back up..." +log "Waiting for k3s to start..." sleep 20 -sudo k3s kubectl get nodes +k3s kubectl get nodes -echo "==> Bootstrap complete" +# Copy kubeconfig to user home (for 'will' user) +mkdir -p /home/will/.kube +cp /etc/rancher/k3s/k3s.yaml /home/will/.kube/config +sed -i "s/127.0.0.1/${LEGION_IP}/g" /home/will/.kube/config +chown will:will /home/will/.kube/config + +# ── 5. NVIDIA device plugin for k3s ────────────────────────────────────────── +log "Installing NVIDIA k3s device plugin" +k3s kubectl create -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.14.5/nvidia-device-plugin.yml \ + || k3s kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.14.5/nvidia-device-plugin.yml + +# ── 6. Cloudflared (host-level — Gitea SSH tunnel only) ─────────────────────── +# The platform tunnel (neuralplatform.ai) runs as a k8s pod managed by Terraform. +# This host-level cloudflared serves only the Gitea SSH NodePort (git.neuralplatform.ai → :30022). +log "Installing cloudflared" +curl -fsSL https://pkg.cloudflare.com/cloudflare-main.gpg \ + | gpg --dearmor -o /usr/share/keyrings/cloudflare-main.gpg +echo 'deb [signed-by=/usr/share/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared any main' \ + > /etc/apt/sources.list.d/cloudflared.list +apt-get update -qq +apt-get install -y cloudflared + +# Restore tunnel credentials from Mac-side secret (must exist at /tmp/cloudflared-gitea.json) +TUNNEL_ID="b1bf80bf-5448-4c33-8667-d6fce8a82c7a" +if [ -f "/tmp/cloudflared-gitea.json" ]; then + mkdir -p /etc/cloudflared + cp /tmp/cloudflared-gitea.json "/etc/cloudflared/${TUNNEL_ID}.json" + chmod 600 "/etc/cloudflared/${TUNNEL_ID}.json" + cat > /etc/cloudflared/config.yml << EOF +tunnel: ${TUNNEL_ID} +credentials-file: /etc/cloudflared/${TUNNEL_ID}.json + +ingress: + - hostname: git.neuralplatform.ai + service: http://localhost:30322 + - service: http_status:404 +EOF + cloudflared service install + systemctl enable --now cloudflared + log "Cloudflared installed and running" +else + log "WARNING: /tmp/cloudflared-gitea.json not found — skipping cloudflared setup" + log " To fix: scp ~/Secrets/certs/cloudflared-gitea.json legion:/tmp/cloudflared-gitea.json" + log " Then re-run: cloudflared service install && systemctl enable --now cloudflared" +fi + +# ── 7. CLI tools (Terraform, Helm) ──────────────────────────────────────────── +log "Installing Terraform" +wget -qO- https://apt.releases.hashicorp.com/gpg \ + | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" \ + > /etc/apt/sources.list.d/hashicorp.list +apt-get update -qq && apt-get install -y terraform + +log "Installing Helm" +curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + +# ── 8. Done ─────────────────────────────────────────────────────────────────── +log "Bootstrap complete!" +echo "" +echo "Next steps:" +echo " 1. Copy kubeconfig to Mac: scp legion:/home/will/.kube/config ~/.kube/legion-config" +echo " 2. Apply Terraform: cd ~/Development/infrastructure/servers/legion && direnv exec . terraform apply" +echo " 3. Restore Gitea data: see RUNBOOK.md § Data Restore" +echo "" +echo "Verify GPU access:" +echo " ssh legion kubectl describe node | grep nvidia.com/gpu"