ci: add gitea actions runner on GCP with WIF-backed deploy SA

A single e2-standard-4 GCE VM in us-central1-a runs act_runner, registered to git.neuralplatform.ai. Workload Identity Federation binds the runner to the existing neuron-ci-pusher SA (artifactregistry.writer + run.developer + serviceAccountUser on the runtime SAs). No long-lived JSON keys live on the runner; the GCP_SA_KEY secret remains as a fallback in case the Gitea OIDC issuer isn't reachable from oauth2.googleapis.com. The runner VM has its own minimal-scope SA (gitea-runner-vm) that can only read the registration token from Secret Manager — splitting boot identity from deploy identity so a runner compromise doesn't grant push. SSH is IAP-only (no public ingress on :22). Reference workflow lives at products/web/.gitea/workflows/deploy.yaml and takes manual gcloud out of the loop: push to main triggers build, push to AR, parallel deploy to all 3 marketing prod regions, traffic flip, smoke check.
2026-05-02 12:45:25 -05:00
parent 7ee6099139
commit 0c32964ead
3 changed files with 534 additions and 0 deletions
@@ -0,0 +1,287 @@
+# ── Gitea Actions runner on GCE ───────────────────────────────────────────────
+# A single e2-standard-4 VM in us-central1-a runs `act_runner`, registered
+# against https://git.neuralplatform.ai. CI workflows that say
+# `runs-on: ubuntu-latest` get scheduled here.
+#
+# Identity model
+#   • The VM mounts the `gitea-runner-vm` SA so it can pull the runner
+#     registration token from Secret Manager at first boot.
+#   • Workflows authenticate to GCP via Workload Identity Federation —
+#     they exchange a Gitea OIDC token for a short-lived access token
+#     impersonating the existing `neuron-ci-pusher` SA. No long-lived
+#     JSON keys live on the runner.
+#   • If Gitea OIDC is unavailable (older Gitea), the workflow can fall
+#     back to `credentials_json: ${{ secrets.GCP_SA_KEY }}` and the
+#     existing JSON key flow keeps working — see the README for details.
+#
+# IAM scope
+#   The deploy SA already holds artifactregistry.writer + run.developer
+#   + serviceAccountUser on the runtime SAs (see artifact-registry.tf).
+#   We add only what's missing here: Cloud Run service-level developer
+#   bindings and the WIF binding itself.
+
+# ── Runner VM service account ─────────────────────────────────────────────────
+# Boot-time identity. Only role: read the registration token. We deliberately
+# DO NOT grant deploy permissions to this SA — workflow steps assume the
+# deploy SA via WIF instead, so a runner compromise doesn't grant push.
+
+resource "google_service_account" "gitea_runner_vm" {
+  account_id   = "gitea-runner-vm"
+  display_name = "Gitea Actions runner (VM identity)"
+  description  = "Boot-time identity for the GCE-hosted Gitea Actions runner. Read-only access to the registration token."
+  project      = var.project_id
+}
+
+resource "google_secret_manager_secret_iam_member" "runner_token_access" {
+  project   = var.project_id
+  secret_id = "gitea-runner-token"
+  role      = "roles/secretmanager.secretAccessor"
+  member    = "serviceAccount:${google_service_account.gitea_runner_vm.email}"
+}
+
+# Logs/metrics from the VM's ops agent (if installed later)
+resource "google_project_iam_member" "runner_log_writer" {
+  project = var.project_id
+  role    = "roles/logging.logWriter"
+  member  = "serviceAccount:${google_service_account.gitea_runner_vm.email}"
+}
+
+resource "google_project_iam_member" "runner_metric_writer" {
+  project = var.project_id
+  role    = "roles/monitoring.metricWriter"
+  member  = "serviceAccount:${google_service_account.gitea_runner_vm.email}"
+}
+
+# ── Cloud Run service-level deploy bindings ───────────────────────────────────
+# `roles/run.developer` is granted project-wide on the ci_pusher SA in
+# artifact-registry.tf — that's the simple path. We layer service-level
+# bindings on top so the principle of least privilege is documented per
+# service (and so future scoping reductions are mechanical).
+
+locals {
+  ci_marketing_services = {
+    prod_us   = { region = "us-central1",     name = google_cloud_run_v2_service.prod_us.name }
+    prod_eu   = { region = "europe-west1",    name = google_cloud_run_v2_service.prod_eu.name }
+    prod_apac = { region = "asia-northeast1", name = google_cloud_run_v2_service.prod_apac.name }
+  }
+}
+
+resource "google_cloud_run_v2_service_iam_member" "ci_pusher_marketing_developer" {
+  for_each = local.ci_marketing_services
+  project  = var.project_id
+  location = each.value.region
+  name     = each.value.name
+  role     = "roles/run.developer"
+  member   = "serviceAccount:${google_service_account.ci_pusher.email}"
+}
+
+# ── Startup script staged in GCS ──────────────────────────────────────────────
+# We stage the bootstrap script in a private GCS bucket and pull it via
+# `gsutil cat` from a tiny bootstrap one-liner in metadata. This keeps the
+# Terraform diff readable when the script changes.
+
+resource "google_storage_bucket" "runner_assets" {
+  name                        = "${var.project_id}-runner-assets"
+  location                    = "US-CENTRAL1"
+  project                     = var.project_id
+  uniform_bucket_level_access = true
+  force_destroy               = false
+
+  versioning {
+    enabled = true
+  }
+
+  lifecycle_rule {
+    condition {
+      num_newer_versions = 5
+    }
+    action {
+      type = "Delete"
+    }
+  }
+}
+
+resource "google_storage_bucket_iam_member" "runner_bucket_read" {
+  bucket = google_storage_bucket.runner_assets.name
+  role   = "roles/storage.objectViewer"
+  member = "serviceAccount:${google_service_account.gitea_runner_vm.email}"
+}
+
+resource "google_storage_bucket_object" "runner_startup" {
+  name   = "gitea-runner/startup.sh"
+  bucket = google_storage_bucket.runner_assets.name
+  source = "${path.module}/runners/startup.sh"
+
+  # Force a re-deploy of the VM when the script changes
+  metadata = {
+    sha256 = filesha256("${path.module}/runners/startup.sh")
+  }
+}
+
+# ── The runner VM ─────────────────────────────────────────────────────────────
+
+resource "google_compute_instance" "gitea_runner" {
+  name         = "gitea-runner-1"
+  machine_type = "e2-standard-4" # 4 vCPU / 16 GiB — comfortable for docker builds
+  zone         = "us-central1-a"
+  project      = var.project_id
+
+  tags = ["gitea-runner"]
+
+  boot_disk {
+    initialize_params {
+      image = "projects/ubuntu-os-cloud/global/images/family/ubuntu-2404-lts-amd64"
+      size  = 50
+      type  = "pd-balanced"
+    }
+  }
+
+  network_interface {
+    network = "default"
+    # Ephemeral external IP — the runner needs outbound to git.neuralplatform.ai
+    # via Cloudflare and to *.googleapis.com. No inbound is permitted (no
+    # public SSH firewall rule below); ops is via IAP.
+    access_config {}
+  }
+
+  service_account {
+    email  = google_service_account.gitea_runner_vm.email
+    scopes = ["cloud-platform"]
+  }
+
+  metadata = {
+    # Bootstrap shim — fetch and execute the real startup script from GCS.
+    # gsutil ships with the Google SDK preinstalled on Ubuntu base images.
+    startup-script = <<-EOT
+      #!/usr/bin/env bash
+      set -euxo pipefail
+      apt-get update
+      apt-get install -y curl ca-certificates apt-transport-https gnupg
+      curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+      echo "deb https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list
+      apt-get update
+      apt-get install -y google-cloud-cli
+      gsutil cat gs://${google_storage_bucket.runner_assets.name}/${google_storage_bucket_object.runner_startup.name} > /tmp/startup.sh
+      chmod +x /tmp/startup.sh
+      /tmp/startup.sh
+    EOT
+
+    enable-oslogin = "TRUE"
+  }
+
+  # Stop -> change machine type / metadata -> start cleanly
+  allow_stopping_for_update = true
+
+  depends_on = [
+    google_secret_manager_secret_iam_member.runner_token_access,
+    google_storage_bucket_object.runner_startup,
+  ]
+}
+
+# ── Firewall: IAP-only SSH ────────────────────────────────────────────────────
+# 35.235.240.0/20 is GCP's IAP CIDR. SSH from anywhere else is dropped.
+# Ops shell:
+#   gcloud compute ssh gitea-runner-1 --zone=us-central1-a --tunnel-through-iap
+
+resource "google_compute_firewall" "gitea_runner_iap_ssh" {
+  name      = "gitea-runner-iap-ssh"
+  network   = "default"
+  project   = var.project_id
+  direction = "INGRESS"
+  priority  = 1000
+
+  allow {
+    protocol = "tcp"
+    ports    = ["22"]
+  }
+
+  source_ranges = ["35.235.240.0/20"]
+  target_tags   = ["gitea-runner"]
+}
+
+# ── Workload Identity Federation: Gitea → GCP ─────────────────────────────────
+# Gitea Actions issues OIDC tokens with `iss=https://git.neuralplatform.ai`
+# (verified at runtime via the .well-known endpoint). The pool below trusts
+# that issuer; the provider maps token claims into Google principal subjects.
+#
+# A workflow then runs:
+#   uses: google-github-actions/auth@v2
+#   with:
+#     workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER }}
+#     service_account: ${{ secrets.GCP_DEPLOY_SA }}
+# and the auth action exchanges the OIDC token for a short-lived access
+# token that impersonates google_service_account.ci_pusher.
+#
+# If Gitea's OIDC isn't reachable from `oauth2.googleapis.com` (CF Access in
+# front of the .well-known endpoint, etc.), the JSON-key fallback documented
+# in artifact-registry.tf still works.
+
+resource "google_iam_workload_identity_pool" "gitea" {
+  workload_identity_pool_id = "gitea-pool"
+  project                   = var.project_id
+  display_name              = "Gitea Actions"
+  description               = "Trust pool for OIDC tokens from git.neuralplatform.ai"
+}
+
+resource "google_iam_workload_identity_pool_provider" "gitea" {
+  workload_identity_pool_id          = google_iam_workload_identity_pool.gitea.workload_identity_pool_id
+  workload_identity_pool_provider_id = "gitea-oidc"
+  project                            = var.project_id
+  display_name                       = "Gitea OIDC"
+
+  attribute_mapping = {
+    "google.subject"       = "assertion.sub"
+    "attribute.repository" = "assertion.repository"
+    "attribute.actor"      = "assertion.actor"
+    "attribute.ref"        = "assertion.ref"
+  }
+
+  # Restrict which Gitea repos can mint tokens for this pool.
+  attribute_condition = "assertion.repository.startsWith('neuron-technologies/')"
+
+  oidc {
+    issuer_uri = "https://git.neuralplatform.ai"
+    # No `allowed_audiences` set — we accept the default audience the
+    # google-github-actions/auth action requests.
+  }
+}
+
+# Allow workflows from the neuron-technologies/neuron-web repo to impersonate
+# the deploy SA. To extend to additional repos, add a second member.
+resource "google_service_account_iam_member" "ci_pusher_wif_neuron_web" {
+  service_account_id = google_service_account.ci_pusher.name
+  role               = "roles/iam.workloadIdentityUser"
+  member             = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.gitea.name}/attribute.repository/neuron-technologies/neuron-web"
+}
+
+# ── Outputs ───────────────────────────────────────────────────────────────────
+
+output "gitea_runner_vm_name" {
+  description = "GCE instance name for the Gitea Actions runner"
+  value       = google_compute_instance.gitea_runner.name
+}
+
+output "gitea_runner_vm_zone" {
+  description = "GCE zone for the runner"
+  value       = google_compute_instance.gitea_runner.zone
+}
+
+output "gitea_runner_vm_external_ip" {
+  description = "Ephemeral external IP for the runner (for outbound only; no inbound public SSH)"
+  value       = google_compute_instance.gitea_runner.network_interface[0].access_config[0].nat_ip
+}
+
+output "gitea_runner_sa_email" {
+  description = "Boot-time identity for the runner VM"
+  value       = google_service_account.gitea_runner_vm.email
+}
+
+output "gitea_wif_provider" {
+  description = "Full resource name for the Gitea OIDC WIF provider — set as Gitea repo secret GCP_WIF_PROVIDER"
+  value       = google_iam_workload_identity_pool_provider.gitea.name
+}
+
+output "gitea_wif_deploy_sa" {
+  description = "Service account workflows impersonate via WIF — set as Gitea repo secret GCP_DEPLOY_SA"
+  value       = google_service_account.ci_pusher.email
+}
@@ -0,0 +1,150 @@
+# Gitea Actions runner on GCP
+
+A single GCE VM in `us-central1-a` running [`act_runner`](https://gitea.com/gitea/act_runner)
+registered against `https://git.neuralplatform.ai`. Workflows that say
+`runs-on: ubuntu-latest` are scheduled here.
+
+Source of truth: `servers/gcp/gitea-runner.tf` plus this directory.
+
+## What this runner does
+
+- Picks up `.gitea/workflows/*.yaml` jobs from any Gitea repo whose
+  workflow targets one of its labels (`ubuntu-latest`, `ubuntu-22.04`,
+  `ubuntu-24.04`).
+- Builds Docker images on a 4 vCPU / 16 GiB host with native Docker
+  installed (no Docker-in-Docker — `/var/run/docker.sock` is mounted into
+  the runner user's group).
+- Authenticates to GCP via Workload Identity Federation. Workflows
+  exchange a Gitea OIDC token for a short-lived access token impersonating
+  the `neuron-ci-pusher` service account. **No long-lived keys live on the
+  runner.**
+
+## Identity model
+
+| Identity | Used by | Purpose |
+|---|---|---|
+| `gitea-runner-vm@neuron-785695.iam.gserviceaccount.com` | The VM at boot | Read the runner registration token from Secret Manager. Nothing else. |
+| `neuron-ci-pusher@neuron-785695.iam.gserviceaccount.com` | Workflow steps | Push images to Artifact Registry, deploy Cloud Run revisions, act-as the runtime SAs. Impersonated via WIF. |
+
+Splitting these means a runner-level compromise does not grant deploy access.
+
+## Setup checklist (one-time)
+
+1. **Create the registration token secret** (already done — re-run if rotating):
+   ```bash
+   TOKEN=$(curl -s -X GET https://git.neuralplatform.ai/api/v1/admin/runners/registration-token \
+     -H "Authorization: token $(cat ~/Secrets/api-keys/gitea-api-token)" | jq -r .token)
+   echo -n "$TOKEN" | gcloud secrets versions add gitea-runner-token \
+     --project=neuron-785695 --data-file=-
+   ```
+
+2. **`terraform apply`** in `servers/gcp/`. Outputs you will need:
+   ```
+   gitea_wif_provider   = projects/<num>/locations/global/workloadIdentityPools/gitea-pool/providers/gitea-oidc
+   gitea_wif_deploy_sa  = neuron-ci-pusher@neuron-785695.iam.gserviceaccount.com
+   ```
+
+3. **Set Gitea repo secrets** on every repo whose workflows deploy:
+   - `GCP_WIF_PROVIDER` — the `gitea_wif_provider` output above
+   - `GCP_DEPLOY_SA` — the `gitea_wif_deploy_sa` output above
+
+   Via the Gitea UI: repo → Settings → Actions → Secrets, or via API:
+   ```bash
+   tea secret set --repo neuron-technologies/neuron-web GCP_WIF_PROVIDER "<provider>"
+   tea secret set --repo neuron-technologies/neuron-web GCP_DEPLOY_SA    "<sa-email>"
+   ```
+
+4. **Confirm the runner registered**:
+   ```bash
+   curl -s https://git.neuralplatform.ai/api/v1/admin/runners \
+     -H "Authorization: token $(cat ~/Secrets/api-keys/gitea-api-token)"
+   ```
+   The runner appears as `gcp-us-central1-runner-1` after the VM finishes its
+   startup script (~3 minutes from `terraform apply`).
+
+## Adding more runners
+
+Today the count is hard-coded at one. To add a second runner: copy
+`google_compute_instance.gitea_runner` in `gitea-runner.tf`, rename to
+`gitea_runner_2`, give it the name `gitea-runner-2`, and reuse the same
+SA + startup script. (Eventual TODO: parameterize via `count` or
+`for_each`.)
+
+## SSH into the runner (no public exposure)
+
+```bash
+gcloud compute ssh gitea-runner-1 \
+  --zone=us-central1-a \
+  --project=neuron-785695 \
+  --tunnel-through-iap
+```
+
+The firewall only accepts SSH from GCP's IAP CIDR (`35.235.240.0/20`),
+so this is the only way in. Your gcloud account needs
+`roles/iap.tunnelResourceAccessor` (granted automatically to project
+owners).
+
+## Debugging a failing build
+
+On the runner:
+
+```bash
+sudo journalctl -u act_runner -f                     # runner daemon logs
+sudo cat /var/log/runner-bootstrap.log               # boot-time install log
+sudo systemctl status act_runner                     # daemon state
+docker ps                                            # in-flight build containers
+ls -la /opt/runner                                   # config + .runner state file
+```
+
+For a workflow run the live logs are at
+`https://git.neuralplatform.ai/<owner>/<repo>/actions`.
+
+## Rotating the registration token
+
+```bash
+TOKEN=$(curl -s -X GET https://git.neuralplatform.ai/api/v1/admin/runners/registration-token \
+  -H "Authorization: token $(cat ~/Secrets/api-keys/gitea-api-token)" | jq -r .token)
+echo -n "$TOKEN" | gcloud secrets versions add gitea-runner-token \
+  --project=neuron-785695 --data-file=-
+
+# Re-register (token is single-use). Either taint the VM:
+terraform taint google_compute_instance.gitea_runner
+terraform apply
+
+# ...or re-run register on the existing VM via IAP SSH:
+gcloud compute ssh gitea-runner-1 --zone=us-central1-a --tunnel-through-iap -- "\
+  sudo systemctl stop act_runner && \
+  sudo -u runner /usr/local/bin/act_runner register --no-interactive \
+    --instance https://git.neuralplatform.ai \
+    --token $TOKEN \
+    --name gcp-us-central1-runner-1 \
+    --labels ubuntu-latest:host,ubuntu-22.04:host,ubuntu-24.04:host && \
+  sudo systemctl start act_runner"
+```
+
+## Fallback: long-lived JSON key (only if WIF isn't usable)
+
+If Gitea's OIDC discovery endpoint isn't reachable from
+`oauth2.googleapis.com` (e.g. CF Access blocks `.well-known/openid-configuration`
+to anonymous fetchers), workflows can't use WIF. Fall back to the JSON
+key flow:
+
+```bash
+gcloud iam service-accounts keys create /tmp/ci-pusher.json \
+  --iam-account=neuron-ci-pusher@neuron-785695.iam.gserviceaccount.com
+tea secret set --repo neuron-technologies/neuron-web GCP_SA_KEY < /tmp/ci-pusher.json
+rm /tmp/ci-pusher.json
+```
+
+Then in `deploy.yaml`, swap the `auth` step's `workload_identity_provider`/
+`service_account` inputs for `credentials_json: ${{ secrets.GCP_SA_KEY }}`.
+
+The WIF resources stay in Terraform either way — the fallback is purely
+in the workflow. WIF remains the target.
+
+## Cost
+
+`e2-standard-4` in `us-central1` is roughly **$100/mo** sustained-use,
+plus a few cents for the GCS bucket and the ephemeral IP. Not free. If
+that's too much, downsize to `e2-standard-2` (~$50/mo) — most builds
+fit, only the heaviest C++/Rust compiles need 16 GiB.
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# servers/gcp/runners/startup.sh
+#
+# Boot script for the Gitea Actions runner GCE VM.
+#   - Installs docker + git + jq + the Forgejo/Gitea act_runner binary.
+#   - Pulls the runner registration token from GCP Secret Manager
+#     (the VM's attached service account has secretmanager.secretAccessor on it).
+#   - Registers against https://git.neuralplatform.ai with labels matching
+#     the workflows that target ubuntu-latest hosts.
+#   - Starts act_runner under systemd so it auto-recovers on reboot.
+#
+# Logs land at /var/log/runner-bootstrap.log for debugging from IAP SSH.
+
+set -euxo pipefail
+exec > >(tee /var/log/runner-bootstrap.log) 2>&1
+
+apt-get update
+apt-get install -y curl ca-certificates docker.io git jq
+
+# Make docker usable by the unprivileged runner user
+systemctl enable --now docker
+
+useradd -m -s /bin/bash runner || true
+usermod -aG docker runner
+
+# act_runner — pinned to a known-good release. Bump RUNNER_VERSION when
+# upgrading. The project moved from gitea/act_runner to gitea/runner around
+# the 0.6.x series; the binary inside the asset is still called act_runner.
+# Latest releases at: https://gitea.com/gitea/runner/releases
+RUNNER_VERSION="0.6.1"
+curl -fsSL \
+  "https://gitea.com/gitea/runner/releases/download/v${RUNNER_VERSION}/act_runner-${RUNNER_VERSION}-linux-amd64" \
+  -o /usr/local/bin/act_runner
+chmod +x /usr/local/bin/act_runner
+
+mkdir -p /opt/runner
+chown -R runner:runner /opt/runner
+
+# Pull registration token. Fail fast if the secret is missing — the runner
+# is useless without it, and a silent fallback would leave us debugging an
+# unregistered VM later.
+GITEA_RUNNER_TOKEN="$(gcloud secrets versions access latest \
+  --secret=gitea-runner-token \
+  --project=neuron-785695)"
+
+# Generate a baseline config so the daemon has a sane starting point. The
+# register step writes the .runner state file into /opt/runner.
+sudo -u runner /usr/local/bin/act_runner generate-config > /opt/runner/config.yaml
+chown runner:runner /opt/runner/config.yaml
+
+cd /opt/runner
+sudo -u runner /usr/local/bin/act_runner register \
+  --no-interactive \
+  --instance https://git.neuralplatform.ai \
+  --token "${GITEA_RUNNER_TOKEN}" \
+  --name "gcp-us-central1-runner-1" \
+  --labels "ubuntu-latest,ubuntu-22.04,ubuntu-24.04"
+
+# Belt-and-braces: act_runner v0.6 has a habit of wrapping bare labels with
+# the default `:docker://docker.gitea.com/runner-images:<label>` shape on
+# first daemon start. We want host execution (build-stage.sh uses the
+# host's Docker socket), so re-write the labels in the persisted state
+# file before we start the daemon.
+python3 - <<'PY'
+import json
+p = "/opt/runner/.runner"
+with open(p) as f: d = json.load(f)
+d["labels"] = ["ubuntu-latest", "ubuntu-22.04", "ubuntu-24.04"]
+with open(p, "w") as f: json.dump(d, f, indent=2)
+PY
+chown runner:runner /opt/runner/.runner
+
+cat > /etc/systemd/system/act_runner.service <<'EOF'
+[Unit]
+Description=Gitea Actions runner (act_runner)
+After=network-online.target docker.service
+Wants=network-online.target
+Requires=docker.service
+
+[Service]
+Type=simple
+User=runner
+WorkingDirectory=/opt/runner
+ExecStart=/usr/local/bin/act_runner daemon --config /opt/runner/config.yaml
+Restart=always
+RestartSec=5
+# Pass through GCP metadata-server access so workflows can call gcloud
+Environment=HOME=/home/runner
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload
+systemctl enable --now act_runner
+
+echo "act_runner registered and running"