From 0c32964ead9f4e5a7d61dd1c90696f009497ca62 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Sat, 2 May 2026 12:45:25 -0500 Subject: [PATCH] ci: add gitea actions runner on GCP with WIF-backed deploy SA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single e2-standard-4 GCE VM in us-central1-a runs act_runner, registered to git.neuralplatform.ai. Workload Identity Federation binds the runner to the existing neuron-ci-pusher SA (artifactregistry.writer + run.developer + serviceAccountUser on the runtime SAs). No long-lived JSON keys live on the runner; the GCP_SA_KEY secret remains as a fallback in case the Gitea OIDC issuer isn't reachable from oauth2.googleapis.com. The runner VM has its own minimal-scope SA (gitea-runner-vm) that can only read the registration token from Secret Manager — splitting boot identity from deploy identity so a runner compromise doesn't grant push. SSH is IAP-only (no public ingress on :22). Reference workflow lives at products/web/.gitea/workflows/deploy.yaml and takes manual gcloud out of the loop: push to main triggers build, push to AR, parallel deploy to all 3 marketing prod regions, traffic flip, smoke check. --- servers/gcp/gitea-runner.tf | 287 +++++++++++++++++++++++++++++++++ servers/gcp/runners/README.md | 150 +++++++++++++++++ servers/gcp/runners/startup.sh | 97 +++++++++++ 3 files changed, 534 insertions(+) create mode 100644 servers/gcp/gitea-runner.tf create mode 100644 servers/gcp/runners/README.md create mode 100644 servers/gcp/runners/startup.sh diff --git a/servers/gcp/gitea-runner.tf b/servers/gcp/gitea-runner.tf new file mode 100644 index 0000000..d3a9b30 --- /dev/null +++ b/servers/gcp/gitea-runner.tf @@ -0,0 +1,287 @@ +# ── Gitea Actions runner on GCE ─────────────────────────────────────────────── +# A single e2-standard-4 VM in us-central1-a runs `act_runner`, registered +# against https://git.neuralplatform.ai. CI workflows that say +# `runs-on: ubuntu-latest` get scheduled here. +# +# Identity model +# • The VM mounts the `gitea-runner-vm` SA so it can pull the runner +# registration token from Secret Manager at first boot. +# • Workflows authenticate to GCP via Workload Identity Federation — +# they exchange a Gitea OIDC token for a short-lived access token +# impersonating the existing `neuron-ci-pusher` SA. No long-lived +# JSON keys live on the runner. +# • If Gitea OIDC is unavailable (older Gitea), the workflow can fall +# back to `credentials_json: ${{ secrets.GCP_SA_KEY }}` and the +# existing JSON key flow keeps working — see the README for details. +# +# IAM scope +# The deploy SA already holds artifactregistry.writer + run.developer +# + serviceAccountUser on the runtime SAs (see artifact-registry.tf). +# We add only what's missing here: Cloud Run service-level developer +# bindings and the WIF binding itself. + +# ── Runner VM service account ───────────────────────────────────────────────── +# Boot-time identity. Only role: read the registration token. We deliberately +# DO NOT grant deploy permissions to this SA — workflow steps assume the +# deploy SA via WIF instead, so a runner compromise doesn't grant push. + +resource "google_service_account" "gitea_runner_vm" { + account_id = "gitea-runner-vm" + display_name = "Gitea Actions runner (VM identity)" + description = "Boot-time identity for the GCE-hosted Gitea Actions runner. Read-only access to the registration token." + project = var.project_id +} + +resource "google_secret_manager_secret_iam_member" "runner_token_access" { + project = var.project_id + secret_id = "gitea-runner-token" + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.gitea_runner_vm.email}" +} + +# Logs/metrics from the VM's ops agent (if installed later) +resource "google_project_iam_member" "runner_log_writer" { + project = var.project_id + role = "roles/logging.logWriter" + member = "serviceAccount:${google_service_account.gitea_runner_vm.email}" +} + +resource "google_project_iam_member" "runner_metric_writer" { + project = var.project_id + role = "roles/monitoring.metricWriter" + member = "serviceAccount:${google_service_account.gitea_runner_vm.email}" +} + +# ── Cloud Run service-level deploy bindings ─────────────────────────────────── +# `roles/run.developer` is granted project-wide on the ci_pusher SA in +# artifact-registry.tf — that's the simple path. We layer service-level +# bindings on top so the principle of least privilege is documented per +# service (and so future scoping reductions are mechanical). + +locals { + ci_marketing_services = { + prod_us = { region = "us-central1", name = google_cloud_run_v2_service.prod_us.name } + prod_eu = { region = "europe-west1", name = google_cloud_run_v2_service.prod_eu.name } + prod_apac = { region = "asia-northeast1", name = google_cloud_run_v2_service.prod_apac.name } + } +} + +resource "google_cloud_run_v2_service_iam_member" "ci_pusher_marketing_developer" { + for_each = local.ci_marketing_services + project = var.project_id + location = each.value.region + name = each.value.name + role = "roles/run.developer" + member = "serviceAccount:${google_service_account.ci_pusher.email}" +} + +# ── Startup script staged in GCS ────────────────────────────────────────────── +# We stage the bootstrap script in a private GCS bucket and pull it via +# `gsutil cat` from a tiny bootstrap one-liner in metadata. This keeps the +# Terraform diff readable when the script changes. + +resource "google_storage_bucket" "runner_assets" { + name = "${var.project_id}-runner-assets" + location = "US-CENTRAL1" + project = var.project_id + uniform_bucket_level_access = true + force_destroy = false + + versioning { + enabled = true + } + + lifecycle_rule { + condition { + num_newer_versions = 5 + } + action { + type = "Delete" + } + } +} + +resource "google_storage_bucket_iam_member" "runner_bucket_read" { + bucket = google_storage_bucket.runner_assets.name + role = "roles/storage.objectViewer" + member = "serviceAccount:${google_service_account.gitea_runner_vm.email}" +} + +resource "google_storage_bucket_object" "runner_startup" { + name = "gitea-runner/startup.sh" + bucket = google_storage_bucket.runner_assets.name + source = "${path.module}/runners/startup.sh" + + # Force a re-deploy of the VM when the script changes + metadata = { + sha256 = filesha256("${path.module}/runners/startup.sh") + } +} + +# ── The runner VM ───────────────────────────────────────────────────────────── + +resource "google_compute_instance" "gitea_runner" { + name = "gitea-runner-1" + machine_type = "e2-standard-4" # 4 vCPU / 16 GiB — comfortable for docker builds + zone = "us-central1-a" + project = var.project_id + + tags = ["gitea-runner"] + + boot_disk { + initialize_params { + image = "projects/ubuntu-os-cloud/global/images/family/ubuntu-2404-lts-amd64" + size = 50 + type = "pd-balanced" + } + } + + network_interface { + network = "default" + # Ephemeral external IP — the runner needs outbound to git.neuralplatform.ai + # via Cloudflare and to *.googleapis.com. No inbound is permitted (no + # public SSH firewall rule below); ops is via IAP. + access_config {} + } + + service_account { + email = google_service_account.gitea_runner_vm.email + scopes = ["cloud-platform"] + } + + metadata = { + # Bootstrap shim — fetch and execute the real startup script from GCS. + # gsutil ships with the Google SDK preinstalled on Ubuntu base images. + startup-script = <<-EOT + #!/usr/bin/env bash + set -euxo pipefail + apt-get update + apt-get install -y curl ca-certificates apt-transport-https gnupg + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + echo "deb https://packages.cloud.google.com/apt cloud-sdk main" > /etc/apt/sources.list.d/google-cloud-sdk.list + apt-get update + apt-get install -y google-cloud-cli + gsutil cat gs://${google_storage_bucket.runner_assets.name}/${google_storage_bucket_object.runner_startup.name} > /tmp/startup.sh + chmod +x /tmp/startup.sh + /tmp/startup.sh + EOT + + enable-oslogin = "TRUE" + } + + # Stop -> change machine type / metadata -> start cleanly + allow_stopping_for_update = true + + depends_on = [ + google_secret_manager_secret_iam_member.runner_token_access, + google_storage_bucket_object.runner_startup, + ] +} + +# ── Firewall: IAP-only SSH ──────────────────────────────────────────────────── +# 35.235.240.0/20 is GCP's IAP CIDR. SSH from anywhere else is dropped. +# Ops shell: +# gcloud compute ssh gitea-runner-1 --zone=us-central1-a --tunnel-through-iap + +resource "google_compute_firewall" "gitea_runner_iap_ssh" { + name = "gitea-runner-iap-ssh" + network = "default" + project = var.project_id + direction = "INGRESS" + priority = 1000 + + allow { + protocol = "tcp" + ports = ["22"] + } + + source_ranges = ["35.235.240.0/20"] + target_tags = ["gitea-runner"] +} + +# ── Workload Identity Federation: Gitea → GCP ───────────────────────────────── +# Gitea Actions issues OIDC tokens with `iss=https://git.neuralplatform.ai` +# (verified at runtime via the .well-known endpoint). The pool below trusts +# that issuer; the provider maps token claims into Google principal subjects. +# +# A workflow then runs: +# uses: google-github-actions/auth@v2 +# with: +# workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER }} +# service_account: ${{ secrets.GCP_DEPLOY_SA }} +# and the auth action exchanges the OIDC token for a short-lived access +# token that impersonates google_service_account.ci_pusher. +# +# If Gitea's OIDC isn't reachable from `oauth2.googleapis.com` (CF Access in +# front of the .well-known endpoint, etc.), the JSON-key fallback documented +# in artifact-registry.tf still works. + +resource "google_iam_workload_identity_pool" "gitea" { + workload_identity_pool_id = "gitea-pool" + project = var.project_id + display_name = "Gitea Actions" + description = "Trust pool for OIDC tokens from git.neuralplatform.ai" +} + +resource "google_iam_workload_identity_pool_provider" "gitea" { + workload_identity_pool_id = google_iam_workload_identity_pool.gitea.workload_identity_pool_id + workload_identity_pool_provider_id = "gitea-oidc" + project = var.project_id + display_name = "Gitea OIDC" + + attribute_mapping = { + "google.subject" = "assertion.sub" + "attribute.repository" = "assertion.repository" + "attribute.actor" = "assertion.actor" + "attribute.ref" = "assertion.ref" + } + + # Restrict which Gitea repos can mint tokens for this pool. + attribute_condition = "assertion.repository.startsWith('neuron-technologies/')" + + oidc { + issuer_uri = "https://git.neuralplatform.ai" + # No `allowed_audiences` set — we accept the default audience the + # google-github-actions/auth action requests. + } +} + +# Allow workflows from the neuron-technologies/neuron-web repo to impersonate +# the deploy SA. To extend to additional repos, add a second member. +resource "google_service_account_iam_member" "ci_pusher_wif_neuron_web" { + service_account_id = google_service_account.ci_pusher.name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.gitea.name}/attribute.repository/neuron-technologies/neuron-web" +} + +# ── Outputs ─────────────────────────────────────────────────────────────────── + +output "gitea_runner_vm_name" { + description = "GCE instance name for the Gitea Actions runner" + value = google_compute_instance.gitea_runner.name +} + +output "gitea_runner_vm_zone" { + description = "GCE zone for the runner" + value = google_compute_instance.gitea_runner.zone +} + +output "gitea_runner_vm_external_ip" { + description = "Ephemeral external IP for the runner (for outbound only; no inbound public SSH)" + value = google_compute_instance.gitea_runner.network_interface[0].access_config[0].nat_ip +} + +output "gitea_runner_sa_email" { + description = "Boot-time identity for the runner VM" + value = google_service_account.gitea_runner_vm.email +} + +output "gitea_wif_provider" { + description = "Full resource name for the Gitea OIDC WIF provider — set as Gitea repo secret GCP_WIF_PROVIDER" + value = google_iam_workload_identity_pool_provider.gitea.name +} + +output "gitea_wif_deploy_sa" { + description = "Service account workflows impersonate via WIF — set as Gitea repo secret GCP_DEPLOY_SA" + value = google_service_account.ci_pusher.email +} diff --git a/servers/gcp/runners/README.md b/servers/gcp/runners/README.md new file mode 100644 index 0000000..1f250de --- /dev/null +++ b/servers/gcp/runners/README.md @@ -0,0 +1,150 @@ +# Gitea Actions runner on GCP + +A single GCE VM in `us-central1-a` running [`act_runner`](https://gitea.com/gitea/act_runner) +registered against `https://git.neuralplatform.ai`. Workflows that say +`runs-on: ubuntu-latest` are scheduled here. + +Source of truth: `servers/gcp/gitea-runner.tf` plus this directory. + +## What this runner does + +- Picks up `.gitea/workflows/*.yaml` jobs from any Gitea repo whose + workflow targets one of its labels (`ubuntu-latest`, `ubuntu-22.04`, + `ubuntu-24.04`). +- Builds Docker images on a 4 vCPU / 16 GiB host with native Docker + installed (no Docker-in-Docker — `/var/run/docker.sock` is mounted into + the runner user's group). +- Authenticates to GCP via Workload Identity Federation. Workflows + exchange a Gitea OIDC token for a short-lived access token impersonating + the `neuron-ci-pusher` service account. **No long-lived keys live on the + runner.** + +## Identity model + +| Identity | Used by | Purpose | +|---|---|---| +| `gitea-runner-vm@neuron-785695.iam.gserviceaccount.com` | The VM at boot | Read the runner registration token from Secret Manager. Nothing else. | +| `neuron-ci-pusher@neuron-785695.iam.gserviceaccount.com` | Workflow steps | Push images to Artifact Registry, deploy Cloud Run revisions, act-as the runtime SAs. Impersonated via WIF. | + +Splitting these means a runner-level compromise does not grant deploy access. + +## Setup checklist (one-time) + +1. **Create the registration token secret** (already done — re-run if rotating): + ```bash + TOKEN=$(curl -s -X GET https://git.neuralplatform.ai/api/v1/admin/runners/registration-token \ + -H "Authorization: token $(cat ~/Secrets/api-keys/gitea-api-token)" | jq -r .token) + echo -n "$TOKEN" | gcloud secrets versions add gitea-runner-token \ + --project=neuron-785695 --data-file=- + ``` + +2. **`terraform apply`** in `servers/gcp/`. Outputs you will need: + ``` + gitea_wif_provider = projects//locations/global/workloadIdentityPools/gitea-pool/providers/gitea-oidc + gitea_wif_deploy_sa = neuron-ci-pusher@neuron-785695.iam.gserviceaccount.com + ``` + +3. **Set Gitea repo secrets** on every repo whose workflows deploy: + - `GCP_WIF_PROVIDER` — the `gitea_wif_provider` output above + - `GCP_DEPLOY_SA` — the `gitea_wif_deploy_sa` output above + + Via the Gitea UI: repo → Settings → Actions → Secrets, or via API: + ```bash + tea secret set --repo neuron-technologies/neuron-web GCP_WIF_PROVIDER "" + tea secret set --repo neuron-technologies/neuron-web GCP_DEPLOY_SA "" + ``` + +4. **Confirm the runner registered**: + ```bash + curl -s https://git.neuralplatform.ai/api/v1/admin/runners \ + -H "Authorization: token $(cat ~/Secrets/api-keys/gitea-api-token)" + ``` + The runner appears as `gcp-us-central1-runner-1` after the VM finishes its + startup script (~3 minutes from `terraform apply`). + +## Adding more runners + +Today the count is hard-coded at one. To add a second runner: copy +`google_compute_instance.gitea_runner` in `gitea-runner.tf`, rename to +`gitea_runner_2`, give it the name `gitea-runner-2`, and reuse the same +SA + startup script. (Eventual TODO: parameterize via `count` or +`for_each`.) + +## SSH into the runner (no public exposure) + +```bash +gcloud compute ssh gitea-runner-1 \ + --zone=us-central1-a \ + --project=neuron-785695 \ + --tunnel-through-iap +``` + +The firewall only accepts SSH from GCP's IAP CIDR (`35.235.240.0/20`), +so this is the only way in. Your gcloud account needs +`roles/iap.tunnelResourceAccessor` (granted automatically to project +owners). + +## Debugging a failing build + +On the runner: + +```bash +sudo journalctl -u act_runner -f # runner daemon logs +sudo cat /var/log/runner-bootstrap.log # boot-time install log +sudo systemctl status act_runner # daemon state +docker ps # in-flight build containers +ls -la /opt/runner # config + .runner state file +``` + +For a workflow run the live logs are at +`https://git.neuralplatform.ai///actions`. + +## Rotating the registration token + +```bash +TOKEN=$(curl -s -X GET https://git.neuralplatform.ai/api/v1/admin/runners/registration-token \ + -H "Authorization: token $(cat ~/Secrets/api-keys/gitea-api-token)" | jq -r .token) +echo -n "$TOKEN" | gcloud secrets versions add gitea-runner-token \ + --project=neuron-785695 --data-file=- + +# Re-register (token is single-use). Either taint the VM: +terraform taint google_compute_instance.gitea_runner +terraform apply + +# ...or re-run register on the existing VM via IAP SSH: +gcloud compute ssh gitea-runner-1 --zone=us-central1-a --tunnel-through-iap -- "\ + sudo systemctl stop act_runner && \ + sudo -u runner /usr/local/bin/act_runner register --no-interactive \ + --instance https://git.neuralplatform.ai \ + --token $TOKEN \ + --name gcp-us-central1-runner-1 \ + --labels ubuntu-latest:host,ubuntu-22.04:host,ubuntu-24.04:host && \ + sudo systemctl start act_runner" +``` + +## Fallback: long-lived JSON key (only if WIF isn't usable) + +If Gitea's OIDC discovery endpoint isn't reachable from +`oauth2.googleapis.com` (e.g. CF Access blocks `.well-known/openid-configuration` +to anonymous fetchers), workflows can't use WIF. Fall back to the JSON +key flow: + +```bash +gcloud iam service-accounts keys create /tmp/ci-pusher.json \ + --iam-account=neuron-ci-pusher@neuron-785695.iam.gserviceaccount.com +tea secret set --repo neuron-technologies/neuron-web GCP_SA_KEY < /tmp/ci-pusher.json +rm /tmp/ci-pusher.json +``` + +Then in `deploy.yaml`, swap the `auth` step's `workload_identity_provider`/ +`service_account` inputs for `credentials_json: ${{ secrets.GCP_SA_KEY }}`. + +The WIF resources stay in Terraform either way — the fallback is purely +in the workflow. WIF remains the target. + +## Cost + +`e2-standard-4` in `us-central1` is roughly **$100/mo** sustained-use, +plus a few cents for the GCS bucket and the ephemeral IP. Not free. If +that's too much, downsize to `e2-standard-2` (~$50/mo) — most builds +fit, only the heaviest C++/Rust compiles need 16 GiB. diff --git a/servers/gcp/runners/startup.sh b/servers/gcp/runners/startup.sh new file mode 100644 index 0000000..8a4f525 --- /dev/null +++ b/servers/gcp/runners/startup.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# servers/gcp/runners/startup.sh +# +# Boot script for the Gitea Actions runner GCE VM. +# - Installs docker + git + jq + the Forgejo/Gitea act_runner binary. +# - Pulls the runner registration token from GCP Secret Manager +# (the VM's attached service account has secretmanager.secretAccessor on it). +# - Registers against https://git.neuralplatform.ai with labels matching +# the workflows that target ubuntu-latest hosts. +# - Starts act_runner under systemd so it auto-recovers on reboot. +# +# Logs land at /var/log/runner-bootstrap.log for debugging from IAP SSH. + +set -euxo pipefail +exec > >(tee /var/log/runner-bootstrap.log) 2>&1 + +apt-get update +apt-get install -y curl ca-certificates docker.io git jq + +# Make docker usable by the unprivileged runner user +systemctl enable --now docker + +useradd -m -s /bin/bash runner || true +usermod -aG docker runner + +# act_runner — pinned to a known-good release. Bump RUNNER_VERSION when +# upgrading. The project moved from gitea/act_runner to gitea/runner around +# the 0.6.x series; the binary inside the asset is still called act_runner. +# Latest releases at: https://gitea.com/gitea/runner/releases +RUNNER_VERSION="0.6.1" +curl -fsSL \ + "https://gitea.com/gitea/runner/releases/download/v${RUNNER_VERSION}/act_runner-${RUNNER_VERSION}-linux-amd64" \ + -o /usr/local/bin/act_runner +chmod +x /usr/local/bin/act_runner + +mkdir -p /opt/runner +chown -R runner:runner /opt/runner + +# Pull registration token. Fail fast if the secret is missing — the runner +# is useless without it, and a silent fallback would leave us debugging an +# unregistered VM later. +GITEA_RUNNER_TOKEN="$(gcloud secrets versions access latest \ + --secret=gitea-runner-token \ + --project=neuron-785695)" + +# Generate a baseline config so the daemon has a sane starting point. The +# register step writes the .runner state file into /opt/runner. +sudo -u runner /usr/local/bin/act_runner generate-config > /opt/runner/config.yaml +chown runner:runner /opt/runner/config.yaml + +cd /opt/runner +sudo -u runner /usr/local/bin/act_runner register \ + --no-interactive \ + --instance https://git.neuralplatform.ai \ + --token "${GITEA_RUNNER_TOKEN}" \ + --name "gcp-us-central1-runner-1" \ + --labels "ubuntu-latest,ubuntu-22.04,ubuntu-24.04" + +# Belt-and-braces: act_runner v0.6 has a habit of wrapping bare labels with +# the default `:docker://docker.gitea.com/runner-images: