vault: GCE Raft HA cluster replacing defunct Legion k3s deployment

Legion is offline. Vault needs a GCP-native home. This commit provisions a 3-node Vault HA cluster on GCE e2-small VMs (us-central1 a/b/c) backed by Raft consensus with GCP KMS auto-unseal already in place. - vault-nodes.tf: 3 VMs with separate pd-ssd data disks (prevent_destroy), IAP-only SSH firewall, Raft peer firewall (8201), global HTTPS LB for vault.neuralplatform.ai with Google-managed cert, HTTP health checks against /v1/sys/health - vault/startup.sh: idempotent boot script — installs Vault from HashiCorp APT repo, mounts/formats the data disk, writes Raft config with dynamic retry_join peers resolved via gcloud metadata, starts vault.service - vault-auth-gcp.tf: GCP Secret Manager secret for init script; outputs the service account emails needed for GCP auth roles - vault/configure-vault-auth.sh: post-init script to enable gcp auth method, write marketing-policy and soma-policy, bind roles to Cloud Run service account identities After apply: set vault_lb_ip output as A record for vault.neuralplatform.ai in Cloudflare, then run vault operator init on vault-node-1 to bootstrap.
2026-05-04 10:17:12 -05:00
parent 89564ccec7
commit 246368c132
4 changed files with 750 additions and 0 deletions
@@ -0,0 +1,48 @@
+# ── Vault GCP IAM Auth Method — GCP IAM for Cloud Run services ───────────────
+#
+# This file defines the GCP Secret Manager secret that holds the Vault GCP
+# auth configuration script, and the Vault policies for each service identity.
+#
+# IMPORTANT: The Vault Terraform provider is NOT configured in this workspace.
+# The actual Vault auth method configuration is performed by running:
+#
+#   servers/gcp/vault/configure-vault-auth.sh
+#
+# That script uses the `vault` CLI and must be run AFTER:
+#   1. The vault-nodes are up and initialized (vault operator init on node-1)
+#   2. VAULT_ADDR and VAULT_TOKEN are set in the environment
+#
+# Why no Vault Terraform provider here:
+#   - Adding the provider requires a working Vault cluster at `terraform init`
+#   - The cluster doesn't exist yet when this plan is first applied
+#   - The provider would encode the root token in TF state (undesirable)
+#   - A shell script run once post-bootstrap is simpler and more auditable
+#
+# If you want to add the Vault provider in a future iteration, the script
+# below maps directly to `vault_auth_backend`, `vault_policy`, and
+# `vault_gcp_auth_backend_role` resources.
+
+# ── GCP Secret Manager secret for Vault auth init script ─────────────────────
+# Store the post-init script here so it can be fetched from any node via
+# gcloud secrets versions access — no file transfer needed.
+
+resource "google_secret_manager_secret" "vault_auth_init_script" {
+  secret_id = "vault-auth-init-script"
+  project   = var.project_id
+
+  replication {
+    auto {}
+  }
+}
+
+# ── Service account SA email data sources (for use in the configure script) ──
+
+# The script uses these SA emails — surfaced here as outputs for convenience
+
+output "vault_auth_sa_emails" {
+  description = "Service account emails for Vault GCP auth roles — used in configure-vault-auth.sh"
+  value = {
+    marketing = google_service_account.marketing.email
+    soma      = google_service_account.soma.email
+  }
+}
@@ -0,0 +1,357 @@
+# ── Vault HA Cluster — GCE-based Raft ────────────────────────────────────────
+#
+# Three GCE e2-small VMs across us-central1-{a,b,c} running HashiCorp Vault
+# in Raft HA mode. Auto-unseal via the existing GCP KMS key (vault-kms.tf).
+#
+# Architecture:
+#   - vault-node-1 (us-central1-a) — bootstrapped first, others join via retry_join
+#   - vault-node-2 (us-central1-b)
+#   - vault-node-3 (us-central1-c)
+#   - Each VM gets the vault-node SA attached for KMS auto-unseal + ADC
+#   - Internal traffic: VMs talk Raft over port 8201 on GCP internal IPs
+#   - External access: HTTPS regional LB → port 8200
+#     (vault.neuralplatform.ai Cloudflare DNS → GCP LB IP)
+#
+# Ops SSH:
+#   gcloud compute ssh vault-node-1 --zone=us-central1-a --tunnel-through-iap
+#
+# After first boot, initialize Vault on node-1:
+#   vault operator init   (save the root token + recovery keys securely)
+# Nodes 2 and 3 auto-join via retry_join once they boot.
+
+locals {
+  vault_version = "1.19.2"
+
+  vault_nodes = {
+    "vault-node-1" = { zone = "us-central1-a", node_id = "vault-node-1" }
+    "vault-node-2" = { zone = "us-central1-b", node_id = "vault-node-2" }
+    "vault-node-3" = { zone = "us-central1-c", node_id = "vault-node-3" }
+  }
+}
+
+# ── Service Account for Vault nodes ──────────────────────────────────────────
+# Reuses the vault-unseal SA from vault-kms.tf for KMS access.
+# Additional roles: logging + metrics from ops agent.
+
+resource "google_project_iam_member" "vault_node_log_writer" {
+  project = var.project_id
+  role    = "roles/logging.logWriter"
+  member  = "serviceAccount:${google_service_account.vault_unseal.email}"
+}
+
+resource "google_project_iam_member" "vault_node_metric_writer" {
+  project = var.project_id
+  role    = "roles/monitoring.metricWriter"
+  member  = "serviceAccount:${google_service_account.vault_unseal.email}"
+}
+
+# Allow Vault nodes to read their own instance metadata (needed for GCP auth method later)
+resource "google_project_iam_member" "vault_node_compute_viewer" {
+  project = var.project_id
+  role    = "roles/compute.viewer"
+  member  = "serviceAccount:${google_service_account.vault_unseal.email}"
+}
+
+# ── Startup script staged in GCS ─────────────────────────────────────────────
+# Stored in the existing runner-assets bucket (reuse infrastructure).
+# The script installs Vault, writes the config, and starts the systemd unit.
+
+resource "google_storage_bucket_object" "vault_startup" {
+  name   = "vault/startup.sh"
+  bucket = google_storage_bucket.runner_assets.name
+  source = "${path.module}/vault/startup.sh"
+
+  metadata = {
+    sha256 = filesha256("${path.module}/vault/startup.sh")
+  }
+}
+
+resource "google_storage_bucket_iam_member" "vault_node_bucket_read" {
+  bucket = google_storage_bucket.runner_assets.name
+  role   = "roles/storage.objectViewer"
+  member = "serviceAccount:${google_service_account.vault_unseal.email}"
+}
+
+# ── Vault node VMs ────────────────────────────────────────────────────────────
+
+resource "google_compute_instance" "vault_node" {
+  for_each     = local.vault_nodes
+  name         = each.key
+  machine_type = "e2-small"
+  zone         = each.value.zone
+  project      = var.project_id
+
+  tags = ["vault-node", "allow-iap-ssh"]
+
+  boot_disk {
+    initialize_params {
+      image = "projects/debian-cloud/global/images/family/debian-12"
+      size  = 20
+      type  = "pd-balanced"
+    }
+  }
+
+  # Separate persistent disk for Raft data — survives VM recreation
+  attached_disk {
+    source      = google_compute_disk.vault_data[each.key].self_link
+    device_name = "vault-data"
+    mode        = "READ_WRITE"
+  }
+
+  network_interface {
+    network = "default"
+    # No external IP — accessed via IAP SSH and the internal LB
+    # Vault API is published externally via the regional LB below
+    access_config {}
+  }
+
+  service_account {
+    email  = google_service_account.vault_unseal.email
+    scopes = ["cloud-platform"]
+  }
+
+  metadata = {
+    # Pull and execute the real startup script from GCS
+    startup-script = <<-EOT
+      #!/usr/bin/env bash
+      set -euxo pipefail
+      apt-get update -y
+      apt-get install -y curl ca-certificates apt-transport-https gnupg google-cloud-cli
+      gsutil cat gs://${google_storage_bucket.runner_assets.name}/${google_storage_bucket_object.vault_startup.name} \
+        > /tmp/vault-startup.sh
+      chmod +x /tmp/vault-startup.sh
+      VAULT_NODE_ID="${each.value.node_id}" \
+      VAULT_VERSION="${local.vault_version}" \
+        /tmp/vault-startup.sh
+    EOT
+
+    enable-oslogin = "TRUE"
+  }
+
+  allow_stopping_for_update = true
+
+  depends_on = [
+    google_compute_disk.vault_data,
+    google_storage_bucket_object.vault_startup,
+    google_storage_bucket_iam_member.vault_node_bucket_read,
+  ]
+}
+
+# ── Persistent data disks ─────────────────────────────────────────────────────
+# 10 GiB per node. Kept separate from the boot disk so Raft data
+# survives a full VM deletion and recreation.
+
+resource "google_compute_disk" "vault_data" {
+  for_each = local.vault_nodes
+  name     = "${each.key}-data"
+  zone     = each.value.zone
+  project  = var.project_id
+  type     = "pd-ssd"
+  size     = 10
+
+  labels = {
+    managed-by = "terraform"
+    service    = "vault"
+    node       = each.key
+  }
+
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
+# ── Firewall rules ────────────────────────────────────────────────────────────
+
+# IAP SSH — ops access without a public SSH port
+resource "google_compute_firewall" "vault_iap_ssh" {
+  name      = "vault-iap-ssh"
+  network   = "default"
+  project   = var.project_id
+  direction = "INGRESS"
+  priority  = 1000
+
+  allow {
+    protocol = "tcp"
+    ports    = ["22"]
+  }
+
+  source_ranges = ["35.235.240.0/20"] # GCP IAP CIDR
+  target_tags   = ["vault-node"]
+}
+
+# Raft cluster traffic — node-to-node port 8201 (internal only)
+resource "google_compute_firewall" "vault_raft" {
+  name      = "vault-raft-internal"
+  network   = "default"
+  project   = var.project_id
+  direction = "INGRESS"
+  priority  = 1000
+
+  allow {
+    protocol = "tcp"
+    ports    = ["8201"]
+  }
+
+  source_tags = ["vault-node"]
+  target_tags = ["vault-node"]
+}
+
+# Vault API — allow from the GCP health check ranges and the LB
+resource "google_compute_firewall" "vault_api" {
+  name      = "vault-api-from-lb"
+  network   = "default"
+  project   = var.project_id
+  direction = "INGRESS"
+  priority  = 1000
+
+  allow {
+    protocol = "tcp"
+    ports    = ["8200"]
+  }
+
+  # GCP health check ranges (130.211.0.0/22, 35.191.0.0/16)
+  # and RFC1918 for any internal service access
+  source_ranges = ["130.211.0.0/22", "35.191.0.0/16", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]
+  target_tags   = ["vault-node"]
+}
+
+# ── Instance Groups (one unmanaged group per zone for the LB) ─────────────────
+
+resource "google_compute_instance_group" "vault" {
+  for_each  = local.vault_nodes
+  name      = "vault-${each.key}"
+  zone      = each.value.zone
+  project   = var.project_id
+  instances = [google_compute_instance.vault_node[each.key].self_link]
+
+  named_port {
+    name = "vault-api"
+    port = 8200
+  }
+}
+
+# ── Regional HTTPS LB for vault.neuralplatform.ai ─────────────────────────────
+# We use a global external HTTPS LB (same scheme as the prod marketing LB)
+# so we can attach a Google-managed cert for vault.neuralplatform.ai.
+#
+# TLS is terminated at the LB. Vault listens on plain 8200 internally.
+# The Cloudflare DNS A record for vault.neuralplatform.ai (neuralplatform zone)
+# must point to vault_lb_ip output below — add it in Cloudflare dashboard
+# or in the legion Terraform if you bring that zone under TF management.
+
+resource "google_compute_global_address" "vault" {
+  name    = "vault-ip"
+  project = var.project_id
+}
+
+resource "google_compute_managed_ssl_certificate" "vault" {
+  name    = "vault-cert"
+  project = var.project_id
+
+  managed {
+    domains = ["vault.neuralplatform.ai"]
+  }
+}
+
+resource "google_compute_health_check" "vault" {
+  name    = "vault-health"
+  project = var.project_id
+
+  # Vault listens on plain HTTP (tls_disable = true) — TLS is terminated at the LB
+  http_health_check {
+    port         = 8200
+    request_path = "/v1/sys/health?standbyok=true&sealedok=true&uninitcode=200"
+  }
+
+  check_interval_sec  = 15
+  timeout_sec         = 5
+  healthy_threshold   = 2
+  unhealthy_threshold = 3
+}
+
+resource "google_compute_backend_service" "vault" {
+  name                  = "vault-backend"
+  project               = var.project_id
+  load_balancing_scheme = "EXTERNAL_MANAGED"
+  protocol              = "HTTP"   # Vault serves plain HTTP; TLS terminates at the LB
+  timeout_sec           = 30
+  port_name             = "vault-api"
+
+  health_checks = [google_compute_health_check.vault.self_link]
+
+  dynamic "backend" {
+    for_each = local.vault_nodes
+    content {
+      group           = google_compute_instance_group.vault[backend.key].self_link
+      balancing_mode  = "UTILIZATION"
+      max_utilization = 0.8
+    }
+  }
+
+  log_config {
+    enable      = true
+    sample_rate = 1.0
+  }
+}
+
+resource "google_compute_url_map" "vault" {
+  name            = "vault-urlmap"
+  project         = var.project_id
+  default_service = google_compute_backend_service.vault.self_link
+}
+
+resource "google_compute_target_https_proxy" "vault" {
+  name             = "vault-https-proxy"
+  project          = var.project_id
+  url_map          = google_compute_url_map.vault.self_link
+  ssl_certificates = [google_compute_managed_ssl_certificate.vault.self_link]
+}
+
+resource "google_compute_target_http_proxy" "vault_redirect" {
+  name    = "vault-http-proxy"
+  project = var.project_id
+  url_map = google_compute_url_map.vault_redirect.self_link
+}
+
+resource "google_compute_url_map" "vault_redirect" {
+  name    = "vault-urlmap-http-redirect"
+  project = var.project_id
+
+  default_url_redirect {
+    https_redirect         = true
+    redirect_response_code = "MOVED_PERMANENTLY_DEFAULT"
+    strip_query            = false
+  }
+}
+
+resource "google_compute_global_forwarding_rule" "vault_https" {
+  name                  = "vault-fwd-https"
+  project               = var.project_id
+  target                = google_compute_target_https_proxy.vault.self_link
+  ip_address            = google_compute_global_address.vault.address
+  port_range            = "443"
+  load_balancing_scheme = "EXTERNAL_MANAGED"
+}
+
+resource "google_compute_global_forwarding_rule" "vault_http" {
+  name                  = "vault-fwd-http"
+  project               = var.project_id
+  target                = google_compute_target_http_proxy.vault_redirect.self_link
+  ip_address            = google_compute_global_address.vault.address
+  port_range            = "80"
+  load_balancing_scheme = "EXTERNAL_MANAGED"
+}
+
+# ── Outputs ───────────────────────────────────────────────────────────────────
+
+output "vault_lb_ip" {
+  description = "Global IP for vault.neuralplatform.ai — set as DNS A record in Cloudflare (neuralplatform.ai zone)"
+  value       = google_compute_global_address.vault.address
+}
+
+output "vault_node_zones" {
+  description = "Zone placement of each Vault node"
+  value = {
+    for k, v in local.vault_nodes : k => v.zone
+  }
+}
@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+# vault/configure-vault-auth.sh
+#
+# Configures Vault GCP IAM auth method and access policies for Cloud Run services.
+#
+# Run this ONCE after the Vault cluster is initialized and unsealed:
+#
+#   export VAULT_ADDR=https://vault.neuralplatform.ai
+#   export VAULT_TOKEN=<root-token-from-vault-operator-init>
+#   bash servers/gcp/vault/configure-vault-auth.sh
+#
+# What this script does:
+#   1. Enables the GCP auth method at auth/gcp
+#   2. Creates Vault policies for marketing and soma
+#   3. Creates GCP IAM auth roles bound to each service's Cloud Run SA
+#   4. (Optional) Creates the kv-v2 secret engines at secret/ if not present
+#
+# After running this script, Cloud Run services can authenticate to Vault using
+# their GCP service account identity — no long-lived tokens needed.
+#
+# Auth flow for Cloud Run:
+#   1. Service calls metadata server to get an OIDC token for its SA
+#   2. POST /v1/auth/gcp/login with role=<role> and jwt=<oidc-token>
+#   3. Vault validates the token against GCP IAM, returns a Vault token
+#   4. Service uses the Vault token to read secrets
+#
+# Vault CLI docs:
+#   https://developer.hashicorp.com/vault/docs/auth/gcp
+
+set -euo pipefail
+
+: "${VAULT_ADDR:?Set VAULT_ADDR to the Vault cluster address}"
+: "${VAULT_TOKEN:?Set VAULT_TOKEN to a root or admin token}"
+
+GCP_PROJECT="neuron-785695"
+
+# SA emails from terraform outputs (vault-auth-gcp.tf)
+MARKETING_SA="neuron-marketing-sa@${GCP_PROJECT}.iam.gserviceaccount.com"
+SOMA_SA="neuron-soma-sa@${GCP_PROJECT}.iam.gserviceaccount.com"
+
+echo "=== Vault GCP Auth Bootstrap ==="
+echo "VAULT_ADDR: ${VAULT_ADDR}"
+echo "GCP Project: ${GCP_PROJECT}"
+echo ""
+
+# ── Step 1: Enable kv-v2 secret engine ───────────────────────────────────────
+echo "--- Enabling kv-v2 secret engine at secret/ ---"
+if vault secrets list | grep -q "^secret/"; then
+  echo "secret/ already enabled — skipping"
+else
+  vault secrets enable -path=secret kv-v2
+  echo "Enabled secret/ (kv-v2)"
+fi
+
+# ── Step 2: Enable GCP auth method ────────────────────────────────────────────
+echo ""
+echo "--- Enabling GCP auth method ---"
+if vault auth list | grep -q "^gcp/"; then
+  echo "gcp/ auth already enabled — skipping"
+else
+  vault auth enable gcp
+  echo "Enabled gcp/ auth method"
+fi
+
+# Configure the auth method to trust the GCP project
+vault write auth/gcp/config \
+  project_id="${GCP_PROJECT}"
+
+echo "Configured auth/gcp for project ${GCP_PROJECT}"
+
+# ── Step 3: Write Vault policies ──────────────────────────────────────────────
+echo ""
+echo "--- Writing Vault policies ---"
+
+vault policy write marketing-policy - <<'POLICY'
+# marketing-policy: read access to marketing secrets
+path "secret/data/marketing/*" {
+  capabilities = ["read"]
+}
+path "secret/metadata/marketing/*" {
+  capabilities = ["list", "read"]
+}
+POLICY
+echo "Wrote marketing-policy"
+
+vault policy write soma-policy - <<'POLICY'
+# soma-policy: read access to soma secrets
+path "secret/data/soma/*" {
+  capabilities = ["read"]
+}
+path "secret/metadata/soma/*" {
+  capabilities = ["list", "read"]
+}
+POLICY
+echo "Wrote soma-policy"
+
+# ── Step 4: Create GCP IAM auth roles ────────────────────────────────────────
+echo ""
+echo "--- Creating GCP auth roles ---"
+
+# Marketing role — bound to the marketing Cloud Run SA
+# type=iam means auth is validated against GCP IAM (service account JWT)
+vault write auth/gcp/role/marketing \
+  type="iam" \
+  project_id="${GCP_PROJECT}" \
+  bound_service_accounts="${MARKETING_SA}" \
+  policies="marketing-policy" \
+  ttl="1h" \
+  max_ttl="24h"
+echo "Created role: marketing (bound to ${MARKETING_SA})"
+
+# Soma role — bound to the soma Cloud Run SA
+vault write auth/gcp/role/soma \
+  type="iam" \
+  project_id="${GCP_PROJECT}" \
+  bound_service_accounts="${SOMA_SA}" \
+  policies="soma-policy" \
+  ttl="1h" \
+  max_ttl="24h"
+echo "Created role: soma (bound to ${SOMA_SA})"
+
+# ── Step 5: Smoke test ────────────────────────────────────────────────────────
+echo ""
+echo "--- Smoke test ---"
+echo "Verifying policies exist:"
+vault policy list | grep -E "marketing-policy|soma-policy"
+
+echo ""
+echo "Verifying auth roles exist:"
+vault list auth/gcp/role
+
+echo ""
+echo "=== Done ==="
+echo ""
+echo "Vault GCP auth is configured. Cloud Run services can now authenticate using:"
+echo "  POST \${VAULT_ADDR}/v1/auth/gcp/login"
+echo "  body: { role: 'marketing', jwt: '<gcp-oidc-token>' }"
+echo ""
+echo "Next steps:"
+echo "  1. Write secrets:  vault kv put secret/marketing/... key=value"
+echo "  2. Write secrets:  vault kv put secret/soma/... key=value"
+echo "  3. Rotate the root token: vault token create -policy=... and revoke root"
+echo "  4. Store the recovery keys securely (used if KMS key needs re-wrapping)"
@@ -0,0 +1,202 @@
+#!/usr/bin/env bash
+# vault/startup.sh
+#
+# Boot script for Vault HA cluster nodes on GCE.
+# Idempotent — safe to re-run on VM restart.
+#
+# Required env vars (set by the GCE metadata startup-script):
+#   VAULT_NODE_ID  — e.g. "vault-node-1"
+#   VAULT_VERSION  — e.g. "1.19.2"
+#
+# After first boot:
+#   1. On vault-node-1: vault operator init
+#      → save root token and recovery keys in a secure location (Vault bootstrap
+#        requires them; the recovery keys are for KMS-sealed Vault)
+#   2. Nodes 2 and 3 auto-join via retry_join once initialized
+#
+# Ops access:
+#   gcloud compute ssh vault-node-1 --zone=us-central1-a --tunnel-through-iap
+
+set -euxo pipefail
+exec > >(tee /var/log/vault-bootstrap.log) 2>&1
+
+: "${VAULT_NODE_ID:?VAULT_NODE_ID must be set}"
+: "${VAULT_VERSION:?VAULT_VERSION must be set}"
+
+# ── Install Vault ─────────────────────────────────────────────────────────────
+
+apt-get update -y
+apt-get install -y curl ca-certificates gnupg lsb-release unzip
+
+# HashiCorp GPG key + repo
+curl -fsSL https://apt.releases.hashicorp.com/gpg \
+  | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+
+echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] \
+  https://apt.releases.hashicorp.com $(lsb_release -cs) main" \
+  > /etc/apt/sources.list.d/hashicorp.list
+
+apt-get update -y
+apt-get install -y vault="${VAULT_VERSION}-1"
+
+# Give Vault cap to lock memory (prevents secrets being swapped to disk)
+setcap cap_ipc_lock=+ep /usr/bin/vault
+
+# ── Mount and prepare the data disk ──────────────────────────────────────────
+# The data disk is attached as /dev/disk/by-id/google-vault-data
+# First boot: format it. Subsequent boots: just mount it.
+
+DATA_DEVICE="/dev/disk/by-id/google-vault-data"
+DATA_MOUNT="/opt/vault/data"
+mkdir -p "${DATA_MOUNT}"
+
+# Only format if not already formatted (idempotent)
+if ! blkid "${DATA_DEVICE}" | grep -q ext4; then
+  mkfs.ext4 -F "${DATA_DEVICE}"
+fi
+
+# Mount (skip if already mounted)
+if ! mountpoint -q "${DATA_MOUNT}"; then
+  mount "${DATA_DEVICE}" "${DATA_MOUNT}"
+fi
+
+# Persist in fstab
+DEVICE_UUID="$(blkid -s UUID -o value "${DATA_DEVICE}")"
+if ! grep -q "${DEVICE_UUID}" /etc/fstab; then
+  echo "UUID=${DEVICE_UUID} ${DATA_MOUNT} ext4 defaults,noatime 0 2" >> /etc/fstab
+fi
+
+# ── Derive internal IPs for retry_join ───────────────────────────────────────
+# GCE metadata server gives us the IPs of the other nodes by instance name.
+# We query the Compute API via the attached SA (ADC — no key file needed).
+# For the initial single-node bootstrap, retry_join failures are harmless
+# (Raft simply starts as a single-node cluster until peers join).
+
+fetch_internal_ip() {
+  local instance_name="$1"
+  local zone="$2"
+  gcloud compute instances describe "${instance_name}" \
+    --zone="${zone}" \
+    --format="get(networkInterfaces[0].networkIP)" \
+    2>/dev/null || echo ""
+}
+
+# Map of all Vault nodes: name → zone
+declare -A NODE_ZONES
+NODE_ZONES["vault-node-1"]="us-central1-a"
+NODE_ZONES["vault-node-2"]="us-central1-b"
+NODE_ZONES["vault-node-3"]="us-central1-c"
+
+# Build retry_join stanzas for all nodes other than self
+RETRY_JOIN_STANZAS=""
+for node in "vault-node-1" "vault-node-2" "vault-node-3"; do
+  if [[ "${node}" == "${VAULT_NODE_ID}" ]]; then
+    continue
+  fi
+  zone="${NODE_ZONES[$node]}"
+  ip="$(fetch_internal_ip "${node}" "${zone}")"
+  if [[ -n "${ip}" ]]; then
+    RETRY_JOIN_STANZAS="${RETRY_JOIN_STANZAS}
+  retry_join {
+    leader_api_addr = \"http://${ip}:8200\"
+  }"
+  fi
+done
+
+# ── Get the internal IP of this node ─────────────────────────────────────────
+MY_INTERNAL_IP="$(curl -sf -H "Metadata-Flavor: Google" \
+  "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip")"
+
+# ── Write Vault config ────────────────────────────────────────────────────────
+
+mkdir -p /etc/vault.d
+cat > /etc/vault.d/vault.hcl <<EOF
+ui = true
+
+listener "tcp" {
+  address       = "0.0.0.0:8200"
+  cluster_address = "0.0.0.0:8201"
+  # TLS terminated at GCP LB — plain HTTP internally is fine
+  tls_disable   = true
+}
+
+storage "raft" {
+  path    = "/opt/vault/data"
+  node_id = "${VAULT_NODE_ID}"
+${RETRY_JOIN_STANZAS}
+}
+
+seal "gcpckms" {
+  project    = "neuron-785695"
+  region     = "global"
+  key_ring   = "vault"
+  crypto_key = "vault-unseal"
+}
+
+api_addr     = "http://${MY_INTERNAL_IP}:8200"
+cluster_addr = "http://${MY_INTERNAL_IP}:8201"
+
+# Telemetry for Cloud Monitoring
+telemetry {
+  prometheus_retention_time = "30s"
+  disable_hostname          = false
+}
+EOF
+
+chmod 640 /etc/vault.d/vault.hcl
+
+# ── Create vault user and fix permissions ─────────────────────────────────────
+
+id vault &>/dev/null || useradd --system --home /etc/vault.d --shell /bin/false vault
+chown -R vault:vault /etc/vault.d /opt/vault/data
+
+# ── Systemd unit ──────────────────────────────────────────────────────────────
+
+cat > /etc/systemd/system/vault.service <<'EOF'
+[Unit]
+Description=HashiCorp Vault — a tool for managing secrets
+Documentation=https://developer.hashicorp.com/vault/docs
+Requires=network-online.target
+After=network-online.target
+ConditionFileNotEmpty=/etc/vault.d/vault.hcl
+StartLimitIntervalSec=60
+StartLimitBurst=3
+
+[Service]
+Type=notify
+User=vault
+Group=vault
+ProtectSystem=full
+ProtectHome=read-only
+PrivateTmp=yes
+PrivateDevices=yes
+SecureBits=keep-caps
+AmbientCapabilities=CAP_IPC_LOCK
+CapabilityBoundingSet=CAP_SYSLOG CAP_IPC_LOCK
+NoNewPrivileges=yes
+ExecStart=/usr/bin/vault server -config=/etc/vault.d/vault.hcl
+ExecReload=/bin/kill --signal HUP $MAINPID
+KillMode=process
+KillSignal=SIGINT
+Restart=on-failure
+RestartSec=5
+TimeoutStopSec=30
+LimitNOFILE=65536
+LimitMEMLOCK=infinity
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload
+systemctl enable vault
+
+# Start or restart (restart is idempotent — re-reads config on VM metadata updates)
+if systemctl is-active vault &>/dev/null; then
+  systemctl restart vault
+else
+  systemctl start vault
+fi
+
+echo "Vault ${VAULT_NODE_ID} started successfully"
+echo "Run 'vault operator init' on vault-node-1 to bootstrap the cluster"