vault: GCE Raft HA cluster replacing defunct Legion k3s deployment

Legion is offline. Vault needs a GCP-native home. This commit provisions a 3-node Vault HA cluster on GCE e2-small VMs (us-central1 a/b/c) backed by Raft consensus with GCP KMS auto-unseal already in place. - vault-nodes.tf: 3 VMs with separate pd-ssd data disks (prevent_destroy), IAP-only SSH firewall, Raft peer firewall (8201), global HTTPS LB for vault.neuralplatform.ai with Google-managed cert, HTTP health checks against /v1/sys/health - vault/startup.sh: idempotent boot script — installs Vault from HashiCorp APT repo, mounts/formats the data disk, writes Raft config with dynamic retry_join peers resolved via gcloud metadata, starts vault.service - vault-auth-gcp.tf: GCP Secret Manager secret for init script; outputs the service account emails needed for GCP auth roles - vault/configure-vault-auth.sh: post-init script to enable gcp auth method, write marketing-policy and soma-policy, bind roles to Cloud Run service account identities After apply: set vault_lb_ip output as A record for vault.neuralplatform.ai in Cloudflare, then run vault operator init on vault-node-1 to bootstrap.
2026-05-04 10:17:12 -05:00
parent 89564ccec7
commit 246368c132
4 changed files with 750 additions and 0 deletions
@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+# vault/configure-vault-auth.sh
+#
+# Configures Vault GCP IAM auth method and access policies for Cloud Run services.
+#
+# Run this ONCE after the Vault cluster is initialized and unsealed:
+#
+#   export VAULT_ADDR=https://vault.neuralplatform.ai
+#   export VAULT_TOKEN=<root-token-from-vault-operator-init>
+#   bash servers/gcp/vault/configure-vault-auth.sh
+#
+# What this script does:
+#   1. Enables the GCP auth method at auth/gcp
+#   2. Creates Vault policies for marketing and soma
+#   3. Creates GCP IAM auth roles bound to each service's Cloud Run SA
+#   4. (Optional) Creates the kv-v2 secret engines at secret/ if not present
+#
+# After running this script, Cloud Run services can authenticate to Vault using
+# their GCP service account identity — no long-lived tokens needed.
+#
+# Auth flow for Cloud Run:
+#   1. Service calls metadata server to get an OIDC token for its SA
+#   2. POST /v1/auth/gcp/login with role=<role> and jwt=<oidc-token>
+#   3. Vault validates the token against GCP IAM, returns a Vault token
+#   4. Service uses the Vault token to read secrets
+#
+# Vault CLI docs:
+#   https://developer.hashicorp.com/vault/docs/auth/gcp
+
+set -euo pipefail
+
+: "${VAULT_ADDR:?Set VAULT_ADDR to the Vault cluster address}"
+: "${VAULT_TOKEN:?Set VAULT_TOKEN to a root or admin token}"
+
+GCP_PROJECT="neuron-785695"
+
+# SA emails from terraform outputs (vault-auth-gcp.tf)
+MARKETING_SA="neuron-marketing-sa@${GCP_PROJECT}.iam.gserviceaccount.com"
+SOMA_SA="neuron-soma-sa@${GCP_PROJECT}.iam.gserviceaccount.com"
+
+echo "=== Vault GCP Auth Bootstrap ==="
+echo "VAULT_ADDR: ${VAULT_ADDR}"
+echo "GCP Project: ${GCP_PROJECT}"
+echo ""
+
+# ── Step 1: Enable kv-v2 secret engine ───────────────────────────────────────
+echo "--- Enabling kv-v2 secret engine at secret/ ---"
+if vault secrets list | grep -q "^secret/"; then
+  echo "secret/ already enabled — skipping"
+else
+  vault secrets enable -path=secret kv-v2
+  echo "Enabled secret/ (kv-v2)"
+fi
+
+# ── Step 2: Enable GCP auth method ────────────────────────────────────────────
+echo ""
+echo "--- Enabling GCP auth method ---"
+if vault auth list | grep -q "^gcp/"; then
+  echo "gcp/ auth already enabled — skipping"
+else
+  vault auth enable gcp
+  echo "Enabled gcp/ auth method"
+fi
+
+# Configure the auth method to trust the GCP project
+vault write auth/gcp/config \
+  project_id="${GCP_PROJECT}"
+
+echo "Configured auth/gcp for project ${GCP_PROJECT}"
+
+# ── Step 3: Write Vault policies ──────────────────────────────────────────────
+echo ""
+echo "--- Writing Vault policies ---"
+
+vault policy write marketing-policy - <<'POLICY'
+# marketing-policy: read access to marketing secrets
+path "secret/data/marketing/*" {
+  capabilities = ["read"]
+}
+path "secret/metadata/marketing/*" {
+  capabilities = ["list", "read"]
+}
+POLICY
+echo "Wrote marketing-policy"
+
+vault policy write soma-policy - <<'POLICY'
+# soma-policy: read access to soma secrets
+path "secret/data/soma/*" {
+  capabilities = ["read"]
+}
+path "secret/metadata/soma/*" {
+  capabilities = ["list", "read"]
+}
+POLICY
+echo "Wrote soma-policy"
+
+# ── Step 4: Create GCP IAM auth roles ────────────────────────────────────────
+echo ""
+echo "--- Creating GCP auth roles ---"
+
+# Marketing role — bound to the marketing Cloud Run SA
+# type=iam means auth is validated against GCP IAM (service account JWT)
+vault write auth/gcp/role/marketing \
+  type="iam" \
+  project_id="${GCP_PROJECT}" \
+  bound_service_accounts="${MARKETING_SA}" \
+  policies="marketing-policy" \
+  ttl="1h" \
+  max_ttl="24h"
+echo "Created role: marketing (bound to ${MARKETING_SA})"
+
+# Soma role — bound to the soma Cloud Run SA
+vault write auth/gcp/role/soma \
+  type="iam" \
+  project_id="${GCP_PROJECT}" \
+  bound_service_accounts="${SOMA_SA}" \
+  policies="soma-policy" \
+  ttl="1h" \
+  max_ttl="24h"
+echo "Created role: soma (bound to ${SOMA_SA})"
+
+# ── Step 5: Smoke test ────────────────────────────────────────────────────────
+echo ""
+echo "--- Smoke test ---"
+echo "Verifying policies exist:"
+vault policy list | grep -E "marketing-policy|soma-policy"
+
+echo ""
+echo "Verifying auth roles exist:"
+vault list auth/gcp/role
+
+echo ""
+echo "=== Done ==="
+echo ""
+echo "Vault GCP auth is configured. Cloud Run services can now authenticate using:"
+echo "  POST \${VAULT_ADDR}/v1/auth/gcp/login"
+echo "  body: { role: 'marketing', jwt: '<gcp-oidc-token>' }"
+echo ""
+echo "Next steps:"
+echo "  1. Write secrets:  vault kv put secret/marketing/... key=value"
+echo "  2. Write secrets:  vault kv put secret/soma/... key=value"
+echo "  3. Rotate the root token: vault token create -policy=... and revoke root"
+echo "  4. Store the recovery keys securely (used if KMS key needs re-wrapping)"
@@ -0,0 +1,202 @@
+#!/usr/bin/env bash
+# vault/startup.sh
+#
+# Boot script for Vault HA cluster nodes on GCE.
+# Idempotent — safe to re-run on VM restart.
+#
+# Required env vars (set by the GCE metadata startup-script):
+#   VAULT_NODE_ID  — e.g. "vault-node-1"
+#   VAULT_VERSION  — e.g. "1.19.2"
+#
+# After first boot:
+#   1. On vault-node-1: vault operator init
+#      → save root token and recovery keys in a secure location (Vault bootstrap
+#        requires them; the recovery keys are for KMS-sealed Vault)
+#   2. Nodes 2 and 3 auto-join via retry_join once initialized
+#
+# Ops access:
+#   gcloud compute ssh vault-node-1 --zone=us-central1-a --tunnel-through-iap
+
+set -euxo pipefail
+exec > >(tee /var/log/vault-bootstrap.log) 2>&1
+
+: "${VAULT_NODE_ID:?VAULT_NODE_ID must be set}"
+: "${VAULT_VERSION:?VAULT_VERSION must be set}"
+
+# ── Install Vault ─────────────────────────────────────────────────────────────
+
+apt-get update -y
+apt-get install -y curl ca-certificates gnupg lsb-release unzip
+
+# HashiCorp GPG key + repo
+curl -fsSL https://apt.releases.hashicorp.com/gpg \
+  | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+
+echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] \
+  https://apt.releases.hashicorp.com $(lsb_release -cs) main" \
+  > /etc/apt/sources.list.d/hashicorp.list
+
+apt-get update -y
+apt-get install -y vault="${VAULT_VERSION}-1"
+
+# Give Vault cap to lock memory (prevents secrets being swapped to disk)
+setcap cap_ipc_lock=+ep /usr/bin/vault
+
+# ── Mount and prepare the data disk ──────────────────────────────────────────
+# The data disk is attached as /dev/disk/by-id/google-vault-data
+# First boot: format it. Subsequent boots: just mount it.
+
+DATA_DEVICE="/dev/disk/by-id/google-vault-data"
+DATA_MOUNT="/opt/vault/data"
+mkdir -p "${DATA_MOUNT}"
+
+# Only format if not already formatted (idempotent)
+if ! blkid "${DATA_DEVICE}" | grep -q ext4; then
+  mkfs.ext4 -F "${DATA_DEVICE}"
+fi
+
+# Mount (skip if already mounted)
+if ! mountpoint -q "${DATA_MOUNT}"; then
+  mount "${DATA_DEVICE}" "${DATA_MOUNT}"
+fi
+
+# Persist in fstab
+DEVICE_UUID="$(blkid -s UUID -o value "${DATA_DEVICE}")"
+if ! grep -q "${DEVICE_UUID}" /etc/fstab; then
+  echo "UUID=${DEVICE_UUID} ${DATA_MOUNT} ext4 defaults,noatime 0 2" >> /etc/fstab
+fi
+
+# ── Derive internal IPs for retry_join ───────────────────────────────────────
+# GCE metadata server gives us the IPs of the other nodes by instance name.
+# We query the Compute API via the attached SA (ADC — no key file needed).
+# For the initial single-node bootstrap, retry_join failures are harmless
+# (Raft simply starts as a single-node cluster until peers join).
+
+fetch_internal_ip() {
+  local instance_name="$1"
+  local zone="$2"
+  gcloud compute instances describe "${instance_name}" \
+    --zone="${zone}" \
+    --format="get(networkInterfaces[0].networkIP)" \
+    2>/dev/null || echo ""
+}
+
+# Map of all Vault nodes: name → zone
+declare -A NODE_ZONES
+NODE_ZONES["vault-node-1"]="us-central1-a"
+NODE_ZONES["vault-node-2"]="us-central1-b"
+NODE_ZONES["vault-node-3"]="us-central1-c"
+
+# Build retry_join stanzas for all nodes other than self
+RETRY_JOIN_STANZAS=""
+for node in "vault-node-1" "vault-node-2" "vault-node-3"; do
+  if [[ "${node}" == "${VAULT_NODE_ID}" ]]; then
+    continue
+  fi
+  zone="${NODE_ZONES[$node]}"
+  ip="$(fetch_internal_ip "${node}" "${zone}")"
+  if [[ -n "${ip}" ]]; then
+    RETRY_JOIN_STANZAS="${RETRY_JOIN_STANZAS}
+  retry_join {
+    leader_api_addr = \"http://${ip}:8200\"
+  }"
+  fi
+done
+
+# ── Get the internal IP of this node ─────────────────────────────────────────
+MY_INTERNAL_IP="$(curl -sf -H "Metadata-Flavor: Google" \
+  "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip")"
+
+# ── Write Vault config ────────────────────────────────────────────────────────
+
+mkdir -p /etc/vault.d
+cat > /etc/vault.d/vault.hcl <<EOF
+ui = true
+
+listener "tcp" {
+  address       = "0.0.0.0:8200"
+  cluster_address = "0.0.0.0:8201"
+  # TLS terminated at GCP LB — plain HTTP internally is fine
+  tls_disable   = true
+}
+
+storage "raft" {
+  path    = "/opt/vault/data"
+  node_id = "${VAULT_NODE_ID}"
+${RETRY_JOIN_STANZAS}
+}
+
+seal "gcpckms" {
+  project    = "neuron-785695"
+  region     = "global"
+  key_ring   = "vault"
+  crypto_key = "vault-unseal"
+}
+
+api_addr     = "http://${MY_INTERNAL_IP}:8200"
+cluster_addr = "http://${MY_INTERNAL_IP}:8201"
+
+# Telemetry for Cloud Monitoring
+telemetry {
+  prometheus_retention_time = "30s"
+  disable_hostname          = false
+}
+EOF
+
+chmod 640 /etc/vault.d/vault.hcl
+
+# ── Create vault user and fix permissions ─────────────────────────────────────
+
+id vault &>/dev/null || useradd --system --home /etc/vault.d --shell /bin/false vault
+chown -R vault:vault /etc/vault.d /opt/vault/data
+
+# ── Systemd unit ──────────────────────────────────────────────────────────────
+
+cat > /etc/systemd/system/vault.service <<'EOF'
+[Unit]
+Description=HashiCorp Vault — a tool for managing secrets
+Documentation=https://developer.hashicorp.com/vault/docs
+Requires=network-online.target
+After=network-online.target
+ConditionFileNotEmpty=/etc/vault.d/vault.hcl
+StartLimitIntervalSec=60
+StartLimitBurst=3
+
+[Service]
+Type=notify
+User=vault
+Group=vault
+ProtectSystem=full
+ProtectHome=read-only
+PrivateTmp=yes
+PrivateDevices=yes
+SecureBits=keep-caps
+AmbientCapabilities=CAP_IPC_LOCK
+CapabilityBoundingSet=CAP_SYSLOG CAP_IPC_LOCK
+NoNewPrivileges=yes
+ExecStart=/usr/bin/vault server -config=/etc/vault.d/vault.hcl
+ExecReload=/bin/kill --signal HUP $MAINPID
+KillMode=process
+KillSignal=SIGINT
+Restart=on-failure
+RestartSec=5
+TimeoutStopSec=30
+LimitNOFILE=65536
+LimitMEMLOCK=infinity
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload
+systemctl enable vault
+
+# Start or restart (restart is idempotent — re-reads config on VM metadata updates)
+if systemctl is-active vault &>/dev/null; then
+  systemctl restart vault
+else
+  systemctl start vault
+fi
+
+echo "Vault ${VAULT_NODE_ID} started successfully"
+echo "Run 'vault operator init' on vault-node-1 to bootstrap the cluster"