#!/usr/bin/env bash # vault/startup.sh # # Boot script for Vault HA cluster nodes on GCE. # Idempotent — safe to re-run on VM restart. # # Required env vars (set by the GCE metadata startup-script): # VAULT_NODE_ID — e.g. "vault-node-1" # VAULT_VERSION — e.g. "1.19.2" # # After first boot: # 1. On vault-node-1: vault operator init # → save root token and recovery keys in a secure location (Vault bootstrap # requires them; the recovery keys are for KMS-sealed Vault) # 2. Nodes 2 and 3 auto-join via retry_join once initialized # # Ops access: # gcloud compute ssh vault-node-1 --zone=us-central1-a --tunnel-through-iap set -euxo pipefail exec > >(tee /var/log/vault-bootstrap.log) 2>&1 : "${VAULT_NODE_ID:?VAULT_NODE_ID must be set}" : "${VAULT_VERSION:?VAULT_VERSION must be set}" # ── Install Vault ───────────────────────────────────────────────────────────── apt-get update -y apt-get install -y curl ca-certificates gnupg lsb-release unzip # HashiCorp GPG key + repo curl -fsSL https://apt.releases.hashicorp.com/gpg \ | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] \ https://apt.releases.hashicorp.com $(lsb_release -cs) main" \ > /etc/apt/sources.list.d/hashicorp.list apt-get update -y apt-get install -y vault="${VAULT_VERSION}-1" # Give Vault cap to lock memory (prevents secrets being swapped to disk) setcap cap_ipc_lock=+ep /usr/bin/vault # ── Mount and prepare the data disk ────────────────────────────────────────── # The data disk is attached as /dev/disk/by-id/google-vault-data # First boot: format it. Subsequent boots: just mount it. DATA_DEVICE="/dev/disk/by-id/google-vault-data" DATA_MOUNT="/opt/vault/data" mkdir -p "${DATA_MOUNT}" # Only format if not already formatted (idempotent) if ! blkid "${DATA_DEVICE}" | grep -q ext4; then mkfs.ext4 -F "${DATA_DEVICE}" fi # Mount (skip if already mounted) if ! mountpoint -q "${DATA_MOUNT}"; then mount "${DATA_DEVICE}" "${DATA_MOUNT}" fi # Persist in fstab DEVICE_UUID="$(blkid -s UUID -o value "${DATA_DEVICE}")" if ! grep -q "${DEVICE_UUID}" /etc/fstab; then echo "UUID=${DEVICE_UUID} ${DATA_MOUNT} ext4 defaults,noatime 0 2" >> /etc/fstab fi # ── Derive internal IPs for retry_join ─────────────────────────────────────── # GCE metadata server gives us the IPs of the other nodes by instance name. # We query the Compute API via the attached SA (ADC — no key file needed). # For the initial single-node bootstrap, retry_join failures are harmless # (Raft simply starts as a single-node cluster until peers join). fetch_internal_ip() { local instance_name="$1" local zone="$2" gcloud compute instances describe "${instance_name}" \ --zone="${zone}" \ --format="get(networkInterfaces[0].networkIP)" \ 2>/dev/null || echo "" } # Map of all Vault nodes: name → zone declare -A NODE_ZONES NODE_ZONES["vault-node-1"]="us-central1-a" NODE_ZONES["vault-node-2"]="us-central1-b" NODE_ZONES["vault-node-3"]="us-central1-c" # Build retry_join stanzas for all nodes other than self RETRY_JOIN_STANZAS="" for node in "vault-node-1" "vault-node-2" "vault-node-3"; do if [[ "${node}" == "${VAULT_NODE_ID}" ]]; then continue fi zone="${NODE_ZONES[$node]}" ip="$(fetch_internal_ip "${node}" "${zone}")" if [[ -n "${ip}" ]]; then RETRY_JOIN_STANZAS="${RETRY_JOIN_STANZAS} retry_join { leader_api_addr = \"http://${ip}:8200\" }" fi done # ── Get the internal IP of this node ───────────────────────────────────────── MY_INTERNAL_IP="$(curl -sf -H "Metadata-Flavor: Google" \ "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip")" # ── Write Vault config ──────────────────────────────────────────────────────── mkdir -p /etc/vault.d cat > /etc/vault.d/vault.hcl </dev/null || useradd --system --home /etc/vault.d --shell /bin/false vault chown -R vault:vault /etc/vault.d /opt/vault/data # ── Systemd unit ────────────────────────────────────────────────────────────── cat > /etc/systemd/system/vault.service <<'EOF' [Unit] Description=HashiCorp Vault — a tool for managing secrets Documentation=https://developer.hashicorp.com/vault/docs Requires=network-online.target After=network-online.target ConditionFileNotEmpty=/etc/vault.d/vault.hcl StartLimitIntervalSec=60 StartLimitBurst=3 [Service] Type=notify User=vault Group=vault ProtectSystem=full ProtectHome=read-only PrivateTmp=yes PrivateDevices=yes SecureBits=keep-caps AmbientCapabilities=CAP_IPC_LOCK CapabilityBoundingSet=CAP_SYSLOG CAP_IPC_LOCK NoNewPrivileges=yes ExecStart=/usr/bin/vault server -config=/etc/vault.d/vault.hcl ExecReload=/bin/kill --signal HUP $MAINPID KillMode=process KillSignal=SIGINT Restart=on-failure RestartSec=5 TimeoutStopSec=30 LimitNOFILE=65536 LimitMEMLOCK=infinity [Install] WantedBy=multi-user.target EOF systemctl daemon-reload systemctl enable vault # Start or restart (restart is idempotent — re-reads config on VM metadata updates) if systemctl is-active vault &>/dev/null; then systemctl restart vault else systemctl start vault fi echo "Vault ${VAULT_NODE_ID} started successfully" echo "Run 'vault operator init' on vault-node-1 to bootstrap the cluster"