diff --git a/servers/legion/google-storage.tf b/servers/legion/google-storage.tf new file mode 100644 index 0000000..5ef862e --- /dev/null +++ b/servers/legion/google-storage.tf @@ -0,0 +1,70 @@ +# Google Cloud Storage — neuron backup bucket + dedicated service account +# Paired with: k8s/neuron-technologies/prod/backup-cronjob.yaml (restic → GCS) +# k8s/backup/cronjob.yaml (restic → GCS for Gitea/Postgres) + +variable "gcp_project_id" { + description = "Google Cloud project ID" + type = string + default = "neuron-785695" +} + +provider "google" { + project = var.gcp_project_id + credentials = fileexists("${path.module}/vault-gcp-sa.json") ? file("${path.module}/vault-gcp-sa.json") : null +} + +# ── Neuron prod DB backup bucket ────────────────────────────────────────────── +resource "google_storage_bucket" "neuron_backup" { + name = "neuron-backup-${var.gcp_project_id}" + location = "US" + force_destroy = false + + uniform_bucket_level_access = true + + versioning { + enabled = false + } + + lifecycle_rule { + condition { age = 90 } + action { type = "Delete" } + } + + labels = { + managed-by = "terraform" + service = "neuron" + purpose = "backup" + tier = "prod" + } +} + +# ── Dedicated service account for backup writes ─────────────────────────────── +resource "google_service_account" "neuron_backup" { + account_id = "neuron-backup" + display_name = "Neuron Backup" + description = "Used by k8s restic CronJobs to write hourly backups to GCS" + project = var.gcp_project_id +} + +resource "google_storage_bucket_iam_member" "neuron_backup_writer" { + bucket = google_storage_bucket.neuron_backup.name + role = "roles/storage.objectAdmin" + member = "serviceAccount:${google_service_account.neuron_backup.email}" +} + +resource "google_service_account_key" "neuron_backup" { + service_account_id = google_service_account.neuron_backup.name + # After first apply: copy the base64 key into Vault: + # vault kv put secret/data/gcs neuron_backup_sa_key="$(terraform output -raw neuron_backup_sa_key_base64 | base64 -d)" +} + +output "neuron_backup_gcs_bucket" { + value = google_storage_bucket.neuron_backup.name + description = "GCS bucket name for use in RESTIC_REPOSITORY_GCS" +} + +output "neuron_backup_sa_key_base64" { + value = google_service_account_key.neuron_backup.private_key + sensitive = true + description = "Base64-encoded SA key JSON — store in Vault: secret/data/gcs.neuron_backup_sa_key" +} diff --git a/servers/legion/k8s/neuron-technologies/prod/backup-cronjob.yaml b/servers/legion/k8s/neuron-technologies/prod/backup-cronjob.yaml index d133836..6f4e252 100644 --- a/servers/legion/k8s/neuron-technologies/prod/backup-cronjob.yaml +++ b/servers/legion/k8s/neuron-technologies/prod/backup-cronjob.yaml @@ -1,5 +1,5 @@ --- -# neuron-backup-credentials — restic + R2 for neuron prod DB backups +# neuron-backup-credentials — restic + R2 + GCS for neuron prod DB backups apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: @@ -15,10 +15,16 @@ spec: creationPolicy: Owner template: data: + # ── R2 (primary) ── RESTIC_PASSWORD: "{{ .restic_password }}" AWS_ACCESS_KEY_ID: "{{ .r2_access_key_id }}" AWS_SECRET_ACCESS_KEY: "{{ .r2_secret_access_key }}" RESTIC_REPOSITORY: "s3:https://651161e0a3d321561b4c90b5bcd5f15b.r2.cloudflarestorage.com/legion-neuron-backup" + # ── GCS (secondary) ── + RESTIC_PASSWORD_GCS: "{{ .restic_password_gcs }}" + GCS_SA_KEY_JSON: "{{ .gcs_neuron_backup_sa_key }}" + GOOGLE_PROJECT_ID: "neuron-785695" + RESTIC_REPOSITORY_GCS: "gs:neuron-backup-neuron-785695:/" data: - secretKey: restic_password remoteRef: @@ -32,10 +38,17 @@ spec: remoteRef: key: secret/data/r2 property: secret_access_key + - secretKey: restic_password_gcs + remoteRef: + key: secret/data/gcs + property: neuron_restic_password + - secretKey: gcs_neuron_backup_sa_key + remoteRef: + key: secret/data/gcs + property: neuron_backup_sa_key --- -# Hourly neuron prod DB backup to Cloudflare R2 via restic. -# Retention: 24 hourly, 7 daily, 4 weekly. -# At current DB size (~10MB) this is <$0.01/month. +# Hourly neuron prod DB backup to Cloudflare R2 + Google Cloud Storage via restic. +# Retention: 24 hourly, 7 daily, 4 weekly on both backends. # Mounts neuron-prod-data PVC read-only — safe on single-node k3s. apiVersion: batch/v1 kind: CronJob @@ -46,7 +59,7 @@ spec: schedule: "0 * * * *" concurrencyPolicy: Forbid failedJobsHistoryLimit: 3 - successfulJobsHistoryLimit: 3 + successfulJobsHisticsLimit: 3 jobTemplate: spec: backoffLimit: 2 @@ -62,11 +75,12 @@ spec: - | set -e - # Init repo on first run (no-op if already initialized) + echo "=== Neuron DB backup: $(date -u) ===" + + # ── R2 backup (primary) ────────────────────────────────────────── + echo "--- R2 backup ---" restic snapshots 2>/dev/null || restic init - # Backup the SQLite DB file (WAL mode: exclude -shm/-wal, they're volatile) - echo "Backing up neuron prod DB..." restic backup \ /data/neuron-prod.db \ --tag neuron-prod \ @@ -74,15 +88,46 @@ spec: --exclude '*.db-shm' \ --exclude '*.db-wal' - # Prune: 24 hourly, 7 daily, 4 weekly restic forget \ --keep-hourly 24 \ --keep-daily 7 \ --keep-weekly 4 \ --prune - echo "Backup complete." - restic snapshots --latest 3 + echo "R2 backup complete. Latest snapshots:" + restic snapshots --latest 2 + + # ── GCS backup (secondary) ─────────────────────────────────────── + echo "--- GCS backup ---" + echo "$GCS_SA_KEY_JSON" > /tmp/gcs-sa.json + + ( + export RESTIC_REPOSITORY="$RESTIC_REPOSITORY_GCS" + export RESTIC_PASSWORD="$RESTIC_PASSWORD_GCS" + export GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcs-sa.json + unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + + restic snapshots 2>/dev/null || restic init + + restic backup \ + /data/neuron-prod.db \ + --tag neuron-prod \ + --host legion \ + --exclude '*.db-shm' \ + --exclude '*.db-wal' + + restic forget \ + --keep-hourly 24 \ + --keep-daily 7 \ + --keep-weekly 4 \ + --prune + + echo "GCS backup complete. Latest snapshots:" + restic snapshots --latest 2 + ) + + rm -f /tmp/gcs-sa.json + echo "=== All backups complete ===" envFrom: - secretRef: name: neuron-backup-credentials diff --git a/servers/legion/main.tf b/servers/legion/main.tf index ccf8aa2..df2d9f5 100644 --- a/servers/legion/main.tf +++ b/servers/legion/main.tf @@ -14,6 +14,10 @@ terraform { source = "cloudflare/cloudflare" version = "~> 4.0" } + google = { + source = "hashicorp/google" + version = "~> 5.0" + } } backend "s3" {