Add GCS backup bucket + dual-destination hourly backup (R2 + GCS)

Provision Google Cloud Storage bucket for neuron prod DB backups via Terraform.
Create dedicated backup service account with objectAdmin on the bucket.
Update neuron-prod backup CronJob to run restic against both R2 and GCS hourly —
R2 as primary, GCS as secondary, independent credentials and repositories.

servers/legion/google-storage.tf

@@ -0,0 +1,70 @@
+# Google Cloud Storage — neuron backup bucket + dedicated service account
+# Paired with: k8s/neuron-technologies/prod/backup-cronjob.yaml (restic → GCS)
+#              k8s/backup/cronjob.yaml (restic → GCS for Gitea/Postgres)
+
+variable "gcp_project_id" {
+  description = "Google Cloud project ID"
+  type        = string
+  default     = "neuron-785695"
+}
+
+provider "google" {
+  project     = var.gcp_project_id
+  credentials = fileexists("${path.module}/vault-gcp-sa.json") ? file("${path.module}/vault-gcp-sa.json") : null
+}
+
+# ── Neuron prod DB backup bucket ──────────────────────────────────────────────
+resource "google_storage_bucket" "neuron_backup" {
+  name          = "neuron-backup-${var.gcp_project_id}"
+  location      = "US"
+  force_destroy = false
+
+  uniform_bucket_level_access = true
+
+  versioning {
+    enabled = false
+  }
+
+  lifecycle_rule {
+    condition { age = 90 }
+    action    { type = "Delete" }
+  }
+
+  labels = {
+    managed-by  = "terraform"
+    service     = "neuron"
+    purpose     = "backup"
+    tier        = "prod"
+  }
+}
+
+# ── Dedicated service account for backup writes ───────────────────────────────
+resource "google_service_account" "neuron_backup" {
+  account_id   = "neuron-backup"
+  display_name = "Neuron Backup"
+  description  = "Used by k8s restic CronJobs to write hourly backups to GCS"
+  project      = var.gcp_project_id
+}
+
+resource "google_storage_bucket_iam_member" "neuron_backup_writer" {
+  bucket = google_storage_bucket.neuron_backup.name
+  role   = "roles/storage.objectAdmin"
+  member = "serviceAccount:${google_service_account.neuron_backup.email}"
+}
+
+resource "google_service_account_key" "neuron_backup" {
+  service_account_id = google_service_account.neuron_backup.name
+  # After first apply: copy the base64 key into Vault:
+  #   vault kv put secret/data/gcs neuron_backup_sa_key="$(terraform output -raw neuron_backup_sa_key_base64 | base64 -d)"
+}
+
+output "neuron_backup_gcs_bucket" {
+  value       = google_storage_bucket.neuron_backup.name
+  description = "GCS bucket name for use in RESTIC_REPOSITORY_GCS"
+}
+
+output "neuron_backup_sa_key_base64" {
+  value       = google_service_account_key.neuron_backup.private_key
+  sensitive   = true
+  description = "Base64-encoded SA key JSON — store in Vault: secret/data/gcs.neuron_backup_sa_key"
+}

servers/legion/k8s/neuron-technologies/prod/backup-cronjob.yaml

@@ -1,5 +1,5 @@
 ---
-# neuron-backup-credentials — restic + R2 for neuron prod DB backups
+# neuron-backup-credentials — restic + R2 + GCS for neuron prod DB backups
 apiVersion: external-secrets.io/v1beta1
 kind: ExternalSecret
 metadata:
@@ -15,10 +15,16 @@ spec:
     creationPolicy: Owner
     template:
       data:
+        # ── R2 (primary) ──
         RESTIC_PASSWORD: "{{ .restic_password }}"
         AWS_ACCESS_KEY_ID: "{{ .r2_access_key_id }}"
         AWS_SECRET_ACCESS_KEY: "{{ .r2_secret_access_key }}"
         RESTIC_REPOSITORY: "s3:https://651161e0a3d321561b4c90b5bcd5f15b.r2.cloudflarestorage.com/legion-neuron-backup"
+        # ── GCS (secondary) ──
+        RESTIC_PASSWORD_GCS: "{{ .restic_password_gcs }}"
+        GCS_SA_KEY_JSON: "{{ .gcs_neuron_backup_sa_key }}"
+        GOOGLE_PROJECT_ID: "neuron-785695"
+        RESTIC_REPOSITORY_GCS: "gs:neuron-backup-neuron-785695:/"
   data:
     - secretKey: restic_password
       remoteRef:
@@ -32,10 +38,17 @@ spec:
       remoteRef:
         key: secret/data/r2
         property: secret_access_key
+    - secretKey: restic_password_gcs
+      remoteRef:
+        key: secret/data/gcs
+        property: neuron_restic_password
+    - secretKey: gcs_neuron_backup_sa_key
+      remoteRef:
+        key: secret/data/gcs
+        property: neuron_backup_sa_key
 ---
-# Hourly neuron prod DB backup to Cloudflare R2 via restic.
+# Hourly neuron prod DB backup to Cloudflare R2 + Google Cloud Storage via restic.
-# Retention: 24 hourly, 7 daily, 4 weekly.
+# Retention: 24 hourly, 7 daily, 4 weekly on both backends.
-# At current DB size (~10MB) this is <$0.01/month.
 # Mounts neuron-prod-data PVC read-only — safe on single-node k3s.
 apiVersion: batch/v1
 kind: CronJob
@@ -46,7 +59,7 @@ spec:
   schedule: "0 * * * *"
   concurrencyPolicy: Forbid
   failedJobsHistoryLimit: 3
-  successfulJobsHistoryLimit: 3
+  successfulJobsHisticsLimit: 3
   jobTemplate:
     spec:
       backoffLimit: 2
@@ -62,11 +75,12 @@ spec:
                 - |
                   set -e
 
-                  # Init repo on first run (no-op if already initialized)
+                  echo "=== Neuron DB backup: $(date -u) ==="
+
+                  # ── R2 backup (primary) ──────────────────────────────────────────
+                  echo "--- R2 backup ---"
                   restic snapshots 2>/dev/null || restic init
 
-                  # Backup the SQLite DB file (WAL mode: exclude -shm/-wal, they're volatile)
-                  echo "Backing up neuron prod DB..."
                   restic backup \
                     /data/neuron-prod.db \
                     --tag neuron-prod \
@@ -74,15 +88,46 @@ spec:
                     --exclude '*.db-shm' \
                     --exclude '*.db-wal'
 
-                  # Prune: 24 hourly, 7 daily, 4 weekly
                   restic forget \
                     --keep-hourly 24 \
                     --keep-daily 7 \
                     --keep-weekly 4 \
                     --prune
 
-                  echo "Backup complete."
+                  echo "R2 backup complete. Latest snapshots:"
-                  restic snapshots --latest 3
+                  restic snapshots --latest 2
+
+                  # ── GCS backup (secondary) ───────────────────────────────────────
+                  echo "--- GCS backup ---"
+                  echo "$GCS_SA_KEY_JSON" > /tmp/gcs-sa.json
+
+                  (
+                    export RESTIC_REPOSITORY="$RESTIC_REPOSITORY_GCS"
+                    export RESTIC_PASSWORD="$RESTIC_PASSWORD_GCS"
+                    export GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcs-sa.json
+                    unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
+
+                    restic snapshots 2>/dev/null || restic init
+
+                    restic backup \
+                      /data/neuron-prod.db \
+                      --tag neuron-prod \
+                      --host legion \
+                      --exclude '*.db-shm' \
+                      --exclude '*.db-wal'
+
+                    restic forget \
+                      --keep-hourly 24 \
+                      --keep-daily 7 \
+                      --keep-weekly 4 \
+                      --prune
+
+                    echo "GCS backup complete. Latest snapshots:"
+                    restic snapshots --latest 2
+                  )
+
+                  rm -f /tmp/gcs-sa.json
+                  echo "=== All backups complete ==="
               envFrom:
                 - secretRef:
                     name: neuron-backup-credentials

servers/legion/main.tf

@@ -14,6 +14,10 @@ terraform {
       source  = "cloudflare/cloudflare"
       version = "~> 4.0"
     }
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 5.0"
+    }
   }
 
   backend "s3" {