Wire GCS backup to neuron-db-backup-prod (neuron-494301)
This commit is contained in:
@@ -0,0 +1,11 @@
|
|||||||
|
# Google Cloud Storage — neuron backup bucket (neuron-494301 / neurontechnologies)
|
||||||
|
# Bucket: gs://neuron-db-backup-prod (created 2026-04-25, US multi-region)
|
||||||
|
# SA: neuron-agent@neuron-494301.iam.gserviceaccount.com (Storage Admin)
|
||||||
|
# Key: stored in Vault at secret/gcs.neuron_backup_sa_key
|
||||||
|
# CronJob: k8s/neuron-technologies/prod/backup-cronjob.yaml
|
||||||
|
|
||||||
|
variable "gcp_project_id" {
|
||||||
|
description = "Google Cloud project ID for neurontechnologies"
|
||||||
|
type = string
|
||||||
|
default = "neuron-494301"
|
||||||
|
}
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
---
|
---
|
||||||
# neuron-backup-credentials — restic + R2 for neuron prod DB backups
|
# neuron-backup-credentials — restic + R2 + GCS for neuron prod DB backups
|
||||||
apiVersion: external-secrets.io/v1beta1
|
apiVersion: external-secrets.io/v1beta1
|
||||||
kind: ExternalSecret
|
kind: ExternalSecret
|
||||||
metadata:
|
metadata:
|
||||||
@@ -15,10 +15,16 @@ spec:
|
|||||||
creationPolicy: Owner
|
creationPolicy: Owner
|
||||||
template:
|
template:
|
||||||
data:
|
data:
|
||||||
|
# ── R2 (primary) ──
|
||||||
RESTIC_PASSWORD: "{{ .restic_password }}"
|
RESTIC_PASSWORD: "{{ .restic_password }}"
|
||||||
AWS_ACCESS_KEY_ID: "{{ .r2_access_key_id }}"
|
AWS_ACCESS_KEY_ID: "{{ .r2_access_key_id }}"
|
||||||
AWS_SECRET_ACCESS_KEY: "{{ .r2_secret_access_key }}"
|
AWS_SECRET_ACCESS_KEY: "{{ .r2_secret_access_key }}"
|
||||||
RESTIC_REPOSITORY: "s3:https://651161e0a3d321561b4c90b5bcd5f15b.r2.cloudflarestorage.com/legion-neuron-backup"
|
RESTIC_REPOSITORY: "s3:https://651161e0a3d321561b4c90b5bcd5f15b.r2.cloudflarestorage.com/legion-neuron-backup"
|
||||||
|
# ── GCS (secondary) ──
|
||||||
|
RESTIC_PASSWORD_GCS: "{{ .restic_password_gcs }}"
|
||||||
|
GCS_SA_KEY_JSON: "{{ .gcs_neuron_backup_sa_key }}"
|
||||||
|
GOOGLE_PROJECT_ID: "neuron-494301"
|
||||||
|
RESTIC_REPOSITORY_GCS: "gs:neuron-db-backup-prod:/"
|
||||||
data:
|
data:
|
||||||
- secretKey: restic_password
|
- secretKey: restic_password
|
||||||
remoteRef:
|
remoteRef:
|
||||||
@@ -32,10 +38,17 @@ spec:
|
|||||||
remoteRef:
|
remoteRef:
|
||||||
key: secret/data/r2
|
key: secret/data/r2
|
||||||
property: secret_access_key
|
property: secret_access_key
|
||||||
|
- secretKey: restic_password_gcs
|
||||||
|
remoteRef:
|
||||||
|
key: secret/data/gcs
|
||||||
|
property: neuron_restic_password
|
||||||
|
- secretKey: gcs_neuron_backup_sa_key
|
||||||
|
remoteRef:
|
||||||
|
key: secret/data/gcs
|
||||||
|
property: neuron_backup_sa_key
|
||||||
---
|
---
|
||||||
# Hourly neuron prod DB backup to Cloudflare R2 via restic.
|
# Hourly neuron prod DB backup to Cloudflare R2 + Google Cloud Storage via restic.
|
||||||
# Retention: 24 hourly, 7 daily, 4 weekly.
|
# Retention: 24 hourly, 7 daily, 4 weekly on both backends.
|
||||||
# At current DB size (~10MB) this is <$0.01/month.
|
|
||||||
# Mounts neuron-prod-data PVC read-only — safe on single-node k3s.
|
# Mounts neuron-prod-data PVC read-only — safe on single-node k3s.
|
||||||
apiVersion: batch/v1
|
apiVersion: batch/v1
|
||||||
kind: CronJob
|
kind: CronJob
|
||||||
@@ -62,11 +75,12 @@ spec:
|
|||||||
- |
|
- |
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
# Init repo on first run (no-op if already initialized)
|
echo "=== Neuron DB backup: $(date -u) ==="
|
||||||
|
|
||||||
|
# ── R2 backup (primary) ──────────────────────────────────────────
|
||||||
|
echo "--- R2 backup ---"
|
||||||
restic snapshots 2>/dev/null || restic init
|
restic snapshots 2>/dev/null || restic init
|
||||||
|
|
||||||
# Backup the SQLite DB file (WAL mode: exclude -shm/-wal, they're volatile)
|
|
||||||
echo "Backing up neuron prod DB..."
|
|
||||||
restic backup \
|
restic backup \
|
||||||
/data/neuron-prod.db \
|
/data/neuron-prod.db \
|
||||||
--tag neuron-prod \
|
--tag neuron-prod \
|
||||||
@@ -74,15 +88,46 @@ spec:
|
|||||||
--exclude '*.db-shm' \
|
--exclude '*.db-shm' \
|
||||||
--exclude '*.db-wal'
|
--exclude '*.db-wal'
|
||||||
|
|
||||||
# Prune: 24 hourly, 7 daily, 4 weekly
|
|
||||||
restic forget \
|
restic forget \
|
||||||
--keep-hourly 24 \
|
--keep-hourly 24 \
|
||||||
--keep-daily 7 \
|
--keep-daily 7 \
|
||||||
--keep-weekly 4 \
|
--keep-weekly 4 \
|
||||||
--prune
|
--prune
|
||||||
|
|
||||||
echo "Backup complete."
|
echo "R2 backup complete. Latest snapshots:"
|
||||||
restic snapshots --latest 3
|
restic snapshots --latest 2
|
||||||
|
|
||||||
|
# ── GCS backup (secondary) ───────────────────────────────────────
|
||||||
|
echo "--- GCS backup ---"
|
||||||
|
echo "$GCS_SA_KEY_JSON" > /tmp/gcs-sa.json
|
||||||
|
|
||||||
|
(
|
||||||
|
export RESTIC_REPOSITORY="$RESTIC_REPOSITORY_GCS"
|
||||||
|
export RESTIC_PASSWORD="$RESTIC_PASSWORD_GCS"
|
||||||
|
export GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcs-sa.json
|
||||||
|
unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
|
||||||
|
|
||||||
|
restic snapshots 2>/dev/null || restic init
|
||||||
|
|
||||||
|
restic backup \
|
||||||
|
/data/neuron-prod.db \
|
||||||
|
--tag neuron-prod \
|
||||||
|
--host legion \
|
||||||
|
--exclude '*.db-shm' \
|
||||||
|
--exclude '*.db-wal'
|
||||||
|
|
||||||
|
restic forget \
|
||||||
|
--keep-hourly 24 \
|
||||||
|
--keep-daily 7 \
|
||||||
|
--keep-weekly 4 \
|
||||||
|
--prune
|
||||||
|
|
||||||
|
echo "GCS backup complete. Latest snapshots:"
|
||||||
|
restic snapshots --latest 2
|
||||||
|
)
|
||||||
|
|
||||||
|
rm -f /tmp/gcs-sa.json
|
||||||
|
echo "=== All backups complete ==="
|
||||||
envFrom:
|
envFrom:
|
||||||
- secretRef:
|
- secretRef:
|
||||||
name: neuron-backup-credentials
|
name: neuron-backup-credentials
|
||||||
|
|||||||
@@ -14,6 +14,10 @@ terraform {
|
|||||||
source = "cloudflare/cloudflare"
|
source = "cloudflare/cloudflare"
|
||||||
version = "~> 4.0"
|
version = "~> 4.0"
|
||||||
}
|
}
|
||||||
|
google = {
|
||||||
|
source = "hashicorp/google"
|
||||||
|
version = "~> 5.0"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
backend "s3" {
|
backend "s3" {
|
||||||
|
|||||||
Reference in New Issue
Block a user