infrastructure/servers/legion/k8s/backup/cronjob.yaml

---
# Nightly backup — dumps Postgres + Gitea data to Cloudflare R2 via restic
# Schedule: 2am daily. Retains 30 daily / 12 weekly / 6 monthly snapshots.
apiVersion: batch/v1
kind: CronJob
metadata:
  name: gitea-backup
  namespace: git
spec:
  schedule: "0 2 * * *"
  concurrencyPolicy: Forbid
  failedJobsHistoryLimit: 3
  successfulJobsHistoryLimit: 3
  jobTemplate:
    spec:
      backoffLimit: 2
      template:
        spec:
          restartPolicy: OnFailure
          initContainers:
            - name: pg-dump
              image: postgres:18-alpine
              command:
                - /bin/sh
                - -c
                - >-
                  pg_dumpall
                  -h postgres-postgresql.platform.svc.cluster.local
                  -U postgres
                  --no-role-passwords
                  > /dump/all-databases.sql
                  && echo 'DB dump complete, size:' $(du -sh /dump/all-databases.sql)
              envFrom:
                - secretRef:
                    name: backup-credentials
              volumeMounts:
                - name: dump
                  mountPath: /dump
              resources:
                requests:
                  memory: 128Mi
                  cpu: 100m
                limits:
                  memory: 256Mi
          containers:
            # Pin to explicit version — never use latest for backup tooling.
            # Bump this intentionally after testing; do not let it float.
            - name: backup
              image: restic/restic:0.17.3
              command:
                - /bin/sh
                - -c
                - |
                  set -e

                  # Init repo if first run
                  restic snapshots 2>/dev/null || restic init

                  # Back up Gitea data + full DB dump
                  echo "Running restic backup..."
                  restic backup \
                    /gitea-data \
                    /dump/all-databases.sql \
                    --tag legion \
                    --host legion

                  # Prune old snapshots
                  restic forget \
                    --keep-daily 30 \
                    --keep-weekly 12 \
                    --keep-monthly 6 \
                    --prune

                  echo "Backup complete."
                  restic snapshots --latest 3
              envFrom:
                - secretRef:
                    name: backup-credentials
              volumeMounts:
                - name: gitea-data
                  mountPath: /gitea-data
                  readOnly: true
                - name: dump
                  mountPath: /dump
              resources:
                requests:
                  memory: 256Mi
                  cpu: 100m
                limits:
                  memory: 512Mi
          volumes:
            - name: gitea-data
              persistentVolumeClaim:
                claimName: gitea-data
                readOnly: true
            - name: dump
              emptyDir: {}
---
# Weekly restore-verify job — proves the backup is actually restorable.
# Runs Sunday 4am. Restores latest snapshot to /tmp, validates the SQL dump.
# Alerts via Alertmanager if it fails (BackupJobFailed PrometheusRule watches git namespace).
apiVersion: batch/v1
kind: CronJob
metadata:
  name: backup-verify
  namespace: git
spec:
  schedule: "0 4 * * 0"
  concurrencyPolicy: Forbid
  failedJobsHistoryLimit: 2
  successfulJobsHistoryLimit: 1
  jobTemplate:
    spec:
      backoffLimit: 0
      activeDeadlineSeconds: 1800  # 30 min max
      template:
        spec:
          restartPolicy: Never
          containers:
            - name: verify
              image: restic/restic:0.17.3
              command:
                - /bin/sh
                - -c
                - |
                  set -e

                  echo "=== Backup Restore Verification ==="
                  echo "Listing latest snapshots..."
                  restic snapshots --latest 3

                  echo "Restoring latest snapshot to /tmp/restore-verify..."
                  mkdir -p /tmp/restore-verify
                  restic restore latest \
                    --target /tmp/restore-verify \
                    --include /dump/all-databases.sql

                  SQL_FILE="/tmp/restore-verify/dump/all-databases.sql"
                  if [ ! -f "$SQL_FILE" ]; then
                    echo "ERROR: SQL dump not found in restored snapshot"
                    exit 1
                  fi

                  SIZE=$(du -sh "$SQL_FILE" | cut -f1)
                  echo "SQL dump restored: $SIZE"

                  # Basic structural validation
                  if ! grep -q "PostgreSQL database cluster dump" "$SQL_FILE"; then
                    echo "ERROR: SQL dump does not look like a valid pg_dumpall output"
                    exit 1
                  fi

                  TABLE_COUNT=$(grep -c "^CREATE TABLE" "$SQL_FILE" || true)
                  echo "Tables found in dump: $TABLE_COUNT"

                  if [ "$TABLE_COUNT" -lt 5 ]; then
                    echo "ERROR: Too few tables in dump (expected >= 5, got $TABLE_COUNT)"
                    exit 1
                  fi

                  echo "=== Verification PASSED: backup is restorable ==="
                  rm -rf /tmp/restore-verify
              envFrom:
                - secretRef:
                    name: backup-credentials
              resources:
                requests:
                  memory: 256Mi
                  cpu: 100m
                limits:
                  memory: 512Mi