Disaster Recovery & Backup Guide

# Create a snapshot of a persistent disk
gcloud compute disks snapshot my-data-disk \
  --zone=us-central1-a \
  --snapshot-names=my-data-disk-$(date +%Y%m%d-%H%M%S) \
  --storage-location=us

# Create a snapshot schedule (automated backups)
gcloud compute resource-policies create snapshot-schedule daily-backup \
  --region=us-central1 \
  --max-retention-days=30 \
  --on-source-disk-delete=keep-auto-snapshots \
  --daily-schedule \
  --start-time=02:00 \
  --storage-location=us \
  --labels=backup-policy=daily

# Attach the snapshot schedule to a disk
gcloud compute disks add-resource-policies my-data-disk \
  --zone=us-central1-a \
  --resource-policies=daily-backup

# Create a cross-region snapshot (for DR)
gcloud compute snapshots create dr-snapshot-$(date +%Y%m%d) \
  --source-disk=my-data-disk \
  --source-disk-zone=us-central1-a \
  --storage-location=us-east1 \
  --labels=purpose=disaster-recovery

# Restore a disk from a snapshot
gcloud compute disks create restored-disk \
  --source-snapshot=my-data-disk-20240115-020000 \
  --zone=us-east1-b \
  --type=pd-ssd

# Create a VM from a snapshot (DR recovery)
gcloud compute instances create recovered-vm \
  --zone=us-east1-b \
  --machine-type=e2-standard-4 \
  --create-disk="boot=yes,auto-delete=yes,source-snapshot=my-boot-disk-snapshot"

# List all snapshots
gcloud compute snapshots list \
  --filter="labels.purpose=disaster-recovery" \
  --sort-by=~creationTimestamp

# Delete old snapshots
gcloud compute snapshots delete old-snapshot-name --quiet

# Cold standby infrastructure - only data replication runs normally
# Compute resources are created only during DR activation

# Cross-region Cloud SQL read replica (always running, low cost)
resource "google_sql_database_instance" "dr_replica" {
  name                 = "my-app-db-dr-replica"
  master_instance_name = google_sql_database_instance.primary.name
  region               = "us-east1" # DR region
  database_version     = "POSTGRES_15"

  replica_configuration {
    failover_target = true
  }

  settings {
    tier              = "db-custom-2-7680" # Smaller than primary
    availability_type = "ZONAL"            # No HA needed for replica
    disk_size         = 100
    disk_type         = "PD_SSD"

    ip_configuration {
      ipv4_enabled    = false
      private_network = google_compute_network.dr_vpc.id
    }

    backup_configuration {
      enabled = false # Primary handles backups
    }
  }
}

# Multi-region Cloud Storage for automatic replication
resource "google_storage_bucket" "app_data" {
  name     = "my-app-data-multi-region"
  location = "US" # Multi-region: auto-replicates across US regions

  versioning {
    enabled = true
  }

  lifecycle_rule {
    condition {
      num_newer_versions = 5
    }
    action {
      type = "Delete"
    }
  }
}

# Cross-region snapshot schedule for Compute Engine disks
resource "google_compute_resource_policy" "dr_snapshots" {
  name   = "dr-snapshot-policy"
  region = "us-central1"

  snapshot_schedule_policy {
    schedule {
      hourly_schedule {
        hours_in_cycle = 4
        start_time     = "00:00"
      }
    }

    retention_policy {
      max_retention_days    = 14
      on_source_disk_delete = "KEEP_AUTO_SNAPSHOTS"
    }

    snapshot_properties {
      storage_locations = ["us-east1"] # Store snapshots in DR region
      labels = {
        purpose = "disaster-recovery"
      }
    }
  }
}

# DR activation script (run manually or via automation)
# This creates compute resources in the DR region
# resource "google_compute_instance" "dr_app_server" {
#   count = var.dr_active ? var.instance_count : 0
#   ...
# }

# Warm standby uses Cloud Run (scales to near-zero when idle)
# and Cloud SQL cross-region replicas

# Deploy the application to both regions
# Primary region
gcloud run deploy my-api \
  --image=us-central1-docker.pkg.dev/my-project/images/my-api:latest \
  --region=us-central1 \
  --min-instances=2 \
  --max-instances=100 \
  --set-env-vars=DB_HOST=/cloudsql/my-project:us-central1:primary-db

# DR region (minimal capacity, ready to scale)
gcloud run deploy my-api \
  --image=us-east1-docker.pkg.dev/my-project/images/my-api:latest \
  --region=us-east1 \
  --min-instances=1 \
  --max-instances=100 \
  --set-env-vars=DB_HOST=/cloudsql/my-project:us-east1:dr-replica-db

# Set up Global Load Balancer with health-based failover
# The GLB automatically routes traffic to the healthy region

# Create serverless NEGs for both regions
gcloud compute network-endpoint-groups create api-neg-primary \
  --region=us-central1 \
  --network-endpoint-type=serverless \
  --cloud-run-service=my-api

gcloud compute network-endpoint-groups create api-neg-dr \
  --region=us-east1 \
  --network-endpoint-type=serverless \
  --cloud-run-service=my-api

# Create backend service with both NEGs
gcloud compute backend-services create api-backend \
  --global \
  --load-balancing-scheme=EXTERNAL_MANAGED

# Primary backend (higher capacity)
gcloud compute backend-services add-backend api-backend \
  --global \
  --network-endpoint-group=api-neg-primary \
  --network-endpoint-group-region=us-central1 \
  --capacity-scaler=1.0

# DR backend (lower capacity, scales up on failover)
gcloud compute backend-services add-backend api-backend \
  --global \
  --network-endpoint-group=api-neg-dr \
  --network-endpoint-group-region=us-east1 \
  --capacity-scaler=0.5

# DR FAILOVER PROCEDURE:
# 1. Promote Cloud SQL replica to primary
gcloud sql instances promote-replica dr-replica-db

# 2. Update Cloud Run DR service to point to promoted database
gcloud run services update my-api \
  --region=us-east1 \
  --set-env-vars=DB_HOST=/cloudsql/my-project:us-east1:dr-replica-db

# 3. Scale up DR region capacity
gcloud run services update my-api \
  --region=us-east1 \
  --min-instances=5 \
  --max-instances=200

# 4. Update backend service to route all traffic to DR
gcloud compute backend-services update-backend api-backend \
  --global \
  --network-endpoint-group=api-neg-dr \
  --network-endpoint-group-region=us-east1 \
  --capacity-scaler=1.0

gcloud compute backend-services update-backend api-backend \
  --global \
  --network-endpoint-group=api-neg-primary \
  --network-endpoint-group-region=us-central1 \
  --capacity-scaler=0.0

# Cloud Spanner multi-region instance provides the strongest
# multi-region consistency guarantees of any cloud database

resource "google_spanner_instance" "multi_region" {
  name         = "global-transactions"
  config       = "nam-eur-asia1" # Multi-continent configuration
  display_name = "Global Transaction Database"
  # Processing units: 1000 = 1 node
  processing_units = 3000 # 3 nodes

  labels = {
    environment = "production"
    dr_tier     = "tier-1"
  }
}

resource "google_spanner_database" "orders" {
  instance = google_spanner_instance.multi_region.name
  name     = "orders"

  database_dialect = "GOOGLE_STANDARD_SQL"

  ddl = [
    <<-SQL
      CREATE TABLE Orders (
        OrderId STRING(36) NOT NULL,
        CustomerId STRING(36) NOT NULL,
        Status STRING(20) NOT NULL,
        TotalAmount NUMERIC NOT NULL,
        Region STRING(20) NOT NULL,
        CreatedAt TIMESTAMP NOT NULL OPTIONS (allow_commit_timestamp=true),
        UpdatedAt TIMESTAMP NOT NULL OPTIONS (allow_commit_timestamp=true)
      ) PRIMARY KEY (OrderId)
    SQL
    ,
    <<-SQL
      CREATE INDEX OrdersByCustomer ON Orders(CustomerId)
    SQL
  ]

  deletion_protection = true

  # Automated backups
  enable_drop_protection = true
}

# Spanner backup schedule
resource "google_spanner_backup_schedule" "daily" {
  instance = google_spanner_instance.multi_region.name
  database = google_spanner_database.orders.name

  name = "daily-backup"

  retention_duration = "2592000s" # 30 days

  spec {
    cron_spec {
      text = "0 2 * * *" # Daily at 2 AM
    }
  }

  full_backup_spec {}
}

# Multi-region Cloud Run deployment
resource "google_cloud_run_v2_service" "api_us" {
  name     = "orders-api"
  location = "us-central1"

  template {
    scaling {
      min_instance_count = 2
      max_instance_count = 100
    }

    containers {
      image = "us-central1-docker.pkg.dev/my-project/images/orders-api:latest"
      env {
        name  = "SPANNER_INSTANCE"
        value = google_spanner_instance.multi_region.name
      }
      env {
        name  = "SPANNER_DATABASE"
        value = "orders"
      }
    }
  }
}

resource "google_cloud_run_v2_service" "api_eu" {
  name     = "orders-api"
  location = "europe-west1"

  template {
    scaling {
      min_instance_count = 2
      max_instance_count = 100
    }

    containers {
      image = "europe-west1-docker.pkg.dev/my-project/images/orders-api:latest"
      env {
        name  = "SPANNER_INSTANCE"
        value = google_spanner_instance.multi_region.name
      }
      env {
        name  = "SPANNER_DATABASE"
        value = "orders"
      }
    }
  }
}

# Cloud SQL: Create a cross-region read replica
gcloud sql instances create primary-db-dr-replica \
  --master-instance-name=primary-db \
  --region=us-east1 \
  --tier=db-custom-4-15360 \
  --availability-type=ZONAL \
  --no-assign-ip \
  --network=projects/my-project/global/networks/my-vpc

# Bigtable: Add a cluster in the DR region
gcloud bigtable clusters create dr-cluster \
  --instance=my-bigtable \
  --zone=us-east1-b \
  --num-nodes=3 \
  --storage-type=SSD

# Cloud Storage: Create a dual-region bucket
gcloud storage buckets create gs://my-critical-data-dual \
  --location=us-central1+us-east1 \
  --default-storage-class=STANDARD \
  --enable-turbo-replication

# Cloud Storage: Replicate between single-region buckets
gcloud transfer jobs create \
  gs://source-bucket-central \
  gs://dr-bucket-east \
  --name=dr-replication \
  --schedule-starts=2024-01-01 \
  --schedule-repeats-every=1h \
  --overwrite-when=different \
  --delete-from=never

# Persistent Disk: Async replication (for critical disks)
gcloud compute disks start-async-replication my-data-disk \
  --zone=us-central1-a \
  --secondary-disk=dr-data-disk \
  --secondary-disk-zone=us-east1-b

# Check replication status
gcloud compute disks describe my-data-disk \
  --zone=us-central1-a \
  --format='yaml(asyncPrimaryDisk,asyncSecondaryDisks)'

# Enable the Backup for GKE API
gcloud services enable gkebackup.googleapis.com

# Create a backup plan for the production cluster
gcloud beta container backup-restore backup-plans create prod-daily-backup \
  --project=my-project \
  --location=us-central1 \
  --cluster=projects/my-project/locations/us-central1/clusters/prod-cluster \
  --all-namespaces \
  --include-volume-data \
  --include-secrets \
  --cron-schedule="0 2 * * *" \
  --backup-retain-days=30 \
  --backup-delete-lock-days=7 \
  --labels=team=platform

# Create a backup plan for specific namespaces only
gcloud beta container backup-restore backup-plans create app-backup \
  --project=my-project \
  --location=us-central1 \
  --cluster=projects/my-project/locations/us-central1/clusters/prod-cluster \
  --selected-namespaces=production,staging \
  --include-volume-data \
  --cron-schedule="0 */6 * * *" \
  --backup-retain-days=14

# Manually trigger an on-demand backup
gcloud beta container backup-restore backups create manual-backup-$(date +%Y%m%d) \
  --project=my-project \
  --location=us-central1 \
  --backup-plan=prod-daily-backup \
  --wait-for-completion

# List available backups
gcloud beta container backup-restore backups list \
  --project=my-project \
  --location=us-central1 \
  --backup-plan=prod-daily-backup

# Create a restore plan (can target a different cluster/region for DR)
gcloud beta container backup-restore restore-plans create dr-restore-plan \
  --project=my-project \
  --location=us-east1 \
  --backup-plan=projects/my-project/locations/us-central1/backupPlans/prod-daily-backup \
  --cluster=projects/my-project/locations/us-east1/clusters/dr-cluster \
  --all-namespaces \
  --volume-data-restore-policy=RESTORE_VOLUME_DATA_FROM_BACKUP \
  --cluster-resource-conflict-policy=USE_BACKUP_VERSION

# Execute a restore
gcloud beta container backup-restore restores create dr-restore-$(date +%Y%m%d) \
  --project=my-project \
  --location=us-east1 \
  --restore-plan=dr-restore-plan \
  --backup=projects/my-project/locations/us-central1/backupPlans/prod-daily-backup/backups/manual-backup-20240115 \
  --wait-for-completion

# Create a Cloud SQL instance with HA and automated backups
gcloud sql instances create primary-db \
  --database-version=POSTGRES_15 \
  --tier=db-custom-4-15360 \
  --region=us-central1 \
  --availability-type=REGIONAL \
  --backup-start-time=02:00 \
  --enable-bin-log \
  --enable-point-in-time-recovery \
  --retained-backups-count=30 \
  --retained-transaction-log-days=7 \
  --network=projects/my-project/global/networks/my-vpc \
  --no-assign-ip

# Create a cross-region replica for DR
gcloud sql instances create primary-db-dr \
  --master-instance-name=primary-db \
  --region=us-east1 \
  --tier=db-custom-4-15360 \
  --availability-type=ZONAL \
  --network=projects/my-project/global/networks/my-vpc \
  --no-assign-ip

# Check replication status
gcloud sql instances describe primary-db-dr \
  --format='yaml(replicaConfiguration,replicaNames)'

# DR FAILOVER: Promote the replica to a standalone instance
# WARNING: This is irreversible - the replica becomes independent
gcloud sql instances promote-replica primary-db-dr

# After promotion, create a new replica for continued DR protection
gcloud sql instances create primary-db-dr-new \
  --master-instance-name=primary-db-dr \
  --region=us-central1 \
  --tier=db-custom-4-15360

# Point-in-time recovery (to recover from data corruption)
gcloud sql instances clone primary-db recovered-db \
  --point-in-time='2024-01-15T14:30:00Z'

# Export database for additional backup (to Cloud Storage)
gcloud sql export sql primary-db \
  gs://my-backups/sql/primary-db-$(date +%Y%m%d).sql.gz \
  --database=myapp \
  --offload

#!/bin/bash
# dr-test.sh - Automated DR testing script
# Run this monthly against a test environment

set -euo pipefail

PROJECT_ID="my-project"
PRIMARY_REGION="us-central1"
DR_REGION="us-east1"
TIMESTAMP=$(date +%Y%m%d-%H%M%S)

echo "=== DR Test Started: $TIMESTAMP ==="

# Test 1: Verify backup freshness
echo "--- Test 1: Checking backup freshness ---"
LATEST_BACKUP=$(gcloud sql backups list --instance=primary-db \
  --filter="status=SUCCESSFUL" \
  --sort-by=~startTime \
  --limit=1 \
  --format='value(startTime)')
echo "Latest Cloud SQL backup: $LATEST_BACKUP"

LATEST_SNAPSHOT=$(gcloud compute snapshots list \
  --filter="labels.purpose=disaster-recovery" \
  --sort-by=~creationTimestamp \
  --limit=1 \
  --format='value(creationTimestamp)')
echo "Latest disk snapshot: $LATEST_SNAPSHOT"

# Test 2: Verify cross-region replication status
echo "--- Test 2: Checking replication status ---"
REPLICA_STATUS=$(gcloud sql instances describe primary-db-dr \
  --format='value(state)')
echo "Cloud SQL replica status: $REPLICA_STATUS"

if [ "$REPLICA_STATUS" != "RUNNABLE" ]; then
  echo "CRITICAL: DR replica is not in RUNNABLE state!"
  exit 1
fi

# Test 3: Test snapshot restore (to a temporary disk)
echo "--- Test 3: Testing snapshot restore ---"
gcloud compute disks create dr-test-restore-$TIMESTAMP \
  --source-snapshot=$LATEST_SNAPSHOT \
  --zone=$DR_REGION-b \
  --type=pd-ssd --quiet

echo "Snapshot restore successful"
gcloud compute disks delete dr-test-restore-$TIMESTAMP \
  --zone=$DR_REGION-b --quiet

# Test 4: Verify DR region Cloud Run service health
echo "--- Test 4: Checking DR region service health ---"
DR_URL=$(gcloud run services describe my-api \
  --region=$DR_REGION \
  --format='value(status.url)')

HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$DR_URL/health")
echo "DR service health check: HTTP $HTTP_STATUS"

if [ "$HTTP_STATUS" != "200" ]; then
  echo "WARNING: DR region service returned non-200 status"
fi

# Test 5: Measure actual RTO (time to promote replica)
echo "--- Test 5: Measuring simulated RTO ---"
echo "Simulated RTO measurement would require a full failover test"
echo "Schedule a full failover test for the next maintenance window"

echo "=== DR Test Completed: $(date +%Y%m%d-%H%M%S) ==="
echo "Results saved to gs://$PROJECT_ID-dr-reports/dr-test-$TIMESTAMP.log"