Disaster Recovery Strategies

# CloudFormation - AWS Backup plan with cross-region copy
Resources:
  BackupVault:
    Type: AWS::Backup::BackupVault
    Properties:
      BackupVaultName: primary-region-vault
      EncryptionKeyArn: !GetAtt BackupKMSKey.Arn

  DRBackupVault:
    Type: AWS::Backup::BackupVault
    Properties:
      BackupVaultName: dr-region-vault
      # This vault must be created in the DR region
      # Use a stack set or separate stack in the DR region

  BackupPlan:
    Type: AWS::Backup::BackupPlan
    Properties:
      BackupPlan:
        BackupPlanName: production-backup-plan
        BackupPlanRule:
          # Hourly backups with 24-hour retention
          - RuleName: HourlyBackups
            TargetBackupVault: !Ref BackupVault
            ScheduleExpression: "cron(0 * * * ? *)"
            StartWindowMinutes: 60
            CompletionWindowMinutes: 120
            Lifecycle:
              DeleteAfterDays: 1

          # Daily backups with 30-day retention + cross-region copy
          - RuleName: DailyBackupsWithCrossRegion
            TargetBackupVault: !Ref BackupVault
            ScheduleExpression: "cron(0 3 * * ? *)"
            StartWindowMinutes: 60
            CompletionWindowMinutes: 180
            Lifecycle:
              DeleteAfterDays: 30
            CopyActions:
              - DestinationBackupVaultArn: !Sub "arn:aws:backup:us-west-2:${AWS::AccountId}:backup-vault:dr-region-vault"
                Lifecycle:
                  DeleteAfterDays: 30

          # Weekly backups with 1-year retention
          - RuleName: WeeklyBackups
            TargetBackupVault: !Ref BackupVault
            ScheduleExpression: "cron(0 3 ? * SUN *)"
            StartWindowMinutes: 60
            Lifecycle:
              DeleteAfterDays: 365

  BackupSelection:
    Type: AWS::Backup::BackupSelection
    Properties:
      BackupPlanId: !Ref BackupPlan
      BackupSelection:
        SelectionName: production-resources
        IamRoleArn: !GetAtt BackupRole.Arn
        # Back up all resources tagged for DR
        ListOfTags:
          - ConditionType: STRINGEQUALS
            ConditionKey: backup
            ConditionValue: "true"
        Resources:
          - "arn:aws:rds:*:*:db:*"
          - "arn:aws:dynamodb:*:*:table/*"
          - "arn:aws:ec2:*:*:volume/*"

# Pilot light infrastructure in DR region
Resources:
  # Core data layer - always running
  DRDatabase:
    Type: AWS::RDS::DBInstance
    Properties:
      SourceDBInstanceIdentifier: !Sub "arn:aws:rds:us-east-1:${AWS::AccountId}:db:production-db"
      DBInstanceClass: db.r6g.large  # Can be smaller than production
      MultiAZ: false                  # Single AZ to save cost in standby
      StorageEncrypted: true

  # DynamoDB Global Table (automatically replicated)
  # Note: DynamoDB Global Tables are configured in the primary region
  # and automatically maintain replicas in specified regions

  # VPC and networking - always provisioned
  DRVPC:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: 10.1.0.0/16
      EnableDnsHostnames: true
      Tags:
        - Key: Name
          Value: dr-vpc

  # AMIs and launch templates - always ready
  DRLaunchTemplate:
    Type: AWS::EC2::LaunchTemplate
    Properties:
      LaunchTemplateName: dr-web-server
      LaunchTemplateData:
        ImageId: !Ref LatestAMI
        InstanceType: t3.large
        SecurityGroupIds:
          - !Ref DRWebSecurityGroup
        UserData:
          Fn::Base64: !Sub |
            #!/bin/bash
            # Application bootstrap script
            aws s3 cp s3://deployment-artifacts/latest/app.tar.gz /opt/app/
            tar xzf /opt/app/app.tar.gz -C /opt/app/
            systemctl start myapp

  # Auto Scaling Group - configured but with 0 instances
  DRAutoScalingGroup:
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      AutoScalingGroupName: dr-web-servers
      LaunchTemplate:
        LaunchTemplateId: !Ref DRLaunchTemplate
        Version: !GetAtt DRLaunchTemplate.LatestVersionNumber
      MinSize: 0          # No instances running in standby
      MaxSize: 20
      DesiredCapacity: 0  # Scale up during failover
      VPCZoneIdentifier:
        - !Ref DRPrivateSubnet1
        - !Ref DRPrivateSubnet2
      TargetGroupARNs:
        - !Ref DRTargetGroup

#!/bin/bash
# Warm standby failover script
# Triggered manually or by automated health check failure

set -e

DR_REGION="us-west-2"
PRIMARY_REGION="us-east-1"
HOSTED_ZONE_ID="Z1234567890"
DR_ALB_DNS="dr-alb-123456.us-west-2.elb.amazonaws.com"
DR_ALB_HOSTED_ZONE="Z35SXDOTRQ7X7K"

echo "=== INITIATING DISASTER RECOVERY FAILOVER ==="
echo "Timestamp: $(date -u '+%Y-%m-%dT%H:%M:%SZ')"

# Step 1: Promote RDS read replica to standalone primary
echo "Step 1: Promoting RDS read replica..."
aws rds promote-read-replica \
  --db-instance-identifier dr-production-db \
  --region $DR_REGION

aws rds wait db-instance-available \
  --db-instance-identifier dr-production-db \
  --region $DR_REGION
echo "RDS replica promoted successfully."

# Step 2: Scale up compute
echo "Step 2: Scaling up compute resources..."
aws autoscaling update-auto-scaling-group \
  --auto-scaling-group-name dr-web-servers \
  --min-size 4 \
  --max-size 20 \
  --desired-capacity 8 \
  --region $DR_REGION

# Wait for instances to be healthy
echo "Waiting for instances to register with target group..."
aws elbv2 wait target-in-service \
  --target-group-arn "arn:aws:elasticloadbalancing:$DR_REGION:123456789012:targetgroup/dr-tg/1234567890" \
  --region $DR_REGION
echo "Compute scaled and healthy."

# Step 3: Update DNS to point to DR region
echo "Step 3: Updating DNS..."
aws route53 change-resource-record-sets \
  --hosted-zone-id $HOSTED_ZONE_ID \
  --change-batch '{
    "Changes": [{
      "Action": "UPSERT",
      "ResourceRecordSet": {
        "Name": "app.example.com",
        "Type": "A",
        "AliasTarget": {
          "HostedZoneId": "'$DR_ALB_HOSTED_ZONE'",
          "DNSName": "'$DR_ALB_DNS'",
          "EvaluateTargetHealth": true
        }
      }
    }]
  }'
echo "DNS updated to DR region."

# Step 4: Validate
echo "Step 4: Running health checks..."
for i in {1..5}; do
  STATUS=$(curl -s -o /dev/null -w "%{http_code}" https://app.example.com/health)
  echo "Health check $i: HTTP $STATUS"
  sleep 10
done

echo "=== FAILOVER COMPLETE ==="
echo "DR region $DR_REGION is now serving production traffic."

# Route 53 latency-based routing for active/active
Resources:
  # Health checks for each region
  USEast1HealthCheck:
    Type: AWS::Route53::HealthCheck
    Properties:
      HealthCheckConfig:
        Type: HTTPS
        FullyQualifiedDomainName: us-east-1.internal.example.com
        Port: 443
        ResourcePath: /health
        RequestInterval: 10
        FailureThreshold: 3
        EnableSNI: true

  USWest2HealthCheck:
    Type: AWS::Route53::HealthCheck
    Properties:
      HealthCheckConfig:
        Type: HTTPS
        FullyQualifiedDomainName: us-west-2.internal.example.com
        Port: 443
        ResourcePath: /health
        RequestInterval: 10
        FailureThreshold: 3
        EnableSNI: true

  # Latency-based routing records
  USEast1Record:
    Type: AWS::Route53::RecordSet
    Properties:
      HostedZoneId: !Ref HostedZone
      Name: app.example.com
      Type: A
      SetIdentifier: us-east-1
      Region: us-east-1
      HealthCheckId: !Ref USEast1HealthCheck
      AliasTarget:
        HostedZoneId: Z35SXDOTRQ7X7K
        DNSName: us-east-1-alb.example.com
        EvaluateTargetHealth: true

  USWest2Record:
    Type: AWS::Route53::RecordSet
    Properties:
      HostedZoneId: !Ref HostedZone
      Name: app.example.com
      Type: A
      SetIdentifier: us-west-2
      Region: us-west-2
      HealthCheckId: !Ref USWest2HealthCheck
      AliasTarget:
        HostedZoneId: Z1H1FL5HABSF5
        DNSName: us-west-2-alb.example.com
        EvaluateTargetHealth: true

# Create an RDS cross-region read replica
aws rds create-db-instance-read-replica \
  --db-instance-identifier dr-production-replica \
  --source-db-instance-identifier arn:aws:rds:us-east-1:123456789012:db:production-db \
  --db-instance-class db.r6g.large \
  --region us-west-2 \
  --storage-encrypted \
  --kms-key-id "arn:aws:kms:us-west-2:123456789012:key/dr-key-id"

# Create an Aurora Global Database
aws rds create-global-cluster \
  --global-cluster-identifier my-global-cluster \
  --source-db-cluster-identifier arn:aws:rds:us-east-1:123456789012:cluster:production-aurora \
  --region us-east-1

# Add a secondary region to Aurora Global Database
aws rds create-db-cluster \
  --db-cluster-identifier aurora-dr-cluster \
  --global-cluster-identifier my-global-cluster \
  --engine aurora-postgresql \
  --engine-version 15.4 \
  --region us-west-2

aws rds create-db-instance \
  --db-instance-identifier aurora-dr-instance-1 \
  --db-cluster-identifier aurora-dr-cluster \
  --db-instance-class db.r6g.large \
  --engine aurora-postgresql \
  --region us-west-2

# Enable DynamoDB Global Tables (add region)
aws dynamodb update-table \
  --table-name Orders \
  --replica-updates '[{"Create": {"RegionName": "us-west-2"}}]' \
  --region us-east-1

Resources:
  SourceBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketName: my-app-data-us-east-1
      VersioningConfiguration:
        Status: Enabled  # Required for CRR
      ReplicationConfiguration:
        Role: !GetAtt ReplicationRole.Arn
        Rules:
          - Id: ReplicateAll
            Status: Enabled
            Destination:
              Bucket: !Sub "arn:aws:s3:::my-app-data-us-west-2"
              StorageClass: STANDARD_IA  # Use cheaper storage class in DR
              ReplicationTime:
                Status: Enabled
                Time:
                  Minutes: 15  # S3 Replication Time Control (RTC)
              Metrics:
                Status: Enabled
                EventThreshold:
                  Minutes: 15
            Filter:
              Prefix: ""  # Replicate all objects
            DeleteMarkerReplication:
              Status: Enabled

# Install the DRS agent on a source server
wget -O aws-replication-installer-init   https://aws-elastic-disaster-recovery-us-west-2.s3.us-west-2.amazonaws.com/latest/linux/aws-replication-installer-init

chmod +x aws-replication-installer-init

sudo ./aws-replication-installer-init \
  --region us-west-2 \
  --aws-access-key-id AKIA... \
  --aws-secret-access-key ... \
  --no-prompt

# Initiate a recovery drill (non-destructive test)
aws drs start-recovery \
  --source-servers '[{"sourceServerID": "s-1234567890abcdef0"}]' \
  --is-drill true \
  --region us-west-2

# Monitor recovery job
aws drs describe-jobs \
  --filters '{"jobIDs": ["job-1234567890abcdef0"]}' \
  --region us-west-2

# Disaster Recovery Runbook - Production Failover

## Prerequisites
- [ ] Access to AWS Console with DR admin role
- [ ] Access to DNS management (Route 53)
- [ ] Communication channels established (Slack #incident, PagerDuty)
- [ ] DR runbook reviewed within last 90 days

## Decision Criteria
- Primary region health check failing for > 5 minutes
- AWS Health Dashboard showing regional issue
- Data corruption detected in primary region
- Approved by: VP of Engineering or on-call incident commander

## Failover Steps
1. [ ] Declare incident in Slack #incident channel
2. [ ] Verify DR region health: curl https://dr.internal.example.com/health
3. [ ] Promote RDS read replica: aws rds promote-read-replica ...
4. [ ] Wait for DB promotion: aws rds wait db-instance-available ...
5. [ ] Scale up compute: aws autoscaling update-auto-scaling-group ...
6. [ ] Wait for targets healthy: aws elbv2 wait target-in-service ...
7. [ ] Update application config to point to promoted DB
8. [ ] Switch DNS: aws route53 change-resource-record-sets ...
9. [ ] Verify application health: run smoke test suite
10. [ ] Monitor error rates for 15 minutes
11. [ ] Communicate status to stakeholders

## Failback Steps (after primary region recovery)
1. [ ] Verify primary region is healthy
2. [ ] Re-establish replication from DR to primary
3. [ ] Wait for replication to catch up
4. [ ] Switch DNS back to primary region
5. [ ] Scale down DR compute
6. [ ] Re-establish primary-to-DR replication
7. [ ] Conduct post-incident review

## Contacts
- On-call: PagerDuty rotation "Platform Engineering"
- Escalation: VP Engineering (Jane Doe)
- AWS Support: Enterprise support case (Severity 1)