Memorystore for Redis Guide

# Create a Standard tier Redis instance
gcloud redis instances create my-redis \
  --size=5 \
  --region=us-central1 \
  --zone=us-central1-a \
  --alternative-zone=us-central1-f \
  --tier=STANDARD \
  --redis-version=redis_7_0 \
  --redis-config=maxmemory-policy=allkeys-lru,notify-keyspace-events=Ex \
  --reserved-ip-range=redis-range \
  --network=projects/my-project/global/networks/my-vpc \
  --transit-encryption-mode=SERVER_AUTHENTICATION \
  --maintenance-window-day=SUNDAY \
  --maintenance-window-hour=2 \
  --labels=team=platform,env=production

# Create a Basic tier instance for development
gcloud redis instances create dev-redis \
  --size=1 \
  --region=us-central1 \
  --tier=BASIC \
  --redis-version=redis_7_0 \
  --network=projects/my-project/global/networks/default

# Get connection details
gcloud redis instances describe my-redis \
  --region=us-central1 \
  --format='table(host, port, currentLocationId, redisVersion, memorySizeGb)'

# Scale up memory (online, minimal disruption for Standard tier)
gcloud redis instances update my-redis \
  --region=us-central1 \
  --size=10

# Update Redis configuration
gcloud redis instances update my-redis \
  --region=us-central1 \
  --update-redis-config=maxmemory-policy=volatile-lru

# Add read replicas (Standard tier only)
gcloud redis instances update my-redis \
  --region=us-central1 \
  --replica-count=3

# Enable RDB persistence
gcloud redis instances update my-redis \
  --region=us-central1 \
  --persistence-mode=RDB \
  --rdb-snapshot-period=12h \
  --rdb-snapshot-start-time=2024-01-01T02:00:00Z

# Export an RDB snapshot to Cloud Storage
gcloud redis instances export gs://my-backups/redis/my-redis-backup.rdb \
  my-redis --region=us-central1

# Import an RDB snapshot
gcloud redis instances import gs://my-backups/redis/my-redis-backup.rdb \
  my-redis --region=us-central1

# List all Redis instances
gcloud redis instances list --region=us-central1

# Delete an instance
gcloud redis instances delete dev-redis --region=us-central1

import redis
import json
import hashlib
from typing import Optional, Any

# Connect to Memorystore Redis
redis_client = redis.Redis(
    host="10.0.0.3",  # Memorystore private IP
    port=6379,
    decode_responses=True,
    ssl=True,
    ssl_ca_certs="/path/to/server-ca.pem",
    socket_timeout=5,
    socket_connect_timeout=5,
    retry_on_timeout=True,
    health_check_interval=30,
)

class CacheService:
    """Redis caching service with cache-aside pattern."""

    def __init__(self, redis_client: redis.Redis, default_ttl: int = 3600):
        self.redis = redis_client
        self.default_ttl = default_ttl

    def _make_key(self, prefix: str, identifier: str) -> str:
        """Create a namespaced cache key."""
        return f"{prefix}:{identifier}"

    def get_user(self, user_id: str) -> Optional[dict]:
        """Get user with cache-aside pattern."""
        cache_key = self._make_key("user", user_id)

        # Step 1: Check cache
        cached = self.redis.get(cache_key)
        if cached:
            return json.loads(cached)

        # Step 2: Cache miss - fetch from database
        user = database.get_user(user_id)  # Your DB query
        if user is None:
            # Cache the miss to prevent cache stampede on missing data
            self.redis.setex(cache_key, 300, json.dumps(None))
            return None

        # Step 3: Store in cache with TTL
        self.redis.setex(cache_key, self.default_ttl, json.dumps(user))
        return user

    def invalidate_user(self, user_id: str):
        """Invalidate cache when user data changes."""
        cache_key = self._make_key("user", user_id)
        self.redis.delete(cache_key)

    def get_product_list(self, category: str, page: int = 1) -> list:
        """Cache paginated query results."""
        cache_key = self._make_key("products", f"{category}:page:{page}")

        cached = self.redis.get(cache_key)
        if cached:
            return json.loads(cached)

        products = database.get_products(category=category, page=page)
        self.redis.setex(cache_key, 600, json.dumps(products))
        return products

    def invalidate_product_category(self, category: str):
        """Invalidate all cached pages for a category."""
        pattern = self._make_key("products", f"{category}:*")
        cursor = 0
        while True:
            cursor, keys = self.redis.scan(cursor, match=pattern, count=100)
            if keys:
                self.redis.delete(*keys)
            if cursor == 0:
                break


# Write-through pattern for frequently updated data
class WriteThrough:
    """Write to cache and database simultaneously."""

    def update_user(self, user_id: str, data: dict):
        cache_key = f"user:{user_id}"
        pipe = redis_client.pipeline()

        # Update database first (source of truth)
        database.update_user(user_id, data)

        # Then update cache atomically
        pipe.setex(cache_key, 3600, json.dumps(data))
        pipe.execute()


# Cache warming for predictable access patterns
def warm_cache():
    """Pre-populate cache for top products on startup."""
    top_products = database.get_top_products(limit=1000)
    pipe = redis_client.pipeline()
    for product in top_products:
        key = f"product:{product['id']}"
        pipe.setex(key, 7200, json.dumps(product))
    pipe.execute()
    print(f"Warmed cache with {len(top_products)} products")

const express = require('express');
const session = require('express-session');
const RedisStore = require('connect-redis').default;
const { createClient } = require('redis');

const app = express();

// Create Redis client for Memorystore
const redisClient = createClient({
  socket: {
    host: process.env.REDIS_HOST || '10.0.0.3',
    port: parseInt(process.env.REDIS_PORT || '6379'),
    tls: process.env.REDIS_TLS === 'true',
    connectTimeout: 5000,
    reconnectStrategy: (retries) => {
      if (retries > 10) return new Error('Max reconnection attempts exceeded');
      return Math.min(retries * 100, 3000);
    },
  },
  // Memorystore AUTH password (if configured)
  password: process.env.REDIS_AUTH,
});

redisClient.on('error', (err) => console.error('Redis Client Error:', err));
redisClient.on('connect', () => console.log('Connected to Memorystore Redis'));
redisClient.on('reconnecting', () => console.log('Reconnecting to Redis...'));

(async () => {
  await redisClient.connect();
})();

// Configure session middleware with Redis store
app.use(session({
  store: new RedisStore({
    client: redisClient,
    prefix: 'sess:',
    ttl: 86400, // 24 hours in seconds
    disableTouch: false, // Refresh TTL on each request
  }),
  secret: process.env.SESSION_SECRET,
  resave: false,
  saveUninitialized: false,
  name: 'sessionId',
  cookie: {
    secure: process.env.NODE_ENV === 'production',
    httpOnly: true,
    maxAge: 24 * 60 * 60 * 1000, // 24 hours
    sameSite: 'lax',
  },
}));

// Example: Login endpoint
app.post('/login', async (req, res) => {
  const { email, password } = req.body;
  const user = await authenticateUser(email, password);

  if (user) {
    req.session.userId = user.id;
    req.session.role = user.role;
    req.session.loginTime = Date.now();
    res.json({ message: 'Login successful' });
  } else {
    res.status(401).json({ error: 'Invalid credentials' });
  }
});

// Example: Protected endpoint
app.get('/profile', (req, res) => {
  if (!req.session.userId) {
    return res.status(401).json({ error: 'Not authenticated' });
  }
  res.json({ userId: req.session.userId, role: req.session.role });
});

// Example: Logout with session destruction
app.post('/logout', (req, res) => {
  req.session.destroy((err) => {
    if (err) {
      return res.status(500).json({ error: 'Logout failed' });
    }
    res.clearCookie('sessionId');
    res.json({ message: 'Logged out' });
  });
});

// Example: Invalidate all sessions for a user
app.post('/admin/invalidate-sessions/:userId', async (req, res) => {
  const { userId } = req.params;
  // Scan for all session keys and check for matching userId
  let cursor = 0;
  let invalidated = 0;
  do {
    const result = await redisClient.scan(cursor, {
      MATCH: 'sess:*', COUNT: 100
    });
    cursor = result.cursor;
    for (const key of result.keys) {
      const sessionData = await redisClient.get(key);
      if (sessionData) {
        const parsed = JSON.parse(sessionData);
        if (parsed.userId === userId) {
          await redisClient.del(key);
          invalidated++;
        }
      }
    }
  } while (cursor !== 0);

  res.json({ invalidated });
});

app.listen(8080);

# Enable RDB persistence with 12-hour snapshots
gcloud redis instances update my-redis \
  --region=us-central1 \
  --persistence-mode=RDB \
  --rdb-snapshot-period=12h

# Enable AOF persistence (Redis 7.0+, provides near-zero data loss)
gcloud redis instances update my-redis \
  --region=us-central1 \
  --persistence-mode=AOF \
  --aof-append-fsync=EVERYSEC

# Export current state to Cloud Storage (manual backup)
gcloud redis instances export \
  gs://my-backups/redis/my-redis-$(date +%Y%m%d-%H%M%S).rdb \
  my-redis --region=us-central1

# Schedule automated exports using Cloud Scheduler + Cloud Functions
# Cloud Function to export Redis snapshot
gcloud functions deploy export-redis-backup \
  --runtime=python311 \
  --trigger-topic=redis-backup-trigger \
  --set-env-vars=REDIS_INSTANCE=my-redis,REGION=us-central1,BUCKET=my-backups

# Create a Cloud Scheduler job to trigger daily at 3 AM
gcloud scheduler jobs create pubsub redis-daily-backup \
  --schedule="0 3 * * *" \
  --topic=redis-backup-trigger \
  --message-body='{"action": "export"}' \
  --time-zone=America/New_York

# Import a backup to restore data
gcloud redis instances import \
  gs://my-backups/redis/my-redis-20240115-030000.rdb \
  my-redis --region=us-central1

# Note: Import overwrites ALL existing data in the instance

resource "google_redis_instance" "production" {
  name           = "production-cache"
  tier           = "STANDARD_HA"
  memory_size_gb = 10
  region         = "us-central1"

  location_id             = "us-central1-a"
  alternative_location_id = "us-central1-f"

  redis_version = "REDIS_7_0"

  # Network configuration
  authorized_network = data.google_compute_network.main.id
  connect_mode       = "PRIVATE_SERVICE_ACCESS"

  # Redis configuration
  redis_configs = {
    "maxmemory-policy"       = "allkeys-lru"
    "notify-keyspace-events" = "Ex"
    "activedefrag"           = "yes"
    "lfu-log-factor"         = "10"
    "lfu-decay-time"         = "1"
    "lazyfree-lazy-eviction" = "yes"
  }

  # Persistence
  persistence_config {
    persistence_mode    = "RDB"
    rdb_snapshot_period = "TWELVE_HOURS"
  }

  # In-transit encryption
  transit_encryption_mode = "SERVER_AUTHENTICATION"

  # AUTH
  auth_enabled = true

  # Read replicas
  replica_count          = 3
  read_replicas_mode     = "READ_REPLICAS_ENABLED"

  # Maintenance window
  maintenance_policy {
    weekly_maintenance_window {
      day = "SUNDAY"
      start_time {
        hours   = 2
        minutes = 0
      }
    }
  }

  labels = {
    environment = "production"
    team        = "platform"
  }
}

# Output connection details
output "redis_host" {
  value = google_redis_instance.production.host
}

output "redis_port" {
  value = google_redis_instance.production.port
}

output "redis_auth_string" {
  value     = google_redis_instance.production.auth_string
  sensitive = true
}

output "redis_read_endpoint" {
  value = google_redis_instance.production.read_endpoint
}

output "redis_server_ca_certs" {
  value     = google_redis_instance.production.server_ca_certs
  sensitive = true
}

# Step 1: Configure Private Service Access (required for Memorystore)
# Allocate an IP range for Google services
gcloud compute addresses create google-managed-services \
  --global \
  --purpose=VPC_PEERING \
  --prefix-length=16 \
  --network=my-vpc

# Create the private connection
gcloud services vpc-peerings connect \
  --service=servicenetworking.googleapis.com \
  --ranges=google-managed-services \
  --network=my-vpc

# Step 2: Create Redis instance with AUTH and TLS
gcloud redis instances create secure-redis \
  --size=5 \
  --region=us-central1 \
  --tier=STANDARD \
  --redis-version=redis_7_0 \
  --network=projects/my-project/global/networks/my-vpc \
  --connect-mode=PRIVATE_SERVICE_ACCESS \
  --transit-encryption-mode=SERVER_AUTHENTICATION \
  --auth-enabled

# Step 3: Get the AUTH password
gcloud redis instances get-auth-string secure-redis \
  --region=us-central1

# Step 4: Download the server CA certificate (for TLS verification)
gcloud redis instances describe secure-redis \
  --region=us-central1 \
  --format='value(serverCaCerts[0].cert)' > server-ca.pem

# Step 5: Connect from Cloud Run (requires Serverless VPC Access)
# Create a VPC Access connector
gcloud compute networks vpc-access connectors create redis-connector \
  --region=us-central1 \
  --network=my-vpc \
  --range=10.8.0.0/28

# Deploy Cloud Run service with VPC connector
gcloud run deploy my-api \
  --image=us-central1-docker.pkg.dev/my-project/images/my-api:latest \
  --region=us-central1 \
  --vpc-connector=redis-connector \
  --set-env-vars="REDIS_HOST=10.0.0.3,REDIS_PORT=6379,REDIS_TLS=true"

# For GKE: Ensure pods are in the same VPC as Memorystore
# No additional configuration needed if GKE cluster is in the same VPC

# IAM: Control who can manage Memorystore instances
gcloud projects add-iam-policy-binding my-project \
  --member="serviceAccount:app-sa@my-project.iam.gserviceaccount.com" \
  --role="roles/redis.viewer"

# roles/redis.admin - Full management access
# roles/redis.editor - Create, update, delete instances
# roles/redis.viewer - Read-only access to instance metadata

# Create an alert for high memory usage
gcloud beta monitoring policies create \
  --display-name="Redis Memory Usage > 80%" \
  --condition-display-name="Memory usage exceeds 80%" \
  --condition-filter='resource.type="redis_instance"
    AND metric.type="redis.googleapis.com/stats/memory/usage_ratio"' \
  --condition-threshold-value=0.8 \
  --condition-threshold-comparison=COMPARISON_GT \
  --condition-threshold-duration=300s \
  --notification-channels=projects/my-project/notificationChannels/CHANNEL_ID

# Create an alert for low cache hit ratio
gcloud beta monitoring policies create \
  --display-name="Redis Cache Hit Ratio < 80%" \
  --condition-display-name="Cache hit ratio below threshold" \
  --condition-filter='resource.type="redis_instance"
    AND metric.type="redis.googleapis.com/stats/cache_hit_ratio"' \
  --condition-threshold-value=0.8 \
  --condition-threshold-comparison=COMPARISON_LT \
  --condition-threshold-duration=600s \
  --notification-channels=projects/my-project/notificationChannels/CHANNEL_ID

# Connect to Redis CLI for debugging (from a VM in the same VPC)
redis-cli -h 10.0.0.3 -p 6379 --tls --cacert server-ca.pem -a AUTH_PASSWORD

# Useful Redis CLI commands for debugging
# INFO memory        - Memory usage details
# INFO stats         - Cache hit/miss statistics
# INFO clients       - Connected client information
# INFO replication   - Replication status
# SLOWLOG GET 10     - Last 10 slow queries
# CLIENT LIST        - All connected clients
# DBSIZE             - Total number of keys
# MEMORY DOCTOR      - Memory health diagnosis