Centralized Logging Architecture

# Stream CloudWatch Logs to S3 for centralization
aws logs put-subscription-filter \
  --log-group-name "/aws/lambda/myapp" \
  --filter-name "to-kinesis" \
  --filter-pattern "" \
  --destination-arn "arn:aws:kinesis:us-east-1:123456789012:stream/log-stream"

# Export logs to S3 (batch)
aws logs create-export-task \
  --log-group-name "/aws/lambda/myapp" \
  --from 1709251200000 \
  --to 1709337600000 \
  --destination "log-export-bucket" \
  --destination-prefix "cloudwatch-logs/"

# Query logs with CloudWatch Logs Insights
aws logs start-query \
  --log-group-name "/aws/lambda/myapp" \
  --start-time 1709251200 \
  --end-time 1709337600 \
  --query-string 'fields @timestamp, @message | filter @message like /ERROR/ | sort @timestamp desc | limit 50'

# Create a log sink to BigQuery for analysis
gcloud logging sinks create bigquery-audit-sink \
  "bigquery.googleapis.com/projects/PROJECT/datasets/audit_logs" \
  --log-filter='logName:"cloudaudit.googleapis.com"' \
  --organization=ORG_ID \
  --include-children

# Create a log sink to Pub/Sub for streaming to external SIEM
gcloud logging sinks create pubsub-all-logs \
  "pubsub.googleapis.com/projects/PROJECT/topics/all-logs" \
  --log-filter='severity >= WARNING' \
  --project=PROJECT

# Create a log sink to Cloud Storage for archival
gcloud logging sinks create storage-archive \
  "storage.googleapis.com/log-archive-bucket" \
  --log-filter='resource.type="gce_instance"' \
  --project=PROJECT

# Create a Log Analytics workspace
az monitor log-analytics workspace create \
  --workspace-name central-logs \
  --resource-group monitoring-rg \
  --location eastus \
  --retention-time 90

# Enable diagnostic settings for a resource
az monitor diagnostic-settings create \
  --name "send-to-workspace" \
  --resource "/subscriptions/SUB_ID/resourceGroups/myapp-rg/providers/Microsoft.Web/sites/myapp" \
  --workspace "/subscriptions/SUB_ID/resourceGroups/monitoring-rg/providers/Microsoft.OperationalInsights/workspaces/central-logs" \
  --logs '[{"category": "AppServiceHTTPLogs", "enabled": true}, {"category": "AppServiceConsoleLogs", "enabled": true}]' \
  --metrics '[{"category": "AllMetrics", "enabled": true}]'

# Query with KQL
az monitor log-analytics query \
  --workspace central-logs-id \
  --analytics-query "AppServiceHTTPLogs | where ScStatus >= 500 | summarize count() by bin(TimeGenerated, 1h) | order by TimeGenerated desc"

# OpenTelemetry Collector configuration for multi-cloud logs
receivers:
  # Receive logs from applications via OTLP
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

  # Collect AWS CloudWatch Logs
  awscloudwatch:
    region: us-east-1
    logs:
      poll_interval: 1m
      groups:
        named:
          /aws/lambda/myapp:
          /aws/ecs/myapp:

  # Collect from files (for VMs)
  filelog:
    include:
      - /var/log/app/*.log
    operators:
      - type: json_parser
        timestamp:
          parse_from: attributes.timestamp
          layout: '%Y-%m-%dT%H:%M:%SZ'

processors:
  # Add cloud provider metadata
  attributes:
    actions:
      - key: cloud.provider
        value: "multi-cloud"
        action: upsert

  # Batch for efficiency
  batch:
    send_batch_size: 10000
    timeout: 10s

  # Filter out noisy logs
  filter:
    logs:
      exclude:
        match_type: regexp
        bodies:
          - "health check"
          - "readiness probe"

exporters:
  # Send to Elasticsearch
  elasticsearch:
    endpoints: ["https://elasticsearch.internal:9200"]
    logs_index: "cloud-logs"

  # Send to Grafana Loki
  loki:
    endpoint: "https://loki.internal:3100/loki/api/v1/push"
    labels:
      attributes:
        cloud.provider:
        service.name:

service:
  pipelines:
    logs:
      receivers: [otlp, awscloudwatch, filelog]
      processors: [attributes, batch, filter]
      exporters: [elasticsearch, loki]

{
  "timestamp": "2026-03-14T10:30:00.000Z",
  "level": "INFO",
  "service": "order-api",
  "cloud": "aws",
  "region": "us-east-1",
  "trace_id": "abc123def456",
  "span_id": "span-789",
  "request_id": "req-abc-123",
  "message": "Order created successfully",
  "order_id": "ord-456",
  "customer_id": "cust-789",
  "duration_ms": 142,
  "http_status": 201
}

# AWS: Set CloudWatch log retention to reduce costs
aws logs put-retention-policy \
  --log-group-name "/aws/lambda/myapp" \
  --retention-in-days 14

# GCP: Exclude debug logs from ingestion (save money)
gcloud logging sinks create exclude-debug \
  "logging.googleapis.com/projects/PROJECT/locations/global/buckets/_Default" \
  --log-filter='NOT severity="DEBUG"' \
  --exclusion='name=exclude-debug,filter=severity="DEBUG"'

# GCP: Route logs to cheaper bucket with different retention
gcloud logging buckets create cold-logs \
  --location=global \
  --retention-days=365 \
  --project=PROJECT

gcloud logging sinks create route-to-cold \
  "logging.googleapis.com/projects/PROJECT/locations/global/buckets/cold-logs" \
  --log-filter='severity <= "INFO"' \
  --project=PROJECT