1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# Kubernetes resource utilization
kubectl top pods -n production --sort-by=cpu
kubectl top nodes

# For each service, check utilization vs requests/limits
kubectl get pods -n production -o json | jq -r '
  .items[] |
  .metadata.name + " | CPU req: " +
  (.spec.containers[0].resources.requests.cpu // "none") + " | Mem req: " +
  (.spec.containers[0].resources.requests.memory // "none")
'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# CPU utilization per service (as % of requested)
sum(rate(container_cpu_usage_seconds_total{namespace="production"}[5m])) by (pod)
/
sum(kube_pod_container_resource_requests{namespace="production", resource="cpu"}) by (pod)

# Memory utilization per service
sum(container_memory_working_set_bytes{namespace="production"}) by (pod)
/
sum(kube_pod_container_resource_requests{namespace="production", resource="memory"}) by (pod)

# Database connection pool utilization
# (this metric name varies by driver — pg-specific example)
db_pool_connections_in_use / db_pool_connections_max

# Current RPS per service
sum(rate(http_requests_total{namespace="production"}[5m])) by (service)

1
2
3

brew install k6
# or
docker pull grafana/k6

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

// breaking-point-test.js
import http from 'k6/http';
import { check, sleep } from 'k6';
import { Rate, Trend } from 'k6/metrics';

const errorRate = new Rate('errors');
const checkoutDuration = new Trend('checkout_duration_ms', true);

export const options = {
  // Ramp up until something breaks
  stages: [
    { duration: '2m', target: 100 },   // Warm up
    { duration: '3m', target: 200 },
    { duration: '3m', target: 400 },
    { duration: '3m', target: 600 },
    { duration: '3m', target: 800 },
    { duration: '3m', target: 1000 },  // Keep going until SLO breaks
    { duration: '3m', target: 1200 },
    { duration: '2m', target: 0 },     // Cool down
  ],
  thresholds: {
    // Test "passes" but we observe where it starts degrading
    'http_req_duration': ['p(99)<2000'],
    'errors': ['rate<0.05'],
  },
};

const BASE_URL = __ENV.BASE_URL || 'https://api.example.com';

export default function () {
  const start = Date.now();

  // Simulate a checkout flow
  const cartRes = http.post(
    `${BASE_URL}/api/cart/add`,
    JSON.stringify({ product_id: 'test-product-1', quantity: 1 }),
    { headers: { 'Content-Type': 'application/json',
                 'Authorization': `Bearer ${__ENV.TEST_TOKEN}` } }
  );

  const cartOk = check(cartRes, {
    'cart add: 200': r => r.status === 200,
    'cart add: < 500ms': r => r.timings.duration < 500,
  });

  if (!cartOk) {
    errorRate.add(1);
    return;
  }

  sleep(0.5);

  const checkoutRes = http.post(
    `${BASE_URL}/api/checkout`,
    JSON.stringify({ cart_id: cartRes.json('cart_id'), payment: 'test_visa' }),
    { headers: { 'Content-Type': 'application/json',
                 'Authorization': `Bearer ${__ENV.TEST_TOKEN}` } }
  );

  const checkoutOk = check(checkoutRes, {
    'checkout: 200': r => r.status === 200,
    'checkout: has order_id': r => r.json('order_id') !== undefined,
  });

  const duration = Date.now() - start;
  checkoutDuration.add(duration);
  errorRate.add(!checkoutOk ? 1 : 0);

  sleep(Math.random() * 2 + 0.5);
}

1
2
3
4
5
6

# Run the breaking-point test
k6 run \
  -e BASE_URL=https://staging.example.com \
  -e TEST_TOKEN=$TEST_API_TOKEN \
  --out json=results/breaking-point-$(date +%Y%m%d).json \
  breaking-point-test.js

1
2
3
4
5

# Parse results to find the RPS at which P99 exceeded 500ms
cat results/breaking-point-*.json | \
  jq 'select(.type=="Point" and .metric=="http_req_duration") |
      {time: .data.time, value: .data.value}' | \
  # Plot or analyze to find the inflection point

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

// soak-test.js — run at 70% of identified breaking point for 2 hours
export const options = {
  stages: [
    { duration: '5m',  target: 700 },  // Ramp to 70% of limit (~950 RPS)
    { duration: '2h',  target: 700 },  // Hold for 2 hours
    { duration: '5m',  target: 0 },
  ],
  thresholds: {
    'http_req_duration': ['p(99)<500'],
    'errors': ['rate<0.01'],
  },
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

// spike-test.js
export const options = {
  stages: [
    { duration: '1m',  target: 100 },   // Normal baseline
    { duration: '10s', target: 1000 },  // Sudden 10x spike
    { duration: '3m',  target: 1000 },  // Hold the spike
    { duration: '10s', target: 100 },   // Back to baseline
    { duration: '2m',  target: 100 },   // Recovery validation
  ],
};

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

# k6-configmap.yaml — store test scripts in a ConfigMap
apiVersion: v1
kind: ConfigMap
metadata:
  name: k6-tests
  namespace: loadtesting
data:
  breaking-point.js: |
    // test script here

---
# k6-job.yaml — run k6 as a Job with results streamed to InfluxDB/Prometheus
apiVersion: batch/v1
kind: Job
metadata:
  name: k6-breaking-point-test
  namespace: loadtesting
spec:
  template:
    spec:
      restartPolicy: Never
      containers:
        - name: k6
          image: grafana/k6:latest
          args:
            - run
            - --out
            - experimental-prometheus-rw=http://prometheus-pushgateway:9091
            - /scripts/breaking-point.js
          env:
            - name: BASE_URL
              value: "https://staging.example.com"
            - name: TEST_TOKEN
              valueFrom:
                secretKeyRef:
                  name: load-test-secrets
                  key: token
            - name: K6_PROMETHEUS_RW_SERVER_URL
              value: "http://prometheus:9090/api/v1/write"
          volumeMounts:
            - name: scripts
              mountPath: /scripts
      volumes:
        - name: scripts
          configMap:
            name: k6-tests

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109

#!/usr/bin/env python3
"""
capacity_forecast.py — forecast when you'll hit capacity limits
"""
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
import requests
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures

PROMETHEUS_URL = "http://prometheus:9090"
BREAKING_POINT_RPS = 1350  # From load test
HEADROOM_FACTOR = 0.70     # Provision capacity before hitting 70% of limit

def fetch_metric_history(query: str, days: int = 90) -> pd.DataFrame:
    """Fetch time series from Prometheus."""
    end = datetime.now()
    start = end - timedelta(days=days)

    resp = requests.get(f"{PROMETHEUS_URL}/api/v1/query_range", params={
        "query": query,
        "start": start.timestamp(),
        "end": end.timestamp(),
        "step": "1h",
    })
    resp.raise_for_status()

    results = resp.json()["data"]["result"]
    if not results:
        return pd.DataFrame()

    values = results[0]["values"]
    df = pd.DataFrame(values, columns=["timestamp", "value"])
    df["timestamp"] = pd.to_datetime(df["timestamp"], unit="s")
    df["value"] = df["value"].astype(float)
    return df


def fit_and_forecast(df: pd.DataFrame, days_ahead: int = 180) -> dict:
    """Fit linear and polynomial models, return forecast."""
    # Convert timestamps to numeric (days since start)
    df["day"] = (df["timestamp"] - df["timestamp"].min()).dt.total_seconds() / 86400

    X = df["day"].values.reshape(-1, 1)
    y = df["value"].values

    # Linear model
    linear = LinearRegression().fit(X, y)

    # Polynomial (degree 2) for accelerating growth
    poly = PolynomialFeatures(degree=2)
    X_poly = poly.fit_transform(X)
    poly_model = LinearRegression().fit(X_poly, y)

    # Forecast
    current_day = df["day"].max()
    future_days = np.arange(current_day, current_day + days_ahead, 1).reshape(-1, 1)

    linear_forecast = linear.predict(future_days)
    poly_forecast = poly_model.predict(poly.transform(future_days))

    # Find when each model hits the headroom threshold
    threshold = BREAKING_POINT_RPS * HEADROOM_FACTOR

    linear_breach = next(
        (current_day + i for i, v in enumerate(linear_forecast) if v >= threshold),
        None
    )
    poly_breach = next(
        (current_day + i for i, v in enumerate(poly_forecast) if v >= threshold),
        None
    )

    today = df["timestamp"].max()

    return {
        "current_rps": float(df["value"].iloc[-1]),
        "growth_rate_daily": float(linear.coef_[0]),
        "growth_rate_pct_weekly": float(linear.coef_[0] * 7 / df["value"].mean() * 100),
        "threshold_rps": threshold,
        "linear_breach_date": (today + timedelta(days=linear_breach - current_day)).date()
            if linear_breach else "Beyond forecast window",
        "poly_breach_date": (today + timedelta(days=poly_breach - current_day)).date()
            if poly_breach else "Beyond forecast window",
    }


# Forecast for checkout service
df = fetch_metric_history(
    'sum(rate(http_requests_total{service="checkout-service"}[1h]))'
)
forecast = fit_and_forecast(df)

print(f"""
Checkout Service Capacity Forecast
====================================
Current RPS:          {forecast['current_rps']:.0f}
Breaking point (load test): {BREAKING_POINT_RPS} RPS
Action threshold (70%):     {forecast['threshold_rps']:.0f} RPS

Growth rate:          {forecast['growth_rate_daily']:.1f} RPS/day
                      ({forecast['growth_rate_pct_weekly']:.1f}% per week)

Linear model breach:  {forecast['linear_breach_date']}
Polynomial breach:    {forecast['poly_breach_date']}

Recommendation: Provision additional capacity before {forecast['linear_breach_date']}
""")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

# Upcoming events that affect capacity
EVENTS = [
    {
        "name": "Black Friday",
        "date": "2026-11-27",
        "expected_multiplier": 4.5,  # 4.5x normal peak traffic
        "duration_hours": 24,
    },
    {
        "name": "Product Launch v3.0",
        "date": "2026-05-15",
        "expected_multiplier": 2.0,
        "duration_hours": 6,
    },
    {
        "name": "Marketing Campaign Q2",
        "date": "2026-04-01",
        "expected_multiplier": 1.8,
        "duration_hours": 168,  # 1 week
    },
]

# For Black Friday at 4.5x normal:
# Current peak: 650 RPS
# BF peak estimate: 650 * 4.5 = 2,925 RPS
# Breaking point: 1,350 RPS
# NEED: 2,925 / 1,350 = 2.2x current capacity minimum before BF

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

def analyze_seasonality(df: pd.DataFrame) -> dict:
    """Identify weekly and daily peak patterns."""
    df["hour"] = df["timestamp"].dt.hour
    df["day_of_week"] = df["timestamp"].dt.dayofweek  # 0=Monday

    # Find peak hour and day
    hourly_avg = df.groupby("hour")["value"].mean()
    daily_avg = df.groupby("day_of_week")["value"].mean()

    peak_hour = hourly_avg.idxmax()
    peak_day = daily_avg.idxmax()

    # Peak-to-average ratio
    peak_ratio = hourly_avg.max() / hourly_avg.mean()

    days = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]

    return {
        "peak_hour_utc": peak_hour,
        "peak_day": days[peak_day],
        "peak_to_average_ratio": peak_ratio,
        "effective_peak_rps": df["value"].quantile(0.99),  # 99th percentile
    }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: checkout-service
  namespace: production
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: checkout-service
  minReplicas: 3
  maxReplicas: 50

  metrics:
    # Scale on CPU (most common)
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 60  # Scale up when CPU > 60% (not 80%!)

    # Scale on custom metric: RPS per pod
    - type: Pods
      pods:
        metric:
          name: http_requests_per_second
        target:
          type: AverageValue
          averageValue: "200"  # Target 200 RPS/pod; scale when exceeded

  behavior:
    scaleUp:
      stabilizationWindowSeconds: 0      # Scale up immediately — no delay
      policies:
        - type: Percent
          value: 100                     # Double pods at a time if needed
          periodSeconds: 15
        - type: Pods
          value: 4                       # Or add 4 pods every 15 seconds
          periodSeconds: 15
      selectPolicy: Max                  # Use whichever adds more pods

    scaleDown:
      stabilizationWindowSeconds: 300    # Wait 5 minutes before scaling down
      policies:
        - type: Percent
          value: 10                      # Remove at most 10% of pods per minute
          periodSeconds: 60

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
  name: checkout-service-vpa
  namespace: production
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: checkout-service
  updatePolicy:
    updateMode: "Off"    # "Off" = recommendations only (don't auto-apply in prod)
                         # "Auto" = applies changes (use in staging first)
  resourcePolicy:
    containerPolicies:
      - containerName: checkout-service
        minAllowed:
          cpu: 100m
          memory: 128Mi
        maxAllowed:
          cpu: 4
          memory: 4Gi

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# Check VPA recommendations (in "Off" mode)
kubectl describe vpa checkout-service-vpa -n production

# Output shows:
# Recommendation:
#   Container Recommendations:
#     Container Name: checkout-service
#     Lower Bound:    cpu: 250m, memory: 512Mi
#     Target:         cpu: 620m, memory: 890Mi   ← Apply this
#     Upper Bound:    cpu: 1200m, memory: 1800Mi
#
# If you're currently requesting 200m CPU, you're under-resourced.
# If you're requesting 2 CPU, you're wasting $$.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

# Scale checkout workers based on order queue depth
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: order-processor-scaler
  namespace: production
spec:
  scaleTargetRef:
    name: order-processor
  minReplicaCount: 2
  maxReplicaCount: 100

  triggers:
    # Scale based on SQS queue depth
    - type: aws-sqs-queue
      metadata:
        queueURL: https://sqs.us-east-1.amazonaws.com/123456789/order-queue
        queueLength: "50"    # Target: 50 messages per pod
        awsRegion: us-east-1

    # Also scale on Kafka consumer lag
    - type: kafka
      metadata:
        bootstrapServers: kafka:9092
        consumerGroup: order-processors
        topic: orders
        lagThreshold: "100"  # Scale when consumer lag > 100 messages/pod

  advanced:
    restoreToOriginalReplicaCount: true
    horizontalPodAutoscalerConfig:
      behavior:
        scaleDown:
          stabilizationWindowSeconds: 180

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# cluster-autoscaler Helm values
autoDiscovery:
  clusterName: production-cluster

# Scale up aggressively, scale down conservatively
extraArgs:
  scale-down-delay-after-add: 10m
  scale-down-unneeded-time: 10m
  scale-down-utilization-threshold: "0.5"  # Scale down nodes at < 50% utilization
  max-node-provision-time: 15m
  skip-nodes-with-local-storage: "false"
  balance-similar-node-groups: "true"
  expander: least-waste  # Choose node group that wastes least resources

# Node groups — use spot/preemptible for worker nodes, on-demand for critical
nodeGroups:
  - name: on-demand-general
    minSize: 3
    maxSize: 20
  - name: spot-workers
    minSize: 0
    maxSize: 100

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# Pre-scale 2 hours before a known traffic event
# Add to your deployment runbook for major launches

# Scale up manually
kubectl scale deployment checkout-service --replicas=20 -n production
kubectl scale deployment payment-service --replicas=10 -n production

# Update HPA min replicas temporarily
kubectl patch hpa checkout-service -n production \
  --patch '{"spec":{"minReplicas":15}}'

# After event, restore normal minimums
kubectl patch hpa checkout-service -n production \
  --patch '{"spec":{"minReplicas":3}}'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

-- Connection utilization
SELECT
  count(*) as active_connections,
  (SELECT setting::int FROM pg_settings WHERE name = 'max_connections') as max_connections,
  round(count(*) * 100.0 /
    (SELECT setting::int FROM pg_settings WHERE name = 'max_connections'), 1) as pct_used
FROM pg_stat_activity
WHERE state != 'idle';

-- Table bloat (dead tuples accumulating → I/O waste)
SELECT
  schemaname || '.' || tablename as table,
  n_dead_tup,
  n_live_tup,
  round(n_dead_tup * 100.0 / NULLIF(n_live_tup + n_dead_tup, 0), 1) as dead_pct
FROM pg_stat_user_tables
WHERE n_dead_tup > 10000
ORDER BY n_dead_tup DESC;

-- Slow queries (> 1 second)
SELECT
  query,
  calls,
  round(mean_exec_time::numeric, 2) as avg_ms,
  round(total_exec_time::numeric, 2) as total_ms
FROM pg_stat_statements
WHERE mean_exec_time > 1000
ORDER BY total_exec_time DESC
LIMIT 20;

-- Index hit rate (should be > 99%)
SELECT
  sum(idx_blks_hit) / nullif(sum(idx_blks_hit) + sum(idx_blks_read), 0) * 100
  as index_cache_hit_rate,
  sum(heap_blks_hit) / nullif(sum(heap_blks_hit) + sum(heap_blks_read), 0) * 100
  as heap_cache_hit_rate
FROM pg_statio_user_tables;

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# pgbouncer.ini
[databases]
production = host=postgres-primary port=5432 dbname=myapp

[pgbouncer]
listen_port = 5432
listen_addr = 0.0.0.0
auth_type = scram-sha-256
auth_file = /etc/pgbouncer/userlist.txt

# Transaction pooling: most aggressive, highest compatibility issues
# Session pooling: safest, but less efficient
pool_mode = transaction

# Server connections (to actual PostgreSQL)
max_client_conn = 1000    # App connections to PgBouncer
default_pool_size = 25    # Actual PostgreSQL connections per database
min_pool_size = 5
reserve_pool_size = 5
reserve_pool_timeout = 3

# Timeouts
server_connect_timeout = 5
query_timeout = 30
client_idle_timeout = 300

# Stats
stats_period = 60

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

# Python — route reads to replicas, writes to primary
import psycopg2
from contextlib import contextmanager
import random

PRIMARY_DSN = "postgresql://user:pass@postgres-primary:5432/myapp"
REPLICA_DSNS = [
    "postgresql://user:pass@postgres-replica-1:5432/myapp",
    "postgresql://user:pass@postgres-replica-2:5432/myapp",
]

@contextmanager
def get_read_connection():
    """Use a random replica for reads."""
    dsn = random.choice(REPLICA_DSNS)
    conn = psycopg2.connect(dsn)
    try:
        yield conn
    finally:
        conn.close()

@contextmanager
def get_write_connection():
    """Always use primary for writes."""
    conn = psycopg2.connect(PRIMARY_DSN)
    try:
        yield conn
    finally:
        conn.close()

# Usage
def get_product(product_id: str):
    with get_read_connection() as conn:  # Replica
        cur = conn.cursor()
        cur.execute("SELECT * FROM products WHERE id = %s", (product_id,))
        return cur.fetchone()

def update_inventory(product_id: str, quantity: int):
    with get_write_connection() as conn:  # Primary
        cur = conn.cursor()
        cur.execute(
            "UPDATE products SET stock = stock - %s WHERE id = %s",
            (quantity, product_id)
        )
        conn.commit()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

## Monthly Capacity Review — March 2026

### Traffic growth (last 30 days)
| Service          | March Avg RPS | Feb Avg RPS | Growth |
|------------------|---------------|-------------|--------|
| checkout-service | 480           | 450         | +6.7%  |
| payment-service  | 96            | 90          | +6.7%  |
| inventory-service| 555           | 520         | +6.7%  |

### Current headroom vs targets
| Service          | Current Util | Target Max | Headroom | Status |
|------------------|-------------|------------|----------|--------|
| checkout-service | 65% CPU     | 70%        | 5%       | ⚠️ Low  |
| payment-service  | 40% CPU     | 70%        | 30%      | ✅ OK   |
| inventory DB conn| 92%         | 70%        | -22%     | 🔴 Over |
| postgres-primary | 74% CPU     | 60%        | -14%     | 🔴 Over |

### Forecast: when do we hit 70% threshold?
| Service          | At current growth | Adjusted (incl. May launch) |
|------------------|-------------------|-----------------------------|
| checkout-service | June 15           | April 28                    |
| postgres-primary | Already over      | Already over                |

### Action items
1. Add PgBouncer to reduce DB connections immediately (Owner: @dbteam, Due: Apr 5)
2. Add read replica for inventory reads (Owner: @dbteam, Due: Apr 15)
3. Upgrade checkout-service pod resources (1 → 2 CPU) before May launch (Owner: @platteam, Due: Apr 30)
4. Run load test against staging after resource changes (Owner: @sre, Due: May 1)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

## Pre-Launch Capacity Checklist — v3.0 Launch (May 15)

### Traffic estimation
- [ ] Expected peak RPS documented (with source/confidence)
- [ ] Expected multiplier vs current baseline: ___x
- [ ] Duration of peak: ___ hours

### Load testing
- [ ] Breaking-point test run on staging with new code
- [ ] Spike test run (simulating launch day traffic profile)
- [ ] Soak test run at expected peak for 2+ hours
- [ ] Results reviewed; no regressions vs previous tests

### Infrastructure readiness
- [ ] HPA min replicas pre-scaled to handle expected peak
- [ ] DB connection pools validated at expected connection count
- [ ] CDN cache rules reviewed for new endpoints
- [ ] External API rate limits confirmed sufficient
- [ ] Rollback plan documented and tested

### Monitoring and alerting
- [ ] Alert thresholds reviewed (not too tight for expected traffic increase)
- [ ] Dashboards updated for new services/endpoints
- [ ] On-call engineer briefed on launch and expected traffic
- [ ] Runbooks updated

### Communication
- [ ] Engineering team notified of launch date
- [ ] Customer support briefed on new features
- [ ] Status page incident template prepared

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# Find pods with low utilization vs their requests (candidates for downsizing)
kubectl top pods -n production --no-headers | \
  awk '{print $1, $2, $3}' | \
  while read pod cpu mem; do
    requested_cpu=$(kubectl get pod $pod -n production -o jsonpath='{.spec.containers[0].resources.requests.cpu}')
    echo "$pod: using $cpu, requested $requested_cpu"
  done

# Or use Goldilocks (VPA recommendation viewer)
helm install goldilocks fairwinds-stable/goldilocks --namespace goldilocks
kubectl label namespace production goldilocks.fairwinds.com/enabled=true
kubectl port-forward -n goldilocks svc/goldilocks-dashboard 8080:80
# Open http://localhost:8080 — see VPA recommendations for all workloads

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# Node affinity: prefer spot, fall back to on-demand
affinity:
  nodeAffinity:
    preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 80
        preference:
          matchExpressions:
            - key: node.kubernetes.io/capacity-type
              operator: In
              values: [spot]
      - weight: 20
        preference:
          matchExpressions:
            - key: node.kubernetes.io/capacity-type
              operator: In
              values: [on-demand]

# Tolerate spot node interruption
tolerations:
  - key: "node.kubernetes.io/spot"
    operator: "Exists"
    effect: "NoSchedule"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# Ensure at least 2 pods are always available during node drains/scaling
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: checkout-service-pdb
  namespace: production
spec:
  minAvailable: 2
  selector:
    matchLabels:
      app: checkout-service