Designing for Observability: Building Applications You Can Actually Debug

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

{
  "timestamp": "2026-03-27T14:23:01.234Z",
  "level": "error",
  "message": "order processing failed",
  "order_id": "1234",
  "user_id": "5678",
  "error": "context deadline exceeded",
  "duration_ms": 3021,
  "service": "order-service",
  "version": "2.4.1",
  "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736",
  "span_id": "00f067aa0ba902b7"
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

// pkg/logger/logger.go
package logger

import (
    "context"
    "os"

    "go.uber.org/zap"
    "go.uber.org/zap/zapcore"
)

// contextKey is unexported to prevent collisions with other packages
type contextKey struct{}

var global *zap.Logger

func Init(serviceName, version, environment string) {
    encoderCfg := zap.NewProductionEncoderConfig()
    encoderCfg.TimeKey = "timestamp"
    encoderCfg.EncodeTime = zapcore.ISO8601TimeEncoder
    encoderCfg.MessageKey = "message"

    config := zap.Config{
        Level:            zap.NewAtomicLevelAt(zap.InfoLevel),
        Development:      false,
        Encoding:         "json",
        EncoderConfig:    encoderCfg,
        OutputPaths:      []string{"stdout"},
        ErrorOutputPaths: []string{"stderr"},
    }

    if environment == "development" {
        config.Encoding = "console"
        config.Development = true
        config.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
    }

    logger, _ := config.Build()

    // Always-present fields on every log entry from this service
    global = logger.With(
        zap.String("service", serviceName),
        zap.String("version", version),
        zap.String("env", environment),
        zap.Int("pid", os.Getpid()),
    )
}

// WithContext extracts the trace/span IDs from context and returns
// a logger that will include them on every entry.
func WithContext(ctx context.Context) *zap.Logger {
    log := global
    if traceID := TraceIDFromContext(ctx); traceID != "" {
        log = log.With(
            zap.String("trace_id", traceID),
            zap.String("span_id", SpanIDFromContext(ctx)),
        )
    }
    if userID := UserIDFromContext(ctx); userID != "" {
        log = log.With(zap.String("user_id", userID))
    }
    return log
}

// FromContext returns a sugared logger with context fields
func FromContext(ctx context.Context) *zap.SugaredLogger {
    return WithContext(ctx).Sugar()
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

func (h *OrderHandler) CreateOrder(ctx context.Context, req *CreateOrderRequest) (*Order, error) {
    log := logger.WithContext(ctx).With(
        zap.String("order_id", req.OrderID),
        zap.String("customer_id", req.CustomerID),
        zap.Float64("amount", req.Amount),
    )

    log.Info("creating order")

    order, err := h.store.Create(ctx, req)
    if err != nil {
        // Log the error with all context — no need to repeat it up the call stack
        log.Error("failed to create order",
            zap.Error(err),
            zap.Duration("duration", time.Since(start)),
        )
        return nil, fmt.Errorf("creating order: %w", err)
    }

    log.Info("order created",
        zap.String("order_id", order.ID),
        zap.Duration("duration", time.Since(start)),
    )
    return order, nil
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

# app/logging_config.py
import logging
import structlog

def configure_logging(service: str, version: str, environment: str):
    shared_processors = [
        structlog.contextvars.merge_contextvars,      # merge thread-local context
        structlog.processors.add_log_level,
        structlog.processors.TimeStamper(fmt="iso"),
        structlog.stdlib.add_logger_name,
    ]

    if environment == "development":
        # Human-readable in dev
        processors = shared_processors + [
            structlog.dev.ConsoleRenderer(colors=True)
        ]
    else:
        # JSON in production
        processors = shared_processors + [
            structlog.processors.dict_tracebacks,
            structlog.processors.JSONRenderer(),
        ]

    structlog.configure(
        processors=processors,
        wrapper_class=structlog.make_filtering_bound_logger(logging.INFO),
        context_class=dict,
        logger_factory=structlog.PrintLoggerFactory(),
        cache_logger_on_first_use=True,
    )

    # Bind service-wide fields
    structlog.contextvars.bind_contextvars(
        service=service,
        version=version,
        env=environment,
    )

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

# In a FastAPI middleware — bind request context to all logs in the request
@app.middleware("http")
async def logging_middleware(request: Request, call_next):
    # Generate or extract trace ID
    trace_id = request.headers.get("X-Trace-ID") or str(uuid.uuid4())

    structlog.contextvars.clear_contextvars()
    structlog.contextvars.bind_contextvars(
        trace_id=trace_id,
        path=request.url.path,
        method=request.method,
    )

    start = time.monotonic()
    response = await call_next(request)
    duration_ms = (time.monotonic() - start) * 1000

    log = structlog.get_logger()
    log.info(
        "request completed",
        status_code=response.status_code,
        duration_ms=round(duration_ms, 2),
    )

    response.headers["X-Trace-ID"] = trace_id
    return response

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

## Log Level Guidelines

**DEBUG** — Diagnostic information for developers. Never enabled in production
by default. Safe to enable per-service for short periods during incidents.
  Examples: SQL query text, full request/response bodies, cache hit/miss detail

**INFO** — Normal operational events worth recording. Should be indexable
and searchable. Volume should be manageable (not per-row database operations).
  Examples: request completed, order created, user logged in, job started/finished

**WARN** — Something unexpected that the system handled gracefully. Worth
alerting if the rate increases significantly.
  Examples: retried operation succeeded, using fallback cache, deprecated API called,
  config value missing (using default)

**ERROR** — A request or operation failed. Always actionable. Should generate
an alert if sustained. Include full error context.
  Examples: database query failed, external API timeout, payment processing error,
  validation failed (unexpected — expected validation errors go at INFO or WARN)

**FATAL** — The process cannot continue. Logs and exits immediately.
  Examples: required config missing, cannot bind to port, database migration failed

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

// middleware/tracing.go
package middleware

import (
    "context"
    "net/http"

    "github.com/google/uuid"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/propagation"
    "go.opentelemetry.io/otel/trace"
)

const (
    TraceIDHeader = "X-Trace-ID"
    RequestIDHeader = "X-Request-ID"
)

// TraceMiddleware extracts W3C trace context from incoming requests
// (if present, e.g. from a load balancer or upstream service) or
// generates a new trace ID for edge requests.
func TraceMiddleware(next http.Handler) http.Handler {
    propagator := otel.GetTextMapPropagator()
    tracer := otel.Tracer("http-server")

    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        // Extract existing trace context (W3C traceparent header)
        ctx := propagator.Extract(r.Context(), propagation.HeaderCarrier(r.Header))

        // Start a span for this request
        ctx, span := tracer.Start(ctx, r.Method+" "+r.URL.Path)
        defer span.End()

        // Make trace/span IDs available to loggers downstream
        spanCtx := trace.SpanFromContext(ctx).SpanContext()
        ctx = context.WithValue(ctx, traceIDKey{}, spanCtx.TraceID().String())
        ctx = context.WithValue(ctx, spanIDKey{}, spanCtx.SpanID().String())

        // Propagate trace context to outgoing requests
        w.Header().Set(TraceIDHeader, spanCtx.TraceID().String())

        // Inject into response so clients can correlate
        propagator.Inject(ctx, propagation.HeaderCarrier(w.Header()))

        next.ServeHTTP(w, r.WithContext(ctx))
    })
}

// When making outbound HTTP calls, always propagate trace context:
func NewTracedHTTPClient() *http.Client {
    return &http.Client{
        Transport: &tracingTransport{base: http.DefaultTransport},
    }
}

type tracingTransport struct {
    base http.RoundTripper
}

func (t *tracingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
    propagator := otel.GetTextMapPropagator()
    propagator.Inject(req.Context(), propagation.HeaderCarrier(req.Header))
    return t.base.RoundTrip(req)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

// metrics/http.go
package metrics

import (
    "net/http"
    "strconv"
    "time"

    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promauto"
)

var (
    httpRequestsTotal = promauto.NewCounterVec(
        prometheus.CounterOpts{
            Name: "http_requests_total",
            Help: "Total number of HTTP requests",
        },
        []string{"method", "path", "status"},
    )

    httpRequestDuration = promauto.NewHistogramVec(
        prometheus.HistogramOpts{
            Name: "http_request_duration_seconds",
            Help: "HTTP request duration in seconds",
            // Buckets tuned for a web service — adjust for your SLOs
            Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5},
        },
        []string{"method", "path", "status"},
    )

    httpRequestsInFlight = promauto.NewGauge(
        prometheus.GaugeOpts{
            Name: "http_requests_in_flight",
            Help: "Current number of HTTP requests being processed",
        },
    )
)

// MetricsMiddleware instruments every HTTP handler with RED metrics.
func MetricsMiddleware(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        // Normalize path to avoid high cardinality from IDs
        // /api/users/12345 → /api/users/:id
        path := normalizePath(r.URL.Path)

        httpRequestsInFlight.Inc()
        defer httpRequestsInFlight.Dec()

        start := time.Now()
        rw := &responseWriter{ResponseWriter: w, statusCode: http.StatusOK}
        next.ServeHTTP(rw, r)
        duration := time.Since(start)

        status := strconv.Itoa(rw.statusCode)
        httpRequestsTotal.WithLabelValues(r.Method, path, status).Inc()
        httpRequestDuration.WithLabelValues(r.Method, path, status).Observe(duration.Seconds())
    })
}

type responseWriter struct {
    http.ResponseWriter
    statusCode int
    written    bool
}

func (rw *responseWriter) WriteHeader(code int) {
    if !rw.written {
        rw.statusCode = code
        rw.written = true
    }
    rw.ResponseWriter.WriteHeader(code)
}

// normalizePath replaces path parameters with placeholders to avoid
// cardinality explosion (a unique label per user ID would create millions
// of time series).
func normalizePath(path string) string {
    // Implement using your router's route matching, or a simple regex:
    // /api/users/123      → /api/users/:id
    // /api/orders/abc-xyz → /api/orders/:id
    return path // stub — replace with router-aware normalization
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

var (
    // Domain-specific counters
    ordersCreated = promauto.NewCounterVec(
        prometheus.CounterOpts{
            Name: "orders_created_total",
            Help: "Total orders created",
        },
        []string{"payment_method", "region"},
    )

    orderRevenue = promauto.NewHistogram(
        prometheus.HistogramOpts{
            Name:    "order_revenue_usd",
            Help:    "Order revenue in USD",
            Buckets: prometheus.ExponentialBuckets(1, 2, 15), // $1 to $16k
        },
    )

    // Queue depths — critical for detecting processing lag
    jobQueueDepth = promauto.NewGaugeVec(
        prometheus.GaugeOpts{
            Name: "job_queue_depth",
            Help: "Number of jobs waiting to be processed",
        },
        []string{"queue_name"},
    )

    // External dependency health
    externalAPIErrors = promauto.NewCounterVec(
        prometheus.CounterOpts{
            Name: "external_api_errors_total",
            Help: "Errors from external API calls",
        },
        []string{"api", "error_type"},
    )

    externalAPILatency = promauto.NewHistogramVec(
        prometheus.HistogramOpts{
            Name:    "external_api_duration_seconds",
            Help:    "Latency of external API calls",
            Buckets: prometheus.DefBuckets,
        },
        []string{"api", "method"},
    )
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

# prometheus/recording_rules.yml
groups:
  - name: http_derived
    interval: 30s
    rules:
      # Error rate per endpoint
      - record: job:http_request_errors:rate5m
        expr: |
          sum by (job, method, path) (
            rate(http_requests_total{status=~"5.."}[5m])
          )
          /
          sum by (job, method, path) (
            rate(http_requests_total[5m])
          )

      # p99 latency per endpoint
      - record: job:http_request_duration_p99:5m
        expr: |
          histogram_quantile(0.99,
            sum by (job, method, path, le) (
              rate(http_request_duration_seconds_bucket[5m])
            )
          )

      # Availability (1 - error rate)
      - record: job:availability:5m
        expr: |
          1 - (
            sum by (job) (rate(http_requests_total{status=~"5.."}[5m]))
            /
            sum by (job) (rate(http_requests_total[5m]))
          )

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

// handlers/health.go

// Liveness: I am running. Kill me only if this fails.
// Should NOT check external dependencies — a database being down
// does not mean this process should be killed.
func LivenessHandler(w http.ResponseWriter, r *http.Request) {
    w.Header().Set("Content-Type", "application/json")
    w.WriteHeader(http.StatusOK)
    json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

// Readiness: I am ready to serve traffic.
// SHOULD check critical external dependencies.
// A failing readiness check removes the pod from load balancer rotation
// but does NOT restart it.
func ReadinessHandler(app *App) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {
        ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second)
        defer cancel()

        checks := map[string]error{
            "database": app.db.PingContext(ctx),
            "cache":    app.cache.Ping(ctx).Err(),
        }

        allHealthy := true
        results := make(map[string]interface{})

        for name, err := range checks {
            if err != nil {
                allHealthy = false
                results[name] = map[string]string{"status": "unhealthy", "error": err.Error()}
                logger.FromContext(r.Context()).Warn("dependency unhealthy",
                    zap.String("dependency", name),
                    zap.Error(err),
                )
            } else {
                results[name] = map[string]string{"status": "ok"}
            }
        }

        statusCode := http.StatusOK
        if !allHealthy {
            statusCode = http.StatusServiceUnavailable
        }

        w.Header().Set("Content-Type", "application/json")
        w.WriteHeader(statusCode)
        json.NewEncoder(w).Encode(map[string]interface{}{
            "status":       map[bool]string{true: "ok", false: "degraded"}[allHealthy],
            "checks":       results,
            "timestamp":    time.Now().UTC(),
            "version":      app.version,
        })
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

// Startup: Have I finished initializing?
// Use for slow-starting applications. Kubernetes won't run liveness/readiness
// probes until this returns 200.
func (app *App) StartupHandler(w http.ResponseWriter, r *http.Request) {
    if !app.initialized.Load() {
        w.WriteHeader(http.StatusServiceUnavailable)
        json.NewEncoder(w).Encode(map[string]string{
            "status": "initializing",
            "phase":  app.startupPhase.Load().(string),
        })
        return
    }
    w.WriteHeader(http.StatusOK)
    json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

# deployment.yaml
livenessProbe:
  httpGet:
    path: /healthz
    port: 8080
  initialDelaySeconds: 5
  periodSeconds: 10
  failureThreshold: 3    # 3 failures = kill and restart

readinessProbe:
  httpGet:
    path: /readyz
    port: 8080
  initialDelaySeconds: 5
  periodSeconds: 5
  failureThreshold: 2    # 2 failures = remove from load balancer

startupProbe:
  httpGet:
    path: /startupz
    port: 8080
  failureThreshold: 30   # allow up to 5 minutes for startup
  periodSeconds: 10

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

func InfoHandler(app *App) http.HandlerFunc {
    // Compute once at startup
    info := map[string]interface{}{
        "version":    app.version,
        "commit":     app.commit,
        "build_date": app.buildDate,
        "go_version": runtime.Version(),
        "os_arch":    runtime.GOOS + "/" + runtime.GOARCH,
        "started_at": app.startedAt.UTC(),
    }

    return func(w http.ResponseWriter, r *http.Request) {
        // Add runtime stats
        var mem runtime.MemStats
        runtime.ReadMemStats(&mem)

        w.Header().Set("Content-Type", "application/json")
        json.NewEncoder(w).Encode(map[string]interface{}{
            "build":   info,
            "runtime": map[string]interface{}{
                "goroutines":   runtime.NumGoroutine(),
                "heap_alloc":   mem.HeapAlloc,
                "heap_sys":     mem.HeapSys,
                "num_gc":       mem.NumGC,
                "uptime_secs":  time.Since(app.startedAt).Seconds(),
            },
        })
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

// main.go
func main() {
    app := NewApp(cfg)

    server := &http.Server{
        Addr:         ":8080",
        Handler:      app.router,
        ReadTimeout:  15 * time.Second,
        WriteTimeout: 30 * time.Second,
        IdleTimeout:  60 * time.Second,
    }

    // Start server in a goroutine
    go func() {
        log.Info("server starting", zap.String("addr", server.Addr))
        if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
            log.Fatal("server error", zap.Error(err))
        }
    }()

    // Wait for shutdown signal
    quit := make(chan os.Signal, 1)
    signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
    sig := <-quit

    log.Info("shutdown signal received",
        zap.String("signal", sig.String()),
    )

    // Give in-flight requests time to complete
    // This duration should match your load balancer's connection drain timeout
    shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()

    // Signal app-level background workers to stop
    app.shutdown()

    if err := server.Shutdown(shutdownCtx); err != nil {
        log.Error("graceful shutdown failed", zap.Error(err))
        os.Exit(1)
    }

    log.Info("server stopped cleanly")
}

1
2
3
4
5
6

# deployment.yaml
lifecycle:
  preStop:
    exec:
      command: ["/bin/sh", "-c", "sleep 5"]
terminationGracePeriodSeconds: 60  # must be > preStop sleep + shutdown drain timeout

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

// pkg/circuitbreaker/breaker.go
package circuitbreaker

import (
    "context"
    "errors"
    "sync"
    "time"

    "go.uber.org/zap"
)

type State int

const (
    StateClosed   State = iota // normal operation
    StateOpen                  // failing fast
    StateHalfOpen              // testing recovery
)

var ErrCircuitOpen = errors.New("circuit breaker is open")

type Breaker struct {
    name          string
    maxFailures   int
    resetTimeout  time.Duration
    log           *zap.Logger

    mu            sync.Mutex
    state         State
    failureCount  int
    lastFailure   time.Time
    successCount  int
}

func New(name string, maxFailures int, resetTimeout time.Duration, log *zap.Logger) *Breaker {
    return &Breaker{
        name:         name,
        maxFailures:  maxFailures,
        resetTimeout: resetTimeout,
        log:          log,
        state:        StateClosed,
    }
}

func (b *Breaker) Execute(ctx context.Context, fn func(ctx context.Context) error) error {
    if err := b.allow(); err != nil {
        return err
    }

    err := fn(ctx)
    b.record(err)
    return err
}

func (b *Breaker) allow() error {
    b.mu.Lock()
    defer b.mu.Unlock()

    switch b.state {
    case StateClosed:
        return nil
    case StateOpen:
        if time.Since(b.lastFailure) > b.resetTimeout {
            b.state = StateHalfOpen
            b.successCount = 0
            b.log.Info("circuit half-open, probing",
                zap.String("circuit", b.name))
            return nil
        }
        return ErrCircuitOpen
    case StateHalfOpen:
        return nil
    }
    return nil
}

func (b *Breaker) record(err error) {
    b.mu.Lock()
    defer b.mu.Unlock()

    if err != nil {
        b.failureCount++
        b.lastFailure = time.Now()

        if b.state == StateHalfOpen || b.failureCount >= b.maxFailures {
            if b.state != StateOpen {
                b.log.Warn("circuit opened",
                    zap.String("circuit", b.name),
                    zap.Int("failures", b.failureCount),
                )
            }
            b.state = StateOpen
        }
    } else {
        if b.state == StateHalfOpen {
            b.successCount++
            if b.successCount >= 2 { // require 2 successes to close
                b.log.Info("circuit closed, recovery confirmed",
                    zap.String("circuit", b.name))
                b.state = StateClosed
                b.failureCount = 0
            }
        } else {
            b.failureCount = 0
        }
    }
}

func (b *Breaker) State() State {
    b.mu.Lock()
    defer b.mu.Unlock()
    return b.state
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

// In your metrics setup:
circuitState := promauto.NewGaugeVec(
    prometheus.GaugeOpts{
        Name: "circuit_breaker_state",
        Help: "Circuit breaker state (0=closed, 1=open, 2=half-open)",
    },
    []string{"circuit"},
)

// Update on state transitions (or poll in a goroutine):
go func() {
    ticker := time.NewTicker(10 * time.Second)
    for range ticker.C {
        for name, breaker := range breakers {
            circuitState.WithLabelValues(name).Set(float64(breaker.State()))
        }
    }
}()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

func (h *Handler) ProcessOrder(w http.ResponseWriter, r *http.Request) {
    // Set a total budget for this entire operation
    ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
    defer cancel()

    // Each downstream call consumes from the same budget
    // If inventory takes 4s, the payment call only has 1s left
    inventory, err := h.inventoryClient.Check(ctx, orderID)
    if err != nil {
        if errors.Is(err, context.DeadlineExceeded) {
            h.log.Warn("inventory check timed out",
                zap.String("order_id", orderID),
                zap.Duration("remaining", time.Until(deadline(ctx))),
            )
            http.Error(w, "request timeout", http.StatusGatewayTimeout)
            return
        }
        // ...
    }

    payment, err := h.paymentClient.Charge(ctx, amount)
    // ...
}

func deadline(ctx context.Context) time.Time {
    d, _ := ctx.Deadline()
    return d
}

1
2
3
4
5
6

log.Warn("slow dependency call",
    zap.String("dependency", "inventory-service"),
    zap.Duration("call_duration", duration),
    zap.Duration("budget_remaining", time.Until(deadline(ctx))),
    zap.Bool("budget_exceeded", ctx.Err() != nil),
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

// pkg/errors/errors.go
package errors

import "fmt"

type ErrorType string

const (
    // Client errors — the caller did something wrong. Not actionable by ops.
    ErrorTypeValidation   ErrorType = "validation"
    ErrorTypeNotFound     ErrorType = "not_found"
    ErrorTypeUnauthorized ErrorType = "unauthorized"

    // Server errors — our problem. Always actionable.
    ErrorTypeInternal     ErrorType = "internal"
    ErrorTypeUnavailable  ErrorType = "unavailable"  // dependency down
    ErrorTypeTimeout      ErrorType = "timeout"
)

type AppError struct {
    Type    ErrorType
    Message string
    Cause   error
    Fields  map[string]interface{}
}

func (e *AppError) Error() string {
    if e.Cause != nil {
        return fmt.Sprintf("%s: %v", e.Message, e.Cause)
    }
    return e.Message
}

func (e *AppError) Unwrap() error { return e.Cause }

func (e *AppError) IsClientError() bool {
    return e.Type == ErrorTypeValidation ||
        e.Type == ErrorTypeNotFound ||
        e.Type == ErrorTypeUnauthorized
}

// Usage:
// return &errors.AppError{
//     Type:    errors.ErrorTypeTimeout,
//     Message: "inventory service timed out",
//     Cause:   err,
//     Fields:  map[string]interface{}{"service": "inventory"},
// }

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

// In your HTTP handler — map error types to HTTP status codes:
func handleError(w http.ResponseWriter, err error) {
    var appErr *errors.AppError
    if !errors.As(err, &appErr) {
        // Unexpected error type — treat as internal error
        http.Error(w, "internal server error", http.StatusInternalServerError)
        return
    }

    // Increment error metric with type label
    errorCounter.WithLabelValues(string(appErr.Type)).Inc()

    switch appErr.Type {
    case errors.ErrorTypeValidation:
        http.Error(w, appErr.Message, http.StatusBadRequest)
    case errors.ErrorTypeNotFound:
        http.Error(w, appErr.Message, http.StatusNotFound)
    case errors.ErrorTypeUnauthorized:
        http.Error(w, appErr.Message, http.StatusUnauthorized)
    case errors.ErrorTypeTimeout:
        http.Error(w, "request timeout", http.StatusGatewayTimeout)
    default:
        // Log server errors — not client errors
        logger.FromContext(context.Background()).Error("request failed",
            zap.Error(err),
            zap.String("error_type", string(appErr.Type)),
        )
        http.Error(w, "internal server error", http.StatusInternalServerError)
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

## Pre-Production Observability Checklist

### Logging
- [ ] Structured JSON logging (not printf strings)
- [ ] Every log entry includes: timestamp, level, service, version
- [ ] Request context (trace_id, user_id) bound at middleware and flows through
- [ ] No secrets, PII, or full request bodies logged at INFO+
- [ ] Log levels document what each level means for this service
- [ ] Error logs include enough context to reproduce the problem

### Metrics
- [ ] RED metrics (rate, errors, duration) on all HTTP/gRPC endpoints
- [ ] No unbounded label cardinality (no user IDs, request IDs in labels)
- [ ] Path normalization to prevent cardinality explosion
- [ ] Business metrics for domain events (orders, payments, etc.)
- [ ] External dependency metrics (latency, error rate per dependency)
- [ ] /metrics endpoint secured (not publicly accessible)

### Health Endpoints
- [ ] /healthz (liveness): returns 200 if process is alive
- [ ] /readyz (readiness): returns 200 only when dependencies are reachable
- [ ] /readyz does NOT block indefinitely — has a timeout (e.g. 3s)
- [ ] Kubernetes probes configured with appropriate thresholds
- [ ] /info endpoint with version, commit, uptime

### Tracing
- [ ] W3C traceparent header extracted from incoming requests
- [ ] Trace context propagated to all outbound HTTP/gRPC calls
- [ ] trace_id included in all log entries
- [ ] Custom spans for critical business operations
- [ ] Trace IDs returned in response headers for client correlation

### Graceful Shutdown
- [ ] SIGTERM handler registered
- [ ] In-flight requests drained before exit (typically 30s)
- [ ] Background workers notified and given time to finish
- [ ] preStop hook adds delay for load balancer drain
- [ ] terminationGracePeriodSeconds > preStop + drain timeout

### Error Handling
- [ ] Error types classified (client vs server errors)
- [ ] Client errors (4xx) do not trigger server-side alerts
- [ ] Error metrics labeled by type for targeted alerting
- [ ] Retry logic uses exponential backoff with jitter
- [ ] Circuit breakers on external dependencies with tight timeouts
- [ ] Timeout budget set at entry point and propagated via context