OpenTelemetry: One Instrumentation to Rule Them All

1
2
3
4
5
6
7
8
9

{
  "service.name": "order-api",
  "service.version": "2.3.1",
  "service.instance.id": "pod-abc123",
  "deployment.environment": "production",
  "host.name": "k8s-node-7",
  "k8s.pod.name": "order-api-7d4b9f-xkw8p",
  "k8s.namespace.name": "production"
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

# docker-compose.yml
services:
  otel-collector:
    image: otel/opentelemetry-collector-contrib:latest
    restart: unless-stopped
    command: ["--config=/etc/otel/config.yaml"]
    volumes:
      - ./otel/collector-config.yaml:/etc/otel/config.yaml:ro
    ports:
      - "4317:4317"    # OTLP gRPC
      - "4318:4318"    # OTLP HTTP
      - "8889:8889"    # Prometheus metrics exporter (scrape this)
      - "13133:13133"  # Health check
    depends_on:
      - jaeger
      - loki

  jaeger:
    image: jaegertracing/all-in-one:latest
    restart: unless-stopped
    environment:
      COLLECTOR_OTLP_ENABLED: "true"
    ports:
      - "16686:16686"  # Jaeger UI
      - "4317"         # OTLP gRPC (internal)

  loki:
    image: grafana/loki:latest
    restart: unless-stopped
    command: -config.file=/etc/loki/local-config.yaml
    ports:
      - "3100:3100"

  prometheus:
    image: prom/prometheus:latest
    restart: unless-stopped
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
    ports:
      - "9090:9090"

  grafana:
    image: grafana/grafana:latest
    restart: unless-stopped
    environment:
      GF_SECURITY_ADMIN_PASSWORD: admin
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning:ro
    ports:
      - "3000:3000"

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

# otel/collector-config.yaml

receivers:
  # Accept OTLP from instrumented applications
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

  # Pull host metrics (like Node Exporter — useful for all-in-one setups)
  hostmetrics:
    collection_interval: 15s
    scrapers:
      cpu:
      memory:
      disk:
      filesystem:
      network:
      load:

  # Scrape Prometheus endpoints (replaces running a separate Prometheus)
  prometheus:
    config:
      scrape_configs:
        - job_name: self
          static_configs:
            - targets: ['localhost:8888']  # Collector's own metrics

processors:
  # Batch spans before sending (reduces network overhead)
  batch:
    timeout: 5s
    send_batch_size: 1024
    send_batch_max_size: 2048

  # Add resource attributes from environment
  resource:
    attributes:
      - key: deployment.environment
        value: production
        action: upsert

  # Redact sensitive data from span attributes
  redaction:
    allow_all_keys: true
    blocked_values:
      - "4[0-9]{12}(?:[0-9]{3})?"    # Credit card numbers
      - "[A-Z]{2}[0-9]{2}[A-Z0-9]{4}[0-9]{7}([A-Z0-9]?){0,16}"  # IBAN

  # Memory limiter: prevent OOM
  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 128

  # Probabilistic sampling: keep 10% of non-error traces
  probabilistic_sampler:
    sampling_percentage: 10

  # Tail-based sampling: always keep error traces, sample the rest
  tail_sampling:
    decision_wait: 10s
    num_traces: 100000
    expected_new_traces_per_sec: 1000
    policies:
      - name: always-keep-errors
        type: status_code
        status_code:
          status_codes: [ERROR]
      - name: keep-slow-traces
        type: latency
        latency:
          threshold_ms: 1000
      - name: sample-rest
        type: probabilistic
        probabilistic:
          sampling_percentage: 5

  # Add Kubernetes metadata to all telemetry
  k8sattributes:
    auth_type: serviceAccount
    extract:
      metadata:
        - k8s.pod.name
        - k8s.pod.uid
        - k8s.deployment.name
        - k8s.namespace.name
        - k8s.node.name

exporters:
  # Send traces to Jaeger
  otlp/jaeger:
    endpoint: jaeger:4317
    tls:
      insecure: true

  # Export metrics in Prometheus format (Prometheus scrapes this)
  prometheus:
    endpoint: 0.0.0.0:8889
    namespace: otel
    send_timestamps: true
    metric_expiration: 180m

  # Send logs to Loki
  loki:
    endpoint: http://loki:3100/loki/api/v1/push
    default_labels_enabled:
      exporter: false
      job: true
    labels:
      resource:
        service.name: "service_name"
        deployment.environment: "environment"
      attributes:
        level: "level"

  # Debug exporter (prints to stdout — useful during development)
  debug:
    verbosity: detailed

extensions:
  health_check:
    endpoint: 0.0.0.0:13133
  pprof:
    endpoint: 0.0.0.0:1777

service:
  extensions: [health_check, pprof]
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, resource, redaction, tail_sampling, batch]
      exporters: [otlp/jaeger]

    metrics:
      receivers: [otlp, hostmetrics, prometheus]
      processors: [memory_limiter, resource, batch]
      exporters: [prometheus]

    logs:
      receivers: [otlp]
      processors: [memory_limiter, resource, batch]
      exporters: [loki]

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106

package main

import (
    "context"
    "fmt"
    "net/http"
    "time"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/codes"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/propagation"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
    "go.uber.org/zap"
    "google.golang.org/grpc"
    "google.golang.org/grpc/credentials/insecure"
)

// initTracer sets up the OTel trace provider and returns a shutdown function.
func initTracer(ctx context.Context) (func(context.Context) error, error) {
    // Connect to the OTel Collector
    conn, err := grpc.DialContext(ctx, "otel-collector:4317",
        grpc.WithTransportCredentials(insecure.NewCredentials()),
        grpc.WithBlock(),
    )
    if err != nil {
        return nil, fmt.Errorf("failed to connect to collector: %w", err)
    }

    // OTLP trace exporter over gRPC
    exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn))
    if err != nil {
        return nil, fmt.Errorf("failed to create trace exporter: %w", err)
    }

    // Resource attributes — describe this service
    res, err := resource.Merge(
        resource.Default(),
        resource.NewWithAttributes(
            semconv.SchemaURL,
            semconv.ServiceName("order-api"),
            semconv.ServiceVersion("2.3.1"),
            attribute.String("deployment.environment", "production"),
        ),
    )
    if err != nil {
        return nil, err
    }

    // Trace provider with batch export
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exporter),
        sdktrace.WithResource(res),
        // In production, use sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.1)))
        // for head-based sampling, or leave unset for tail-based at the Collector.
        sdktrace.WithSampler(sdktrace.AlwaysSample()),
    )

    // Register as global provider
    otel.SetTracerProvider(tp)

    // Register W3C traceparent propagator for context propagation
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
        propagation.TraceContext{},
        propagation.Baggage{},
    ))

    return tp.Shutdown, nil
}

// tracer is the package-level tracer for this service.
var tracer = otel.Tracer("order-api")

// fetchOrder demonstrates manual span creation.
func fetchOrder(ctx context.Context, orderID string) (*Order, error) {
    // Create a child span
    ctx, span := tracer.Start(ctx, "fetchOrder",
        oteltrace.WithAttributes(
            attribute.String("order.id", orderID),
        ),
    )
    defer span.End()

    // Add an event (timestamped annotation)
    span.AddEvent("querying database")

    order, err := db.QueryOrder(ctx, orderID)
    if err != nil {
        // Record the error — this marks the span as failed
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        return nil, err
    }

    // Add result attributes
    span.SetAttributes(
        attribute.String("order.status", order.Status),
        attribute.Float64("order.total", order.Total),
    )
    span.SetStatus(codes.Ok, "")

    return order, nil
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

import (
    "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)

func main() {
    shutdown, err := initTracer(context.Background())
    if err != nil {
        log.Fatal(err)
    }
    defer shutdown(context.Background())

    // Wrap the HTTP handler — extracts traceparent from incoming requests,
    // creates a server span, and propagates context.
    handler := otelhttp.NewHandler(
        http.DefaultServeMux,
        "order-api",
        otelhttp.WithMessageEvents(otelhttp.ReadEvents, otelhttp.WriteEvents),
    )

    http.Handle("/api/orders", http.HandlerFunc(ordersHandler))
    http.ListenAndServe(":8080", handler)
}

// Outbound HTTP calls: inject traceparent into the request
func callDownstream(ctx context.Context, url string) (*http.Response, error) {
    client := &http.Client{
        Transport: otelhttp.NewTransport(http.DefaultTransport),
    }
    req, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
    return client.Do(req)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

import (
    "go.opentelemetry.io/contrib/instrumentation/database/sql/otelsql"
    "go.opentelemetry.io/otel/semconv/v1.24.0/dbconv"
)

// Wrap the database driver with OTel instrumentation
func initDB() *sql.DB {
    driverName, err := otelsql.Register("postgres",
        otelsql.WithAttributes(
            semconv.DBSystemPostgreSQL,
            attribute.String("db.name", "orders"),
        ),
        otelsql.WithSpanOptions(otelsql.SpanOptions{
            DisableErrSkip: true,   // Don't create spans for sql.ErrNoRows
            RecordError: func(err error) bool {
                return err != sql.ErrNoRows
            },
        }),
    )
    if err != nil {
        log.Fatal(err)
    }

    db, err := sql.Open(driverName, os.Getenv("DATABASE_URL"))
    if err != nil {
        log.Fatal(err)
    }

    // Record connection pool stats as OTel metrics
    otelsql.RecordStats(db,
        otelsql.WithAttributes(semconv.DBSystemPostgreSQL),
    )

    return db
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

import (
    "go.opentelemetry.io/otel/metric"
    "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
    sdkmetric "go.opentelemetry.io/otel/sdk/metric"
)

func initMeter(ctx context.Context) (func(context.Context) error, error) {
    exporter, err := otlpmetricgrpc.New(ctx,
        otlpmetricgrpc.WithEndpoint("otel-collector:4317"),
        otlpmetricgrpc.WithInsecure(),
    )
    if err != nil {
        return nil, err
    }

    mp := sdkmetric.NewMeterProvider(
        sdkmetric.WithReader(
            sdkmetric.NewPeriodicReader(exporter,
                sdkmetric.WithInterval(15*time.Second),
            ),
        ),
        sdkmetric.WithResource(res),  // same resource as tracer
    )
    otel.SetMeterProvider(mp)
    return mp.Shutdown, nil
}

var meter = otel.Meter("order-api")

func setupMetrics() {
    // Counter
    requestCounter, _ := meter.Int64Counter(
        "http.server.request.total",
        metric.WithDescription("Total HTTP requests"),
        metric.WithUnit("{request}"),
    )

    // Histogram for latency
    requestDuration, _ := meter.Float64Histogram(
        "http.server.request.duration",
        metric.WithDescription("HTTP request duration"),
        metric.WithUnit("s"),
        metric.WithExplicitBucketBoundaries(
            0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
        ),
    )

    // Observable gauge (callback-based, no need to call Record)
    _, _ = meter.Int64ObservableGauge(
        "process.goroutines",
        metric.WithDescription("Number of goroutines"),
        metric.WithInt64Callback(func(_ context.Context, o metric.Int64Observer) error {
            o.Observe(int64(runtime.NumGoroutine()))
            return nil
        }),
    )

    // Use the instruments in your handlers
    _ = requestCounter
    _ = requestDuration
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

import (
    "go.opentelemetry.io/otel/trace"
    "go.uber.org/zap"
)

// fromContext extracts trace context from the span in the context
// and adds it to a zap logger.
func loggerFromContext(ctx context.Context, base *zap.Logger) *zap.Logger {
    span := trace.SpanFromContext(ctx)
    if !span.IsRecording() {
        return base
    }
    sc := span.SpanContext()
    return base.With(
        zap.String("trace_id", sc.TraceID().String()),
        zap.String("span_id", sc.SpanID().String()),
        zap.Bool("trace_sampled", sc.IsSampled()),
    )
}

// Usage in a handler:
func ordersHandler(w http.ResponseWriter, r *http.Request) {
    log := loggerFromContext(r.Context(), globalLogger)
    log.Info("handling order request",
        zap.String("user_id", userID),
    )
    // This log line will contain trace_id and span_id,
    // enabling direct correlation to the Jaeger trace.
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

pip install \
    opentelemetry-distro \
    opentelemetry-exporter-otlp-proto-grpc

# Auto-instrument an existing application — no code changes required
opentelemetry-instrument \
    --traces_exporter otlp_proto_grpc \
    --metrics_exporter otlp_proto_grpc \
    --logs_exporter otlp_proto_grpc \
    --exporter_otlp_endpoint http://otel-collector:4317 \
    --service_name order-api \
    --deployment_environment production \
    uvicorn main:app --host 0.0.0.0 --port 8080

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# Dockerfile
FROM python:3.12-slim

WORKDIR /app
COPY requirements.txt .
RUN pip install -r requirements.txt && \
    pip install opentelemetry-distro opentelemetry-exporter-otlp-proto-grpc && \
    opentelemetry-bootstrap --action=install  # Auto-installs instrumentation for detected libraries

COPY . .
CMD ["opentelemetry-instrument", "uvicorn", "main:app", "--host", "0.0.0.0"]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# Kubernetes deployment env vars
env:
  - name: OTEL_SERVICE_NAME
    value: order-api
  - name: OTEL_EXPORTER_OTLP_ENDPOINT
    value: http://otel-collector.monitoring.svc.cluster.local:4317
  - name: OTEL_EXPORTER_OTLP_PROTOCOL
    value: grpc
  - name: OTEL_RESOURCE_ATTRIBUTES
    value: deployment.environment=production,service.version=2.3.1
  - name: OTEL_TRACES_SAMPLER
    value: parentbased_always_on
  - name: OTEL_PYTHON_LOG_CORRELATION
    value: "true"   # Inject trace IDs into Python log records
  - name: OTEL_PYTHON_LOG_LEVEL
    value: info

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

from opentelemetry import trace, metrics
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource, SERVICE_NAME, SERVICE_VERSION
from opentelemetry.semconv.trace import SpanAttributes
from opentelemetry.trace import Status, StatusCode
import logging

# Initialize tracer
resource = Resource.create({
    SERVICE_NAME: "order-api",
    SERVICE_VERSION: "2.3.1",
    "deployment.environment": "production",
})

provider = TracerProvider(resource=resource)
provider.add_span_processor(
    BatchSpanProcessor(
        OTLPSpanExporter(endpoint="http://otel-collector:4317", insecure=True)
    )
)
trace.set_tracer_provider(provider)

tracer = trace.get_tracer("order-api")

# Add trace context to Python log records
from opentelemetry.instrumentation.logging import LoggingInstrumentor
LoggingInstrumentor().instrument(set_logging_format=True)

# Now every log record includes trace_id and span_id automatically
logger = logging.getLogger(__name__)


def fetch_order(ctx, order_id: str) -> dict:
    with tracer.start_as_current_span(
        "fetch_order",
        attributes={
            "order.id": order_id,
            SpanAttributes.DB_SYSTEM: "postgresql",
        },
    ) as span:
        try:
            span.add_event("querying database")
            order = db.query("SELECT * FROM orders WHERE id = %s", order_id)

            span.set_attribute("order.status", order["status"])
            span.set_attribute("order.total", float(order["total"]))
            span.set_status(Status(StatusCode.OK))

            logger.info(
                "fetched order",
                extra={"order_id": order_id, "status": order["status"]}
            )
            # Log output includes: trace_id=4bf92f35... span_id=a3ce929d...
            return order

        except Exception as e:
            span.record_exception(e)
            span.set_status(Status(StatusCode.ERROR, str(e)))
            raise


# FastAPI middleware for automatic span creation
from fastapi import FastAPI, Request
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor

app = FastAPI()
FastAPIInstrumentor.instrument_app(app)  # Auto-instruments all routes

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

import httpx
from opentelemetry.propagate import inject

async def call_downstream(ctx, url: str) -> dict:
    """Make an outbound HTTP call with trace context propagation."""
    headers = {}
    inject(headers)  # Adds traceparent and tracestate headers
    # headers now contains: {"traceparent": "00-4bf92f35...-a3ce929d...-01"}

    async with httpx.AsyncClient() as client:
        response = await client.get(url, headers=headers)
        return response.json()

1
2
3
4
5

npm install \
    @opentelemetry/sdk-node \
    @opentelemetry/auto-instrumentations-node \
    @opentelemetry/exporter-trace-otlp-grpc \
    @opentelemetry/exporter-metrics-otlp-grpc

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

// tracing.js — import this FIRST before any other requires
const { NodeSDK } = require('@opentelemetry/sdk-node');
const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node');
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-grpc');
const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-grpc');
const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics');
const { Resource } = require('@opentelemetry/resources');
const { SemanticResourceAttributes } = require('@opentelemetry/semantic-conventions');

const sdk = new NodeSDK({
  resource: new Resource({
    [SemanticResourceAttributes.SERVICE_NAME]: 'order-api',
    [SemanticResourceAttributes.SERVICE_VERSION]: '2.3.1',
    'deployment.environment': process.env.NODE_ENV || 'production',
  }),

  traceExporter: new OTLPTraceExporter({
    url: 'http://otel-collector:4317',
  }),

  metricReader: new PeriodicExportingMetricReader({
    exporter: new OTLPMetricExporter({
      url: 'http://otel-collector:4317',
    }),
    exportIntervalMillis: 15000,
  }),

  instrumentations: [
    getNodeAutoInstrumentations({
      // Auto-instruments: http, express, fastify, pg, mysql, redis,
      // mongodb, graphql, grpc, aws-sdk, and many more
      '@opentelemetry/instrumentation-http': {
        ignoreIncomingRequestHook: (req) => {
          // Don't trace health check endpoints
          return req.url === '/health' || req.url === '/metrics';
        },
      },
      '@opentelemetry/instrumentation-pg': {
        enhancedDatabaseReporting: true,  // Include query text in spans
        dbStatementSerializer: (operation, query) => {
          // Sanitize SQL — don't log parameter values
          return query.text;
        },
      },
    }),
  ],
});

sdk.start();
console.log('OpenTelemetry initialized');

process.on('SIGTERM', () => {
  sdk.shutdown().then(() => process.exit(0));
});

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

// server.js — require tracing before anything else
require('./tracing');  // Must be first

const express = require('express');
const { trace, metrics, context, propagation } = require('@opentelemetry/api');

const app = express();
const tracer = trace.getTracer('order-api');
const meter = metrics.getMeter('order-api');

// Custom metrics
const requestCounter = meter.createCounter('http.server.request.total', {
  description: 'Total HTTP requests',
});

const requestDuration = meter.createHistogram('http.server.request.duration', {
  description: 'HTTP request duration',
  unit: 's',
  advice: {
    explicitBucketBoundaries: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
  },
});

app.get('/api/orders/:id', async (req, res) => {
  const start = Date.now();

  // Get the current span (created by the auto-instrumentation middleware)
  const span = trace.getActiveSpan();
  span?.setAttribute('order.id', req.params.id);

  try {
    // Create a child span for a specific operation
    const order = await tracer.startActiveSpan('fetchOrder', async (childSpan) => {
      try {
        const result = await db.query('SELECT * FROM orders WHERE id = $1', [req.params.id]);
        childSpan.setAttribute('db.rows_affected', result.rowCount);
        return result.rows[0];
      } finally {
        childSpan.end();
      }
    });

    requestCounter.add(1, { 'http.status_code': 200, 'http.route': '/api/orders/:id' });
    res.json(order);
  } catch (err) {
    span?.recordException(err);
    span?.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
    requestCounter.add(1, { 'http.status_code': 500, 'http.route': '/api/orders/:id' });
    res.status(500).json({ error: 'Internal server error' });
  } finally {
    requestDuration.record((Date.now() - start) / 1000, {
      'http.route': '/api/orders/:id',
    });
  }
});

1

kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

# Define an Instrumentation resource
apiVersion: opentelemetry.io/v1alpha1
kind: Instrumentation
metadata:
  name: otel-instrumentation
  namespace: production
spec:
  exporter:
    endpoint: http://otel-collector.monitoring.svc.cluster.local:4317

  propagators:
    - tracecontext
    - baggage

  sampler:
    type: parentbased_traceidratio
    argument: "0.25"  # Sample 25% of new traces

  python:
    env:
      - name: OTEL_PYTHON_LOG_CORRELATION
        value: "true"

  nodejs:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-nodejs:latest

  java:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-java:latest

  go:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-go:latest

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# Annotate any pod to enable auto-instrumentation
apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-api
  namespace: production
spec:
  template:
    metadata:
      annotations:
        instrumentation.opentelemetry.io/inject-python: "true"
        # Or for Node.js:
        # instrumentation.opentelemetry.io/inject-nodejs: "true"
        # Or for Java:
        # instrumentation.opentelemetry.io/inject-java: "true"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

# grafana/provisioning/datasources/otel.yaml
apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    url: http://prometheus:9090
    isDefault: true

  - name: Jaeger
    type: jaeger
    url: http://jaeger:16686
    jsonData:
      tracesToLogsV2:
        datasourceUid: loki
        filterByTraceID: true        # Link spans to Loki logs with same trace ID
        filterBySpanID: true

  - name: Loki
    type: loki
    url: http://loki:3100
    jsonData:
      derivedFields:
        - name: TraceID
          matcherRegex: '"trace_id":"(\w+)"'
          url: '$${__value.raw}'
          datasourceUid: jaeger       # Clicking trace ID opens Jaeger

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

// Instead of this:
span.SetAttributes(attribute.String("db.query", "SELECT ..."))

// Use semantic conventions:
import semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
span.SetAttributes(
    semconv.DBSystemPostgreSQL,
    semconv.DBStatement("SELECT * FROM orders WHERE id = ?"),
    semconv.DBName("orders"),
    semconv.DBOperationName("SELECT"),
)

1
2
3
4
5
6
7

# In SDK: sample 10% of all traces
OTEL_TRACES_SAMPLER=traceidratio
OTEL_TRACES_SAMPLER_ARG=0.1

# parent-based: respect sampling decision from upstream services
OTEL_TRACES_SAMPLER=parentbased_traceidratio
OTEL_TRACES_SAMPLER_ARG=0.1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# Collector config (already shown above — the tail_sampling processor)
tail_sampling:
  decision_wait: 10s
  policies:
    - name: errors          # Always keep
      type: status_code
      status_code: {status_codes: [ERROR]}
    - name: slow            # Keep if > 1 second
      type: latency
      latency: {threshold_ms: 1000}
    - name: sample-healthy  # Sample 5% of fast, healthy traces
      type: probabilistic
      probabilistic: {sampling_percentage: 5}

1
2
3
4
5
6

// Use ParentBased sampler — respects the sampling decision from the parent span
sdktrace.WithSampler(
    sdktrace.ParentBased(
        sdktrace.TraceIDRatioBased(0.1),  // 10% for root spans
    ),
)