1
2
3
4
5
6
7
8

# Using the official Helm chart
helm repo add spiffe https://spiffe.github.io/helm-charts-hardened
helm repo update

helm install spire spiffe/spire \
  --namespace spire-system \
  --create-namespace \
  --values spire-values.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

# spire-values.yaml
global:
  spiffe:
    trustDomain: prod.example.com

spire-server:
  replicaCount: 3  # HA with Raft consensus
  dataStorage:
    enabled: true
    size: 10Gi
    storageClass: gp3

  ca:
    keyType: ec-p256
    subject:
      country: US
      organization: Example Corp

  # Kubernetes node attestor
  nodeAttestor:
    k8sPsat:
      enabled: true
      serviceAccountAllowList:
        - spire-system:spire-agent

  # Where to store registration entries
  persistence:
    type: postgres
    postgres:
      host: postgres
      dbName: spire
      username: spire

spire-agent:
  nodeAttestor:
    k8sPsat:
      enabled: true

  workloadAttestors:
    k8s:
      enabled: true
      skipKubeletVerification: false

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# Register the payment-service workload
kubectl exec -n spire-system deploy/spire-server -- \
  spire-server entry create \
    -spiffeID spiffe://prod.example.com/ns/payments/sa/payment-service \
    -parentID spiffe://prod.example.com/k8s-psat/prod-cluster/node \
    -selector k8s:ns:payments \
    -selector k8s:sa:payment-service \
    -ttl 3600  # 1-hour certificate TTL (SPIRE auto-renews at ~50% of TTL)

# Register the order-service
kubectl exec -n spire-system deploy/spire-server -- \
  spire-server entry create \
    -spiffeID spiffe://prod.example.com/ns/orders/sa/order-service \
    -parentID spiffe://prod.example.com/k8s-psat/prod-cluster/node \
    -selector k8s:ns:orders \
    -selector k8s:sa:order-service \
    -ttl 3600

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# ClusterSPIFFEID — automatically register entries for pods matching a selector
apiVersion: spire.spiffe.io/v1alpha1
kind: ClusterSPIFFEID
metadata:
  name: payment-service-identity
spec:
  spiffeIDTemplate: "spiffe://prod.example.com/ns/{{ .PodMeta.Namespace }}/sa/{{ .PodSpec.ServiceAccountName }}"
  podSelector:
    matchLabels:
      app.kubernetes.io/part-of: payment-platform
  namespaceSelector:
    matchLabels:
      environment: production
  ttl: 1h

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# Deploy a debug container and check the Workload API
kubectl run spiffe-test --image=ghcr.io/spiffe/spiffe-helper:latest \
  --overrides='{"spec":{"serviceAccountName":"payment-service"}}' \
  -n payments -- sleep infinity

kubectl exec -n payments spiffe-test -- \
  /opt/spiffe-helper/spiffe-helper \
    -config /dev/stdin <<EOF
agentAddress = "/spiffe-workload-api/spire-agent.sock"
cmd = "openssl"
cmdArgs = "x509 -in /tmp/svid.pem -noout -text"
EOF
# Prints the X.509 certificate with the SPIFFE ID in the SAN field

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110

# envoy-sidecar-config.yaml
static_resources:
  listeners:
    # Inbound: accept connections, require client cert
    - name: inbound_listener
      address:
        socket_address:
          address: 0.0.0.0
          port_value: 15006  # Envoy intercepts inbound traffic
      filter_chains:
        - transport_socket:
            name: envoy.transport_sockets.tls
            typed_config:
              "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext
              require_client_certificate: true  # mTLS — require client cert
              common_tls_context:
                # Get our certificate from SPIRE via SDS
                tls_certificate_sds_secret_configs:
                  - name: "spiffe://prod.example.com/ns/payments/sa/payment-service"
                    sds_config:
                      api_config_source:
                        api_type: GRPC
                        grpc_services:
                          - envoy_grpc:
                              cluster_name: spire_agent
                # Validate client certificates against SPIRE's trust bundle
                validation_context_sds_secret_config:
                  name: "spiffe://prod.example.com"
                  sds_config:
                    api_config_source:
                      api_type: GRPC
                      grpc_services:
                        - envoy_grpc:
                            cluster_name: spire_agent
                tls_params:
                  tls_minimum_protocol_version: TLSv1_3
          filters:
            - name: envoy.filters.network.http_connection_manager
              typed_config:
                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
                stat_prefix: inbound_http
                route_config:
                  virtual_hosts:
                    - name: local
                      domains: ["*"]
                      routes:
                        - match: {prefix: "/"}
                          route:
                            cluster: local_service
                http_filters:
                  - name: envoy.filters.http.router
                    typed_config:
                      "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router

    # Outbound: add client cert when calling other services
    - name: outbound_listener
      address:
        socket_address:
          address: 127.0.0.1
          port_value: 15001
      filter_chains:
        - transport_socket:
            name: envoy.transport_sockets.tls
            typed_config:
              "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
              common_tls_context:
                tls_certificate_sds_secret_configs:
                  - name: "spiffe://prod.example.com/ns/orders/sa/order-service"
                    sds_config:
                      api_config_source:
                        api_type: GRPC
                        grpc_services:
                          - envoy_grpc:
                              cluster_name: spire_agent
                validation_context_sds_secret_config:
                  name: "spiffe://prod.example.com"
                  sds_config:
                    api_config_source:
                      api_type: GRPC
                      grpc_services:
                        - envoy_grpc:
                            cluster_name: spire_agent

  clusters:
    # SPIRE Agent SDS endpoint
    - name: spire_agent
      connect_timeout: 1s
      type: STATIC
      load_assignment:
        cluster_name: spire_agent
        endpoints:
          - lb_endpoints:
              - endpoint:
                  address:
                    pipe:
                      path: /run/spire/sockets/agent.sock  # Unix socket

    # Local application (your service)
    - name: local_service
      connect_timeout: 1s
      type: STATIC
      load_assignment:
        cluster_name: local_service
        endpoints:
          - lb_endpoints:
              - endpoint:
                  address:
                    socket_address:
                      address: 127.0.0.1
                      port_value: 8080

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

apiVersion: apps/v1
kind: Deployment
metadata:
  name: payment-service
  namespace: payments
spec:
  template:
    spec:
      serviceAccountName: payment-service

      initContainers:
        # Wait for SPIRE agent socket to be ready
        - name: wait-for-spire
          image: busybox
          command: ['sh', '-c', 'until [ -S /run/spire/sockets/agent.sock ]; do sleep 1; done']
          volumeMounts:
            - name: spire-agent-socket
              mountPath: /run/spire/sockets
              readOnly: true

      containers:
        - name: payment-service
          image: myorg/payment-service:latest
          ports:
            - containerPort: 8080

        - name: envoy
          image: envoyproxy/envoy:v1.29-latest
          args: ["-c", "/etc/envoy/envoy.yaml"]
          ports:
            - containerPort: 15006  # Inbound (from other services)
            - containerPort: 15001  # Outbound (to other services)
          volumeMounts:
            - name: envoy-config
              mountPath: /etc/envoy
            - name: spire-agent-socket
              mountPath: /run/spire/sockets
              readOnly: true

      volumes:
        - name: envoy-config
          configMap:
            name: envoy-sidecar-config
        - name: spire-agent-socket
          hostPath:
            path: /run/spire/sockets
            type: DirectoryOrCreate

1
2
3
4
5

istioctl install --set profile=default -y

# Label namespace for automatic sidecar injection
kubectl label namespace payments istio-injection=enabled
kubectl label namespace orders istio-injection=enabled

1
2
3
4
5
6
7
8
9

# PeerAuthentication — require mTLS for all services in the mesh
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: default
  namespace: istio-system  # Mesh-wide policy
spec:
  mtls:
    mode: STRICT  # Reject any plaintext connections

1
2
3
4
5
6
7
8
9

# Per-namespace override — PERMISSIVE during migration (accepts both mTLS and plaintext)
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: default
  namespace: legacy-services
spec:
  mtls:
    mode: PERMISSIVE

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

# Only allow order-service to call payment-service
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
  name: payment-service-authz
  namespace: payments
spec:
  selector:
    matchLabels:
      app: payment-service
  action: ALLOW
  rules:
    - from:
        - source:
            # The principal is the SPIFFE ID of the caller
            principals:
              - "cluster.local/ns/orders/sa/order-service"
              - "cluster.local/ns/checkout/sa/checkout-service"
      to:
        - operation:
            methods: ["POST"]
            paths: ["/api/payments/charge", "/api/payments/refund"]
    - from:
        - source:
            principals:
              - "cluster.local/ns/monitoring/sa/prometheus"
      to:
        - operation:
            methods: ["GET"]
            paths: ["/metrics"]

1
2
3
4
5
6
7
8

# Deny everything not explicitly allowed (default-deny)
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
  name: deny-all
  namespace: payments
spec:
  {}  # Empty spec = deny all — explicit ALLOW policies add exceptions

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# istio-operator.yaml
apiVersion: install.istio.io/v1alpha1
kind: IstioOperator
spec:
  meshConfig:
    # Point Istio at SPIRE's Workload API
    ca:
      address: "unix:///run/spire/sockets/agent.sock"

  values:
    pilot:
      env:
        # Use external CA (SPIRE)
        EXTERNAL_CA: "true"
        CA_PROVIDER: "Custom"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

package main

import (
    "context"
    "crypto/tls"
    "net/http"

    "github.com/spiffe/go-spiffe/v2/spiffeid"
    "github.com/spiffe/go-spiffe/v2/spiffetls"
    "github.com/spiffe/go-spiffe/v2/spiffetls/tlsconfig"
    "github.com/spiffe/go-spiffe/v2/workloadapi"
)

const socketPath = "unix:///run/spire/sockets/agent.sock"

// Server: accept only connections from the order-service
func runServer(ctx context.Context) error {
    // Connect to SPIRE Workload API
    source, err := workloadapi.NewX509Source(ctx,
        workloadapi.WithClientOptions(workloadapi.WithAddr(socketPath)),
    )
    if err != nil {
        return fmt.Errorf("workload API: %w", err)
    }
    defer source.Close()

    // Only accept clients with this SPIFFE ID
    authorizedID := spiffeid.RequireIDFromString(
        "spiffe://prod.example.com/ns/orders/sa/order-service",
    )

    tlsConfig := tlsconfig.MTLSServerConfig(
        source,
        source,
        tlsconfig.AuthorizeID(authorizedID),
    )

    server := &http.Server{
        Addr:      ":8443",
        TLSConfig: tlsConfig,
        Handler:   myHandler,
    }
    return server.ListenAndServeTLS("", "")  // Certs come from TLSConfig, not files
}

// Client: present our certificate, verify server is the payment-service
func callPaymentService(ctx context.Context, amount float64) error {
    source, err := workloadapi.NewX509Source(ctx,
        workloadapi.WithClientOptions(workloadapi.WithAddr(socketPath)),
    )
    if err != nil {
        return err
    }
    defer source.Close()

    // Authorize the specific server we're calling
    paymentServiceID := spiffeid.RequireIDFromString(
        "spiffe://prod.example.com/ns/payments/sa/payment-service",
    )

    tlsConfig := tlsconfig.MTLSClientConfig(
        source,
        source,
        tlsconfig.AuthorizeID(paymentServiceID),
    )

    httpClient := &http.Client{
        Transport: &http.Transport{TLSClientConfig: tlsConfig},
    }

    resp, err := httpClient.Post(
        "https://payment-service.payments.svc.cluster.local:8443/charge",
        "application/json",
        body,
    )
    // If payment-service presents a cert for a different SPIFFE ID, this fails at the TLS layer
    return err
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

import asyncio
from pyspiffe.workloadapi.default_workload_api_client import DefaultWorkloadApiClient
from pyspiffe.spiffe_id.spiffe_id import SpiffeId
import ssl

async def get_mtls_context(allowed_peer_spiffe_id: str) -> ssl.SSLContext:
    client = DefaultWorkloadApiClient("unix:///run/spire/sockets/agent.sock")
    x509_svid = await client.fetch_x509_svid()
    x509_bundle = await client.fetch_x509_bundles()

    # Build SSL context with our certificate and validation of peer
    ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)

    # Load our certificate (in-memory, no files on disk)
    ctx.load_verify_locations(cadata=x509_bundle.get_bundle_for_trust_domain("prod.example.com").x509_authorities_pem)

    # Load our SVID
    ctx.load_cert_chain(
        certfile=x509_svid.cert_chain_pem,
        keyfile=x509_svid.private_key_pem,
    )

    ctx.verify_mode = ssl.CERT_REQUIRED
    ctx.check_hostname = False  # SPIFFE uses URI SANs, not hostname

    return ctx

# FastAPI server with mTLS
from fastapi import FastAPI, Request, HTTPException
import uvicorn

app = FastAPI()

@app.middleware("http")
async def verify_spiffe_identity(request: Request, call_next):
    # Extract the peer's SPIFFE ID from the TLS connection
    peer_cert = request.scope.get("ssl_object").getpeercert()
    san_list = [v for t, v in peer_cert.get("subjectAltName", []) if t == "URI"]

    allowed_peers = {
        "spiffe://prod.example.com/ns/orders/sa/order-service",
        "spiffe://prod.example.com/ns/checkout/sa/checkout-service",
    }
    if not any(san in allowed_peers for san in san_list):
        raise HTTPException(403, f"Unauthorized peer: {san_list}")

    return await call_next(request)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

use spiffe::{WorkloadApiClient, X509Source};
use rustls::{ServerConfig, ClientConfig};

async fn build_server_tls_config() -> anyhow::Result<ServerConfig> {
    let source = X509Source::default().await?;
    let svid = source.get_x509_svid().await?;
    let bundle = source.get_x509_bundle_set().await?;

    let cert_chain = svid.cert_chain()
        .iter()
        .map(|c| rustls::Certificate(c.to_der().unwrap()))
        .collect();
    let private_key = rustls::PrivateKey(svid.private_key().to_pkcs8_der().unwrap());

    let roots = build_root_store(&bundle)?;
    let verifier = rustls::server::AllowAnyAuthenticatedClient::new(roots);

    let config = ServerConfig::builder()
        .with_safe_defaults()
        .with_client_cert_verifier(Arc::new(verifier))
        .with_single_cert(cert_chain, private_key)?;

    Ok(config)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

# Prometheus alerts for SPIRE certificate issues
groups:
  - name: spire_certificates
    rules:
      - alert: SPIREAgentDown
        expr: up{job="spire-agent"} == 0
        for: 2m
        annotations:
          summary: "SPIRE agent down on {{ $labels.instance }} — workloads cannot get SVIDs"

      - alert: SVIDExpiryApproaching
        # spire_agent_svid_expiry is exposed by the SPIRE agent
        expr: (spire_agent_svid_expiry - time()) < 300  # Less than 5 minutes
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "SVID for {{ $labels.spiffe_id }} expires in less than 5 minutes"

      - alert: SPIREServerCARenewalFailed
        expr: increase(spire_server_ca_manager_x509_ca_rotation_errors_total[15m]) > 0
        annotations:
          summary: "SPIRE Server CA rotation failed — investigate immediately"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# mTLS connection success rate
sum(rate(envoy_listener_ssl_handshake[5m])) by (listener_address)
/
(
  sum(rate(envoy_listener_ssl_handshake[5m])) by (listener_address)
  + sum(rate(envoy_listener_ssl_connection_error[5m])) by (listener_address)
)

# Connections rejected due to invalid client cert
sum(rate(envoy_listener_ssl_fail_verify_cert_hash[5m])) by (listener_address)

# Istio — traffic that arrived without mTLS (should be 0 in STRICT mode)
istio_requests_total{connection_security_policy="none"}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# Configure trust bundle federation between two SPIRE servers
# On cluster A's SPIRE server:
kubectl exec -n spire-system deploy/spire-server -- \
  spire-server federation create \
    -bundleEndpointURL https://spire-server.cluster-b.example.com:8443 \
    -bundleEndpointProfile https_spiffe \
    -endpointSpiffeID spiffe://cluster-b.example.com/spire/server \
    -trustDomain cluster-b.example.com

# On cluster B's SPIRE server:
kubectl exec -n spire-system deploy/spire-server -- \
  spire-server federation create \
    -bundleEndpointURL https://spire-server.cluster-a.example.com:8443 \
    -bundleEndpointProfile https_spiffe \
    -endpointSpiffeID spiffe://cluster-a.example.com/spire/server \
    -trustDomain cluster-a.example.com

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# Istio AuthorizationPolicy using federated identity
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
  name: cross-cluster-payment-authz
  namespace: payments
spec:
  selector:
    matchLabels:
      app: payment-service
  action: ALLOW
  rules:
    - from:
        - source:
            principals:
              # Trust cross-cluster callers by their SPIFFE ID
              - "cluster-b.example.com/ns/orders/sa/order-service"

1
2
3
4

# Find services still sending plaintext to a given service
kubectl exec -n istio-system deploy/prometheus -- \
  curl -s 'http://localhost:9090/api/v1/query' \
    --data-urlencode 'query=istio_requests_total{destination_service="payment-service.payments.svc.cluster.local", connection_security_policy="none"}'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# Flip payment-service to STRICT — rejects any plaintext
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: payment-service-mtls
  namespace: payments
spec:
  selector:
    matchLabels:
      app: payment-service
  mtls:
    mode: STRICT

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

# "certificate verify failed: CERTIFICATE_VERIFY_FAILED"
# Cause: Server cert issued by different CA than client trusts
# Fix: Verify trust bundles match; check SPIRE federation config

# "certificate verify failed: certificate has expired"
# Cause: SPIRE agent not running or can't reach SPIRE server to renew
# Fix: kubectl logs -n spire-system daemonset/spire-agent

# "connection reset by peer" on mTLS port
# Cause: Client not presenting a certificate (STRICT mode rejects it)
# Fix: Ensure client has Envoy sidecar injected and is in the mesh

# Istio sidecar not injected
kubectl get pod payment-service-abc123 -o jsonpath='{.spec.containers[*].name}'
# Should include "istio-proxy"
# If not: check namespace label and pod annotations

# Check what certificate Envoy is using
kubectl exec -n payments deploy/payment-service -c istio-proxy -- \
  openssl s_client -connect localhost:15006 -showcerts 2>/dev/null | \
  openssl x509 -noout -text | grep -A2 "Subject Alternative Name"
# Should show: URI:spiffe://prod.example.com/ns/payments/sa/payment-service

# Check SPIRE registration entries
kubectl exec -n spire-system deploy/spire-server -- \
  spire-server entry show -selector k8s:ns:payments

# View Envoy's TLS secrets (certificates currently in use)
kubectl exec -n payments deploy/payment-service -c istio-proxy -- \
  curl -s localhost:15000/certs | jq '.certificates[].cert_chain[0].subject_alt_names'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# Check PeerAuthentication and AuthorizationPolicy status
istioctl x authz check pod/payment-service-abc123.payments

# Verify proxy configuration is valid and synced
istioctl analyze -n payments

# Check what mTLS mode is in effect for a pod
istioctl x describe pod payment-service-abc123.payments

# View the Envoy config for a specific pod
istioctl proxy-config listeners payment-service-abc123.payments
istioctl proxy-config clusters payment-service-abc123.payments

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# Server commands
spire-server entry create -spiffeID ... -parentID ... -selector ...
spire-server entry show [-spiffeID <id>]
spire-server entry delete -entryID <id>
spire-server bundle show               # View current CA bundle
spire-server federation show          # View federated trust domains
spire-server agent show               # List attested agents
spire-server healthcheck

# Agent commands
spire-agent api fetch x509            # Fetch SVID from Workload API
spire-agent api validate jwt -audience <aud> -svid <token>
spire-agent healthcheck

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# Check mTLS status
istioctl x authz check <pod>.<namespace>
istioctl x describe pod <pod>.<namespace>

# View applied PeerAuthentication policies
kubectl get peerauthentication -A

# View applied AuthorizationPolicies
kubectl get authorizationpolicy -A

# Check if sidecar is injected
kubectl get pod <pod> -o jsonpath='{.spec.containers[*].name}'

# Watch real-time access log (includes mTLS peer info)
kubectl logs <pod> -c istio-proxy -f | grep -v "200"