Self-Hosted AI Inference: vLLM, llama.cpp, and Running Your Own OpenAI-Compatible API

1
2
3
4
5

# Requires CUDA 12.1+ and a compatible NVIDIA GPU
pip install vllm

# Or with Docker (recommended for production)
docker pull vllm/vllm-openai:latest

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# Serve Mistral 7B with an OpenAI-compatible API
docker run --gpus all \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  -p 8000:8000 \
  vllm/vllm-openai:latest \
  --model mistralai/Mistral-7B-Instruct-v0.3 \
  --dtype bfloat16 \
  --max-model-len 8192 \
  --gpu-memory-utilization 0.90

# Test it — identical to OpenAI API
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "mistralai/Mistral-7B-Instruct-v0.3",
    "messages": [{"role": "user", "content": "Explain PagedAttention in one paragraph."}],
    "temperature": 0.7,
    "max_tokens": 256
  }'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

vllm serve mistralai/Mistral-7B-Instruct-v0.3 \
  # Model loading
  --dtype bfloat16 \              # bfloat16 for Ampere+, float16 otherwise
  --quantization awq \            # AWQ, GPTQ, SqueezeLLM, None
  --max-model-len 32768 \         # Context window (reduces if OOM)
  \
  # Memory and concurrency
  --gpu-memory-utilization 0.90 \ # Reserve 10% for overhead
  --max-num-seqs 256 \            # Max concurrent sequences
  --max-num-batched-tokens 8192 \ # Max tokens per batch step
  \
  # Performance
  --enable-prefix-caching \       # Cache common prefixes (system prompts)
  --enable-chunked-prefill \      # Better latency for long prompts
  --speculative-model eagle \     # Speculative decoding (faster generation)
  \
  # Serving
  --host 0.0.0.0 \
  --port 8000 \
  --api-key your-secret-key \
  --served-model-name gpt-4       # Alias (useful for drop-in replacement)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# Llama 3 70B across 4x A100 80GB
docker run --gpus all \
  -e CUDA_VISIBLE_DEVICES=0,1,2,3 \
  vllm/vllm-openai:latest \
  --model meta-llama/Llama-3-70b-instruct \
  --tensor-parallel-size 4 \
  --dtype bfloat16 \
  --max-model-len 8192

# For very large models (405B), combine tensor + pipeline parallelism
vllm serve meta-llama/Llama-3.1-405B-Instruct \
  --tensor-parallel-size 8 \
  --pipeline-parallel-size 2 \   # 16 GPUs total
  --dtype bfloat16

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

import openai

client = openai.OpenAI(base_url="http://localhost:8000/v1", api_key="key")

SYSTEM_PROMPT = """You are a helpful assistant specialized in DevOps and infrastructure.
You always provide production-ready examples with proper error handling.
[... 2000 tokens of system context ...]"""

# First request: compute and cache the system prompt KV
response = client.chat.completions.create(
    model="mistral-7b",
    messages=[
        {"role": "system", "content": SYSTEM_PROMPT},
        {"role": "user", "content": "How do I set up Prometheus?"}
    ]
)

# Subsequent requests with same system prompt: cache hit, much faster TTFT
response = client.chat.completions.create(
    model="mistral-7b",
    messages=[
        {"role": "system", "content": SYSTEM_PROMPT},  # Cache hit!
        {"role": "user", "content": "Explain Grafana dashboards"}
    ]
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

from vllm import LLM, SamplingParams

llm = LLM(
    model="mistralai/Mistral-7B-Instruct-v0.3",
    dtype="bfloat16",
    gpu_memory_utilization=0.85,
    enable_prefix_caching=True,
)

sampling_params = SamplingParams(
    temperature=0.7,
    top_p=0.95,
    max_tokens=512,
    stop=["</s>", "[INST]"],
)

prompts = [
    "<s>[INST] Summarize this document: ... [/INST]",
    "<s>[INST] Extract the key points: ... [/INST]",
    # ... thousands of prompts
]

# vLLM automatically batches these optimally
outputs = llm.generate(prompts, sampling_params)

for output in outputs:
    print(f"Prompt: {output.prompt[:50]}...")
    print(f"Generated: {output.outputs[0].text}")
    print(f"Tokens: {len(output.outputs[0].token_ids)}")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# Build with CUDA support
git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp
cmake -B build -DGGML_CUDA=ON
cmake --build build --config Release -j$(nproc)

# Download a model from Hugging Face (GGUF format)
# Many pre-quantized models at huggingface.co/TheBloke or bartowski
huggingface-cli download \
  bartowski/Meta-Llama-3.1-8B-Instruct-GGUF \
  Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf \
  --local-dir ./models

# Run the server (OpenAI-compatible API)
./build/bin/llama-server \
  --model ./models/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf \
  --ctx-size 8192 \
  --n-gpu-layers 35 \    # Offload 35 layers to GPU (adjust for VRAM)
  --threads 8 \          # CPU threads for remaining layers
  --parallel 4 \         # Concurrent request slots
  --batch-size 512 \     # Prompt processing batch size
  --host 0.0.0.0 \
  --port 8080

# Test the API
curl http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{"model": "llama3.1", "messages": [{"role": "user", "content": "Hello!"}]}'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# CPU-only: use all cores, enable AVX2/AVX512 optimizations
./build/bin/llama-server \
  --model ./models/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf \
  --ctx-size 4096 \
  --n-gpu-layers 0 \     # No GPU layers
  --threads $(nproc) \
  --parallel 1 \         # CPU inference: 1 concurrent request
  --mlock                # Lock model in RAM, prevent swapping

# Expect ~5-15 tok/s on modern x86 server CPUs for 8B Q4
# Apple M3 Max: ~80-100 tok/s (unified memory = fast!)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# Llama 3 8B: 35 transformer layers total
# With 8GB VRAM at Q4_K_M:
# - Full GPU: needs ~6GB → use --n-gpu-layers 99 (all layers)
# - 6GB VRAM: --n-gpu-layers 28 (partial, rest on CPU)

# Experiment with --n-gpu-layers and check VRAM with nvidia-smi
watch -n 1 nvidia-smi --query-gpu=memory.used,memory.total --format=csv

# For Apple Silicon (Metal), use -ngl instead:
./build/bin/llama-server \
  --model ./models/llama-3.1-8b-instruct-q4_k_m.gguf \
  -ngl 99 \              # All layers on Metal GPU
  --ctx-size 8192

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

# Convert from HuggingFace format to GGUF
python convert_hf_to_gguf.py \
  /path/to/hf-model \
  --outfile models/my-model-f16.gguf \
  --outtype f16

# Quantize to Q4_K_M
./build/bin/llama-quantize \
  models/my-model-f16.gguf \
  models/my-model-Q4_K_M.gguf \
  Q4_K_M

# Verify quantization quality with perplexity test
./build/bin/llama-perplexity \
  --model models/my-model-Q4_K_M.gguf \
  --file wikitext-2-raw/wiki.test.raw \
  --ctx-size 512 \
  --chunks 128

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# Use a draft model (must be same architecture, smaller)
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --speculative-model meta-llama/Llama-3.2-1B-Instruct \
  --num-speculative-tokens 5 \
  --tensor-parallel-size 4

# Or use ngram-based speculation (no draft model needed)
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --speculative-model [ngram] \
  --ngram-prompt-lookup-min 4 \
  --num-speculative-tokens 5

1
2
3
4
5
6

# Run with a draft model
./build/bin/llama-speculative \
  --model models/llama-3.1-70b-Q4_K_M.gguf \
  --model-draft models/llama-3.2-1b-Q8_0.gguf \
  --n-draft 8 \
  --ctx-size 4096

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# Throughput-optimized (batch processing, offline inference)
vllm serve model \
  --max-num-seqs 512 \              # Many concurrent sequences
  --max-num-batched-tokens 32768 \  # Large batch steps
  --scheduling-policy fcfs          # First-come-first-served

# Latency-optimized (interactive chat, low TTFT)
vllm serve model \
  --max-num-seqs 32 \               # Fewer concurrent sequences
  --max-num-batched-tokens 4096 \   # Smaller batches
  --enable-chunked-prefill \        # Don't block short requests behind long prompts
  --preemption-mode swap            # Preempt low-priority requests

# Mixed workload (embeddings + generation)
vllm serve model \
  --task embed \                    # Or: generate, classify
  --max-num-seqs 128

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

# nginx.conf for vLLM cluster
upstream vllm_backends {
    least_conn;  # Route to server with fewest active connections
    server vllm-1:8000;
    server vllm-2:8000;
    server vllm-3:8000;
    server vllm-4:8000;
    keepalive 32;
}

server {
    listen 80;

    location /v1/ {
        proxy_pass http://vllm_backends;
        proxy_http_version 1.1;
        proxy_set_header Connection "";
        proxy_read_timeout 300s;    # Long timeout for streaming
        proxy_buffering off;        # Required for SSE streaming
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# Loading a quantized model with bitsandbytes (for custom inference)
from transformers import AutoModelForCausalLM, BitsAndBytesConfig
import torch

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,   # Nested quantization for extra savings
    bnb_4bit_quant_type="nf4",        # NormalFloat4: better than regular int4
)

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-3.1-8B-Instruct",
    quantization_config=bnb_config,
    device_map="auto",
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

# Apply AWQ quantization (requires calibration data)
pip install autoawq

python -c "
from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer

model_path = 'meta-llama/Llama-3.1-8B-Instruct'
quant_path = 'llama-3.1-8b-awq'

model = AutoAWQForCausalLM.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path)

quant_config = {
    'zero_point': True,
    'q_group_size': 128,
    'w_bit': 4,
    'version': 'GEMM'
}

model.quantize(tokenizer, quant_config=quant_config)
model.save_quantized(quant_path)
tokenizer.save_pretrained(quant_path)
"

# Now serve the AWQ model with vLLM
vllm serve ./llama-3.1-8b-awq --quantization awq

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# vLLM automatically uses FP8 on H100 with --dtype float8
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --dtype float8 \
  --tensor-parallel-size 4

# Or with explicit FP8 quantization
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --quantization fp8 \
  --kv-cache-dtype fp8 \
  --tensor-parallel-size 4

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

import openai
from typing import Iterator

# Point to your local server
client = openai.OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="not-needed-but-required",  # vLLM requires an API key field
)

# List available models
models = client.models.list()
for model in models.data:
    print(f"  {model.id}")

# Streaming chat completion
def stream_chat(prompt: str) -> Iterator[str]:
    stream = client.chat.completions.create(
        model="mistral-7b",
        messages=[{"role": "user", "content": prompt}],
        stream=True,
        temperature=0.7,
        max_tokens=1024,
    )
    for chunk in stream:
        if chunk.choices[0].delta.content:
            yield chunk.choices[0].delta.content

for token in stream_chat("Explain Kubernetes networking in detail"):
    print(token, end="", flush=True)

# Embeddings (vLLM supports embedding models too)
embedding_client = openai.OpenAI(
    base_url="http://localhost:8001/v1",  # separate embedding server
    api_key="key",
)

response = embedding_client.embeddings.create(
    model="BAAI/bge-m3",
    input=["text to embed", "another text"],
)
print(f"Embedding dim: {len(response.data[0].embedding)}")

# Structured outputs (JSON mode)
response = client.chat.completions.create(
    model="mistral-7b",
    messages=[{
        "role": "user",
        "content": "Extract name, age, and city from: John Smith, 34, lives in Seattle"
    }],
    response_format={"type": "json_object"},
    max_tokens=256,
)
import json
data = json.loads(response.choices[0].message.content)
print(data)  # {"name": "John Smith", "age": 34, "city": "Seattle"}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

from vllm import LLM, SamplingParams
from vllm.sampling_params import GuidedDecodingParams

llm = LLM(model="mistralai/Mistral-7B-Instruct-v0.3")

# JSON schema enforcement
schema = {
    "type": "object",
    "properties": {
        "hostname": {"type": "string"},
        "ip_address": {"type": "string", "pattern": r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$"},
        "status": {"type": "string", "enum": ["healthy", "degraded", "down"]},
        "cpu_percent": {"type": "number", "minimum": 0, "maximum": 100}
    },
    "required": ["hostname", "ip_address", "status", "cpu_percent"]
}

sampling_params = SamplingParams(
    temperature=0.1,
    max_tokens=200,
    guided_decoding=GuidedDecodingParams(json=schema)
)

outputs = llm.generate(
    ["Generate a server status report for web-01.prod.example.com"],
    sampling_params
)
print(outputs[0].outputs[0].text)
# Guaranteed valid JSON matching the schema

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# Install benchmarking tool
git clone https://github.com/vllm-project/vllm
cd vllm

# Basic throughput benchmark
python benchmarks/benchmark_throughput.py \
  --backend vllm \
  --model mistralai/Mistral-7B-Instruct-v0.3 \
  --input-len 512 \
  --output-len 128 \
  --num-prompts 200 \
  --dtype bfloat16

# Latency benchmark (single request, no batching)
python benchmarks/benchmark_latency.py \
  --model mistralai/Mistral-7B-Instruct-v0.3 \
  --input-len 512 \
  --output-len 128 \
  --num-iters 50

# Serving benchmark (tests the HTTP API with concurrent requests)
python benchmarks/benchmark_serving.py \
  --backend openai-chat \
  --base-url http://localhost:8000 \
  --model mistralai/Mistral-7B-Instruct-v0.3 \
  --request-rate 10 \     # Requests per second (Poisson arrival)
  --num-prompts 200 \
  --sharegpt-path ./ShareGPT_V3_unfiltered_cleaned_split.json  # Realistic prompts

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

import asyncio
import time
import statistics
import aiohttp
from dataclasses import dataclass

@dataclass
class BenchmarkResult:
    ttft_ms: float
    total_ms: float
    output_tokens: int
    tpot_ms: float

async def single_request(
    session: aiohttp.ClientSession,
    url: str,
    prompt: str,
    max_tokens: int = 256,
) -> BenchmarkResult:
    payload = {
        "model": "mistral-7b",
        "messages": [{"role": "user", "content": prompt}],
        "max_tokens": max_tokens,
        "stream": True,
    }

    start = time.perf_counter()
    ttft = None
    output_tokens = 0

    async with session.post(f"{url}/v1/chat/completions", json=payload) as resp:
        async for line in resp.content:
            line = line.decode().strip()
            if not line.startswith("data: "):
                continue
            data = line[6:]
            if data == "[DONE]":
                break

            if ttft is None:
                ttft = (time.perf_counter() - start) * 1000
            output_tokens += 1

    total_ms = (time.perf_counter() - start) * 1000
    tpot_ms = (total_ms - ttft) / max(output_tokens - 1, 1) if output_tokens > 1 else 0

    return BenchmarkResult(
        ttft_ms=ttft or 0,
        total_ms=total_ms,
        output_tokens=output_tokens,
        tpot_ms=tpot_ms,
    )

async def benchmark(url: str, concurrency: int = 10, total_requests: int = 100):
    prompts = [
        "Explain Kubernetes networking in detail.",
        "Write a Python function to parse a CIDR range.",
        "What is the difference between TCP and UDP?",
        "Describe the CAP theorem with examples.",
    ] * (total_requests // 4 + 1)
    prompts = prompts[:total_requests]

    semaphore = asyncio.Semaphore(concurrency)
    results = []

    async def run_with_semaphore(prompt):
        async with semaphore:
            async with aiohttp.ClientSession() as session:
                return await single_request(session, url, prompt)

    start = time.perf_counter()
    tasks = [run_with_semaphore(p) for p in prompts]
    results = await asyncio.gather(*tasks)
    wall_time = time.perf_counter() - start

    ttfts = [r.ttft_ms for r in results]
    tpots = [r.tpot_ms for r in results if r.tpot_ms > 0]
    total_tokens = sum(r.output_tokens for r in results)

    print(f"\n=== Benchmark Results ===")
    print(f"Concurrency:     {concurrency}")
    print(f"Total requests:  {total_requests}")
    print(f"Wall time:       {wall_time:.1f}s")
    print(f"Throughput:      {total_tokens / wall_time:.1f} tok/s")
    print(f"\nTTFT (ms):")
    print(f"  P50:  {statistics.median(ttfts):.0f}")
    print(f"  P95:  {sorted(ttfts)[int(len(ttfts)*0.95)]:.0f}")
    print(f"  P99:  {sorted(ttfts)[int(len(ttfts)*0.99)]:.0f}")
    print(f"\nTPOT (ms/token):")
    print(f"  P50:  {statistics.median(tpots):.1f}")
    print(f"  P95:  {sorted(tpots)[int(len(tpots)*0.95)]:.1f}")

asyncio.run(benchmark("http://localhost:8000", concurrency=16, total_requests=200))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

# docker-compose.yml
services:
  vllm:
    image: vllm/vllm-openai:latest
    runtime: nvidia
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - HF_TOKEN=${HF_TOKEN}
    volumes:
      - huggingface-cache:/root/.cache/huggingface
    command: >
      --model mistralai/Mistral-7B-Instruct-v0.3
      --dtype bfloat16
      --max-model-len 8192
      --gpu-memory-utilization 0.90
      --enable-prefix-caching
      --max-num-seqs 128
      --api-key ${VLLM_API_KEY}
      --served-model-name mistral-7b
    ports:
      - "8000:8000"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 120s  # Model loading takes time
    restart: unless-stopped
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]

  nginx:
    image: nginx:alpine
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
    ports:
      - "80:80"
    depends_on:
      vllm:
        condition: service_healthy

volumes:
  huggingface-cache:

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104

apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-inference
  namespace: ai-inference
spec:
  replicas: 2
  selector:
    matchLabels:
      app: vllm-inference
  template:
    metadata:
      labels:
        app: vllm-inference
    spec:
      containers:
      - name: vllm
        image: vllm/vllm-openai:latest
        args:
        - "--model"
        - "mistralai/Mistral-7B-Instruct-v0.3"
        - "--dtype"
        - "bfloat16"
        - "--max-model-len"
        - "8192"
        - "--gpu-memory-utilization"
        - "0.90"
        - "--enable-prefix-caching"
        - "--max-num-seqs"
        - "128"
        env:
        - name: HF_TOKEN
          valueFrom:
            secretKeyRef:
              name: hf-credentials
              key: token
        - name: VLLM_API_KEY
          valueFrom:
            secretKeyRef:
              name: vllm-credentials
              key: api-key
        ports:
        - containerPort: 8000
        resources:
          requests:
            nvidia.com/gpu: "1"
            memory: "24Gi"
            cpu: "4"
          limits:
            nvidia.com/gpu: "1"
            memory: "32Gi"
            cpu: "8"
        livenessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 120
          periodSeconds: 30
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 120
          periodSeconds: 10
        volumeMounts:
        - name: hf-cache
          mountPath: /root/.cache/huggingface
      volumes:
      - name: hf-cache
        persistentVolumeClaim:
          claimName: hf-model-cache
      nodeSelector:
        nvidia.com/gpu.product: NVIDIA-A100-SXM4-80GB
---
apiVersion: v1
kind: Service
metadata:
  name: vllm-inference
  namespace: ai-inference
spec:
  selector:
    app: vllm-inference
  ports:
  - port: 80
    targetPort: 8000
---
# KEDA autoscaling based on queue depth
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: vllm-scaledobject
  namespace: ai-inference
spec:
  scaleTargetRef:
    name: vllm-inference
  minReplicaCount: 1
  maxReplicaCount: 8
  triggers:
  - type: prometheus
    metadata:
      serverAddress: http://prometheus:9090
      metricName: vllm_num_requests_waiting
      threshold: "10"
      query: avg(vllm:num_requests_waiting:rate5m)

1
2
3
4
5
6

# prometheus scrape config
scrape_configs:
  - job_name: vllm
    static_configs:
      - targets: ['vllm:8000']
    metrics_path: /metrics

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# GPU KV cache utilization (alert if >90%)
vllm:gpu_cache_usage_perc > 0.90

# Request queue depth (alert if requests piling up)
vllm:num_requests_waiting > 20

# P99 TTFT (alert if latency spikes)
histogram_quantile(0.99, rate(vllm:time_to_first_token_seconds_bucket[5m])) > 2.0

# Generation throughput (alert if drops significantly)
rate(vllm:generation_tokens_total[5m]) < 100

# Request error rate
rate(vllm:request_failure_total[5m]) / rate(vllm:request_success_total[5m]) > 0.01

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# Grafana dashboard panels
- title: "Token Throughput"
  expr: rate(vllm:generation_tokens_total[1m])

- title: "P50/P95/P99 TTFT"
  expr: |
    histogram_quantile(0.50, rate(vllm:time_to_first_token_seconds_bucket[5m]))
    histogram_quantile(0.95, rate(vllm:time_to_first_token_seconds_bucket[5m]))
    histogram_quantile(0.99, rate(vllm:time_to_first_token_seconds_bucket[5m]))

- title: "KV Cache Utilization"
  expr: vllm:gpu_cache_usage_perc * 100

- title: "Active vs Waiting Requests"
  expr: |
    vllm:num_requests_running
    vllm:num_requests_waiting

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# Reduce context length
--max-model-len 4096  # instead of 32768

# Reduce memory reservation
--gpu-memory-utilization 0.80  # instead of 0.90

# Use quantization
--quantization awq

# Add a GPU
--tensor-parallel-size 2

1
2
3
4

# Pre-warm with a dummy request after startup
curl -s http://localhost:8000/v1/completions \
  -d '{"model": "m", "prompt": "warmup", "max_tokens": 1}' \
  -H "Content-Type: application/json"

1
2
3

# Ensure proxy isn't buffering
proxy_buffering off;  # nginx
# Or bypass proxy for inference endpoint

1
2
3
4
5

# Build with CPU-specific optimizations
CMAKE_ARGS="-DGGML_AVX2=ON -DGGML_FMA=ON -DGGML_F16C=ON" pip install llama-cpp-python

# Check CPU features
grep -m1 flags /proc/cpuinfo | grep -o 'avx2\|avx512f'

1
2
3
4

# Enable RoPE scaling for longer contexts
vllm serve model \
  --rope-scaling '{"type": "dynamic", "factor": 2.0}' \
  --max-model-len 16384  # 2× the model's native 8192