Linux Performance Tuning and Monitoring

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# Overall usage
top
htop

# Per-CPU statistics
mpstat -P ALL 1

# Load average
uptime
cat /proc/loadavg
# 1.25 0.87 0.65 2/245 12345
# 1min 5min 15min running/total last_pid

# CPU info
lscpu
cat /proc/cpuinfo

1
2
3

# Check CPU count
nproc
grep -c processor /proc/cpuinfo

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# Find CPU-hungry processes
ps aux --sort=-%cpu | head

# Profile specific process
perf top -p PID
perf record -p PID sleep 30
perf report

# Trace system calls
strace -c command

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# CPU frequency scaling
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
echo "performance" | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor

# Process affinity
taskset -c 0,1 command      # Run on CPUs 0 and 1
taskset -p -c 0,1 PID       # Set for running process

# Nice value
nice -n 19 command          # Low priority
renice -n -10 -p PID        # Higher priority

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# Overview
free -h

# Detailed statistics
vmstat 1
# procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
#  r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st

# Per-process memory
ps aux --sort=-%mem | head
pmap -x PID

# Memory info
cat /proc/meminfo

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# Check for swapping
vmstat 1 | awk '{print $7, $8}'  # si, so columns
swapon --show

# OOM killer logs
dmesg | grep -i "out of memory"
journalctl -k | grep -i oom

# Memory leaks
valgrind --leak-check=full ./program

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# Swappiness (0-100, lower = less swapping)
cat /proc/sys/vm/swappiness
sudo sysctl vm.swappiness=10

# Drop caches (for testing)
sync; echo 3 | sudo tee /proc/sys/vm/drop_caches

# Huge pages
cat /proc/meminfo | grep -i huge
echo 1024 | sudo tee /proc/sys/vm/nr_hugepages

# Per-process limits
ulimit -v unlimited   # Virtual memory

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# Disk activity
iostat -xz 1
# Device  rrqm/s wrqm/s r/s  w/s  rkB/s wkB/s avgrq-sz avgqu-sz await r_await w_await svctm %util

# Per-process I/O
iotop
pidstat -d 1

# Disk usage
df -h
du -sh /path/*

# Filesystem activity
inotifywait -m /path

1
2
3
4
5
6
7
8

# Find I/O heavy processes
iotop -o

# Check disk health
sudo smartctl -a /dev/sda

# Block device info
lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,ROTA,DISC-GRAN

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# I/O scheduler
cat /sys/block/sda/queue/scheduler
echo "mq-deadline" | sudo tee /sys/block/sda/queue/scheduler

# Read-ahead
blockdev --getra /dev/sda
sudo blockdev --setra 4096 /dev/sda

# Filesystem mount options
# In /etc/fstab, add noatime, nodiratime
/dev/sda1  /  ext4  defaults,noatime  0  1

# SSD TRIM
sudo fstrim -v /

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# Interface statistics
ip -s link
sar -n DEV 1

# Bandwidth monitoring
iftop
nethogs
bmon

# Connection statistics
ss -s
netstat -s

# TCP analysis
ss -ti

1
2
3
4
5
6
7
8

# Check for errors/drops
ip -s link show eth0 | grep -E "(errors|dropped)"

# Socket buffer overflows
netstat -s | grep -i overflow

# Connection states
ss -tan | awk '{print $1}' | sort | uniq -c

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# TCP buffer sizes
sysctl net.core.rmem_max
sysctl net.core.wmem_max

# Increase buffers
sudo sysctl -w net.core.rmem_max=16777216
sudo sysctl -w net.core.wmem_max=16777216
sudo sysctl -w net.ipv4.tcp_rmem="4096 87380 16777216"
sudo sysctl -w net.ipv4.tcp_wmem="4096 87380 16777216"

# Connection backlog
sudo sysctl -w net.core.somaxconn=65535
sudo sysctl -w net.ipv4.tcp_max_syn_backlog=65535

# TIME_WAIT optimization
sudo sysctl -w net.ipv4.tcp_fin_timeout=15
sudo sysctl -w net.ipv4.tcp_tw_reuse=1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

# Install sysstat package
sudo apt install sysstat

# Enable data collection
sudo systemctl enable --now sysstat

# CPU history
sar -u

# Memory history
sar -r

# Disk history
sar -d

# Network history
sar -n DEV

# All metrics from specific time
sar -A -s 09:00:00 -e 10:00:00

1
2
3
4
5
6
7
8
9

# Install
sudo apt install pcp pcp-gui

# Start services
sudo systemctl enable --now pmcd pmlogger

# View metrics
pmstat
pmval kernel.all.load

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# View all
sysctl -a

# View specific
sysctl vm.swappiness

# Set temporarily
sudo sysctl vm.swappiness=10

# Set permanently
echo "vm.swappiness=10" | sudo tee /etc/sysctl.d/99-custom.conf
sudo sysctl --system

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

# /etc/sysctl.d/99-performance.conf

# Memory
vm.swappiness=10
vm.dirty_ratio=15
vm.dirty_background_ratio=5

# Network
net.core.somaxconn=65535
net.core.netdev_max_backlog=65535
net.ipv4.tcp_max_syn_backlog=65535
net.ipv4.tcp_fin_timeout=15
net.ipv4.tcp_tw_reuse=1
net.core.rmem_max=16777216
net.core.wmem_max=16777216

# File descriptors
fs.file-max=2097152

1
2
3
4
5

# sysbench
sysbench cpu --threads=4 run

# stress-ng
stress-ng --cpu 4 --timeout 60s --metrics-brief

1

sysbench memory --threads=4 run

1
2
3
4
5
6

# Sequential write
dd if=/dev/zero of=testfile bs=1G count=1 oflag=direct

# fio - flexible I/O tester
fio --name=randread --ioengine=libaio --iodepth=16 \
    --rw=randread --bs=4k --direct=1 --size=1G --numjobs=4

1
2
3
4
5
6

# iperf3
# Server
iperf3 -s

# Client
iperf3 -c server-ip -t 30

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

#!/bin/bash
echo "=== System Overview ==="
uptime
free -h

echo -e "\n=== CPU ==="
mpstat 1 3

echo -e "\n=== Memory ==="
vmstat 1 3

echo -e "\n=== Disk ==="
iostat -xz 1 3

echo -e "\n=== Network ==="
sar -n DEV 1 3

echo -e "\n=== Top Processes (CPU) ==="
ps aux --sort=-%cpu | head -5

echo -e "\n=== Top Processes (Memory) ==="
ps aux --sort=-%mem | head -5

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# CPU
top, htop, mpstat
perf top -p PID

# Memory
free -h, vmstat, pmap PID

# Disk
iostat -xz, iotop, df -h

# Network
ip -s link, ss -s, iftop

# System-wide
sar, dstat, glances