awk: The Text Processing Powerhouse

1

awk 'pattern { action }' file

1
2
3
4
5
6
7
8

# Print first column
awk '{ print $1 }' file.txt

# Print first and third columns
awk '{ print $1, $3 }' file.txt

# Print entire line
awk '{ print $0 }' file.txt

1
2
3
4
5

# CSV file
awk -F',' '{ print $1, $2 }' data.csv

# Colon-separated (like /etc/passwd)
awk -F':' '{ print $1, $3 }' /etc/passwd

1
2
3
4
5

# Lines containing "error"
awk '/error/ { print }' logfile.txt

# Lines starting with #
awk '/^#/ { print }' config.txt

1
2
3
4
5

# Third column greater than 100
awk '$3 > 100 { print }' data.txt

# First column equals "admin"
awk '$1 == "admin" { print }' users.txt

1
2
3
4
5
6
7
8

# AND
awk '$1 == "error" && $2 > 5 { print }' file.txt

# OR
awk '$1 == "warning" || $1 == "error" { print }' file.txt

# NOT
awk '!/^#/ { print }' file.txt  # Lines not starting with #

1
2
3
4
5
6
7
8

# Print line number and line
awk '{ print NR, $0 }' file.txt

# Print last column
awk '{ print $NF }' file.txt

# Print second-to-last column
awk '{ print $(NF-1) }' file.txt

1
2
3
4
5
6
7
8

# Sum third column
awk '{ sum += $3 } END { print sum }' data.txt

# Average
awk '{ sum += $3; count++ } END { print sum/count }' data.txt

# Calculate percentage
awk '{ print $1, ($2/$3)*100 "%" }' data.txt

1
2
3
4

# Header and footer
awk 'BEGIN { print "Name\tScore" }
     { print $1, $2 }
     END { print "Total lines:", NR }' data.txt

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# Length
awk '{ print length($1) }' file.txt

# Substring
awk '{ print substr($1, 1, 3) }' file.txt  # First 3 chars

# Find position
awk '{ print index($0, "error") }' file.txt

# Replace
awk '{ gsub(/old/, "new"); print }' file.txt

# Split
awk '{ split($1, arr, ":"); print arr[1] }' file.txt

1
2
3
4
5

# printf for formatting
awk '{ printf "%-10s %5d\n", $1, $2 }' file.txt

# Custom separator
awk -v OFS="\t" '{ print $1, $2, $3 }' file.txt

1
2
3
4
5

# Count requests per IP
awk '{ print $1 }' access.log | sort | uniq -c | sort -rn | head

# Using awk for everything
awk '{ ips[$1]++ } END { for (ip in ips) print ips[ip], ip }' access.log | sort -rn

1
2
3
4
5

# List users and shells
awk -F':' '{ print $1, $7 }' /etc/passwd

# Users with bash shell
awk -F':' '$7 ~ /bash/ { print $1 }' /etc/passwd

1
2
3
4
5

# Sum a column (skip header)
awk -F',' 'NR > 1 { sum += $3 } END { print sum }' data.csv

# Filter rows
awk -F',' '$2 > 1000 { print }' data.csv

1
2

# Count errors by hour
awk '/error/ { print substr($1, 1, 13) }' app.log | sort | uniq -c

1
2
3
4
5

# Top 5 processes by memory
ps aux | awk 'NR>1 { print $4, $11 }' | sort -rn | head -5

# Disk usage over 80%
df -h | awk 'NR>1 && $5+0 > 80 { print $1, $5 }'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

#!/usr/bin/awk -f
# script.awk

BEGIN {
    FS = ","
    print "Processing CSV..."
}

NR == 1 {
    for (i = 1; i <= NF; i++) {
        header[i] = $i
    }
    next
}

{
    for (i = 1; i <= NF; i++) {
        data[NR][i] = $i
    }
}

END {
    print "Processed", NR-1, "records"
}

1
2
3
4
5
6
7
8

awk '{ print $1 }'         # First column
awk -F',' '{ print $1 }'   # CSV first column
awk '/pattern/'            # Lines matching pattern
awk '$3 > 100'             # Third field > 100
awk '{ sum += $1 } END { print sum }'  # Sum column
awk 'NR > 1'               # Skip first line
awk '{ print NR, $0 }'     # Add line numbers
awk '{ print $NF }'        # Last column