308 lines
13 KiB
Go
308 lines
13 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
const store = "4f1c2a9e8b7d6c5a4f1c2a9e8b7d6c5a4f1c2a9e8b7d6c5a4f1c2a9e8b7d6c5a"
|
|
|
|
// aServerUnderWrites is a machine whose store writes 100 MiB in a 2 s window to an encrypted root on an
|
|
// NVMe disk: the shape of the control node in issue 312. Lines are as a 6.x kernel writes them, with
|
|
// the four discard and two flush fields at the end of /proc/diskstats.
|
|
func aServerUnderWrites(t *testing.T) *fake {
|
|
f := newFake(t)
|
|
f.file("/proc/stat", "cpu 1000 0 500 8000 100 0 0 0 0 0\ncpu0 500 0 250 4000 50 0 0 0 0 0\ncpu1 500 0 250 4000 50 0 0 0 0 0\nintr 1\n")
|
|
f.then("/proc/stat", "cpu 1100 0 550 8250 500 0 0 0 0 0\ncpu0 550 0 275 4125 250 0 0 0 0 0\ncpu1 550 0 275 4125 250 0 0 0 0 0\nintr 2\n")
|
|
|
|
f.file("/proc/diskstats", strings.Join([]string{
|
|
" 259 0 nvme0n1 1000 0 80000 500 2000 0 400000 4000 0 10000 20000 0 0 0 0 10 5",
|
|
" 259 1 nvme0n1p1 10 0 100 5 0 0 0 0 0 5 5 0 0 0 0 0 0",
|
|
" 259 2 nvme0n1p2 990 0 79900 495 2000 0 400000 4000 0 9995 19995 0 0 0 0 0 0",
|
|
" 254 0 dm-0 990 0 79900 600 2000 0 400000 5000 0 10000 21000 0 0 0 0 0 0",
|
|
" 8 0 sda 50 0 4000 900 0 0 0 0 0 800 900 0 0 0 0 0 0",
|
|
" 7 0 loop0 5 0 10 1 0 0 0 0 0 1 1 0 0 0 0 0 0",
|
|
" 252 0 zram0 100 0 800 1 900 0 7200 9 0 9 10 0 0 0 0 0 0",
|
|
}, "\n")+"\n")
|
|
f.then("/proc/diskstats", strings.Join([]string{
|
|
" 259 0 nvme0n1 1200 0 120960 900 3000 0 604800 29000 12 11960 45400 0 0 0 0 12 6",
|
|
" 259 1 nvme0n1p1 10 0 100 5 0 0 0 0 0 5 5 0 0 0 0 0 0",
|
|
" 259 2 nvme0n1p2 1190 0 120860 895 3000 0 604800 29000 12 11955 45395 0 0 0 0 0 0",
|
|
" 254 0 dm-0 1190 0 120860 1100 3000 0 604800 32000 14 11980 50000 0 0 0 0 0 0",
|
|
" 8 0 sda 50 0 4000 900 0 0 0 0 0 800 900 0 0 0 0 0 0",
|
|
" 7 0 loop0 5 0 10 1 0 0 0 0 0 1 1 0 0 0 0 0 0",
|
|
" 252 0 zram0 900 0 7200 9 9000 0 72000 90 0 90 100 0 0 0 0 0 0",
|
|
}, "\n")+"\n")
|
|
|
|
f.file("/proc/pressure/io", "some avg10=38.50 avg60=12.00 avg300=3.10 total=1000000\nfull avg10=25.00 avg60=8.00 avg300=2.00 total=500000\n")
|
|
f.then("/proc/pressure/io", "some avg10=44.00 avg60=14.20 avg300=3.50 total=1900000\nfull avg10=29.00 avg60=9.10 avg300=2.20 total=1100000\n")
|
|
|
|
f.file("/sys/block/nvme0n1/size", "1000215216\n")
|
|
f.file("/sys/block/nvme0n1/queue/rotational", "0\n")
|
|
f.file("/sys/block/nvme0n1/queue/scheduler", "[none] mq-deadline kyber\n")
|
|
f.file("/sys/block/nvme0n1/queue/nr_requests", "1023\n")
|
|
f.file("/sys/block/nvme0n1/device/model", "Example NVMe 512GB \n")
|
|
f.dir("/sys/block/nvme0n1/nvme0n1p1")
|
|
f.dir("/sys/block/nvme0n1/nvme0n1p2")
|
|
f.file("/sys/block/dm-0/dm/name", "root\n")
|
|
f.dir("/sys/block/dm-0/slaves/nvme0n1p2")
|
|
f.file("/sys/block/dm-0/size", "999000000\n")
|
|
f.file("/sys/block/sda/size", "7814037168\n")
|
|
f.file("/sys/block/sda/queue/rotational", "1\n")
|
|
f.file("/sys/block/sda/queue/scheduler", "mq-deadline [bfq] none\n")
|
|
f.dir("/sys/block/sda/sda1")
|
|
f.dir("/sys/block/loop0")
|
|
f.dir("/sys/block/zram0")
|
|
f.file("/proc/uptime", "36000.00 70000.00\n")
|
|
|
|
f.file("/proc/self/mounts", strings.Join([]string{
|
|
"proc /proc proc rw,nosuid 0 0",
|
|
"/dev/mapper/root / ext4 rw,relatime 0 0",
|
|
"tmpfs /tmp tmpfs rw 0 0",
|
|
"/dev/nvme0n1p1 /boot vfat rw 0 0",
|
|
"/dev/mapper/root /var/lib/docker ext4 rw,relatime 0 0",
|
|
"overlay /var/lib/docker/overlay2/abc/merged overlay rw 0 0",
|
|
`/dev/sda1 /mnt/back\040up ext4 rw 0 0`,
|
|
}, "\n")+"\n")
|
|
f.space["/"] = FSStat{Size: 100 * gib, Free: 10 * gib, Avail: 5 * gib, Files: 1000, FilesFree: 600}
|
|
f.space["/boot"] = FSStat{Size: gib, Free: gib / 2, Avail: gib / 2, Files: 0}
|
|
|
|
proc := func(pid, comm, cgroup string, r0, w0, r1, w1 int) {
|
|
f.file("/proc/"+pid+"/comm", comm+"\n")
|
|
f.file("/proc/"+pid+"/cgroup", cgroup+"\n")
|
|
f.file("/proc/"+pid+"/io", ioFile(r0, w0))
|
|
f.then("/proc/"+pid+"/io", ioFile(r1, w1))
|
|
}
|
|
proc("100", "postgres", "0::/system.slice/docker-"+store+".scope", 0, 0, 0, 100*mib)
|
|
proc("101", "postgres", "0::/system.slice/docker-"+store+".scope", 0, 0, 10*mib, 0)
|
|
proc("200", "systemd-journal", "0::/system.slice/systemd-journald.service", 0, 0, 0, 2*mib)
|
|
proc("201", "sleep", "0::/user.slice/user-1000.slice/session-1.scope", 0, 0, 0, 0)
|
|
f.dir("/proc/300") // another account's: its io is not readable
|
|
f.then("/proc/400/io", ioFile(0, 50*mib))
|
|
cg := func(path, at0, at1 string) {
|
|
f.file("/sys/fs/cgroup/"+path+"/io.stat", at0)
|
|
f.then("/sys/fs/cgroup/"+path+"/io.stat", at1)
|
|
}
|
|
zero := "259:0 rbytes=0 wbytes=0 rios=0 wios=0 dbytes=0 dios=0\n254:0 rbytes=0 wbytes=0 rios=0 wios=0 dbytes=0 dios=0\n7:0 rbytes=0 wbytes=0 rios=0 wios=0 dbytes=0 dios=0\n"
|
|
// The store's writes are charged to the encrypted volume and again to the disk under it; a loop
|
|
// device's to the filesystem its file is on. Only the disk is counted.
|
|
cg("system.slice/docker-"+store+".scope", zero,
|
|
"259:0 rbytes="+itoa(10*mib)+" wbytes="+itoa(100*mib)+" rios=200 wios=1000 dbytes=0 dios=0\n"+
|
|
"254:0 rbytes="+itoa(10*mib)+" wbytes="+itoa(100*mib)+" rios=200 wios=1000 dbytes=0 dios=0\n"+
|
|
"7:0 rbytes=0 wbytes="+itoa(900*mib)+" rios=0 wios=9000 dbytes=0 dios=0\n")
|
|
cg("system.slice/systemd-journald.service", zero, "259:0 rbytes=0 wbytes="+itoa(2*mib)+" rios=0 wios=20 dbytes=0 dios=0\n")
|
|
cg("system.slice/idle.service", zero, zero)
|
|
// io.stat counts a cgroup's descendants: the user's manager holds the application's bytes again.
|
|
cg("user.slice/user-1000.slice/user@1000.service", zero, "259:0 rbytes=0 wbytes="+itoa(5*mib)+" rios=0 wios=50 dbytes=0 dios=0\n")
|
|
cg("user.slice/user-1000.slice/user@1000.service/app.slice/editor.scope", zero, "259:0 rbytes=0 wbytes="+itoa(5*mib)+" rios=0 wios=50 dbytes=0 dios=0\n")
|
|
f.answers["docker ps --no-trunc --format {{.ID}}\t{{.Names}}"] = store + "\tstore-check\n"
|
|
return f
|
|
}
|
|
|
|
func ioFile(read, write int) string {
|
|
return "rchar: 99999999\nwchar: 99999999\nsyscr: 1\nsyscw: 1\nread_bytes: " + itoa(read) + "\nwrite_bytes: " + itoa(write) + "\ncancelled_write_bytes: 0\n"
|
|
}
|
|
|
|
func itoa(n int) string {
|
|
b, _ := json.Marshal(n)
|
|
return string(b)
|
|
}
|
|
|
|
func device(t *testing.T, l Load, name string) Device {
|
|
t.Helper()
|
|
for _, d := range l.Devices {
|
|
if d.Device == name {
|
|
return d
|
|
}
|
|
}
|
|
t.Fatalf("no device %s in %+v", name, l.Devices)
|
|
return Device{}
|
|
}
|
|
|
|
func TestAWindowSaysIOWaitEachDisksLoadAndPressure(t *testing.T) {
|
|
f := aServerUnderWrites(t)
|
|
l, err := f.machine().LoadOver(2*time.Second, 5, false, true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if f.waited != 2*time.Second {
|
|
t.Errorf("waited %s, not the window", f.waited)
|
|
}
|
|
if l.CPU != (CPU{CPUs: 2, IOWaitPct: 50, UserPct: 12.5, SystemPct: 6.3, IdlePct: 31.3}) {
|
|
t.Errorf("cpu %+v", l.CPU)
|
|
}
|
|
|
|
n := device(t, l, "nvme0n1")
|
|
want := Device{Device: "nvme0n1", Mounts: []string{"/boot (nvme0n1p1)"}, BusyPct: 98, ReadMiBs: 10, WriteMiBs: 50,
|
|
ReadIOPS: 100, WriteIOPS: 500, ReadWaitMs: 2, WriteWaitMs: 25, QueueDepth: 12.7, InFlight: 12}
|
|
if got, _ := json.Marshal(n); string(got) != string(must(json.Marshal(want))) {
|
|
t.Errorf("nvme0n1\n got %s\nwant %s", got, must(json.Marshal(want)))
|
|
}
|
|
root := device(t, l, "dm-0")
|
|
if root.Name != "root" || strings.Join(root.On, ",") != "nvme0n1p2" || strings.Join(root.Mounts, ",") != "/" {
|
|
t.Errorf("the encrypted root is said by its name, what it is on and what it holds: %+v", root)
|
|
}
|
|
if sda := device(t, l, "sda"); sda.BusyPct != 0 || sda.ReadMiBs != 0 {
|
|
t.Errorf("an idle disk: %+v", sda)
|
|
}
|
|
for _, d := range l.Devices {
|
|
if d.Device == "loop0" || d.Device == "zram0" || d.Device == "nvme0n1p2" {
|
|
t.Errorf("%s is not a disk and was said", d.Device)
|
|
}
|
|
}
|
|
if l.Devices[0].Device != "dm-0" && l.Devices[0].Device != "nvme0n1" {
|
|
t.Errorf("the busiest first: %s", l.Devices[0].Device)
|
|
}
|
|
|
|
p, ok := l.Pressure.(PSI)
|
|
if !ok || p.Some.WindowPct != 45 || p.Full == nil || p.Full.WindowPct != 30 || p.Some.Avg10 != 44 {
|
|
t.Errorf("pressure %+v", l.Pressure)
|
|
}
|
|
|
|
if len(l.Processes) != 3 {
|
|
t.Fatalf("three processes moved bytes in the window, one started inside it: %+v", l.Processes)
|
|
}
|
|
top := l.Processes[0]
|
|
if top.PID != 100 || top.Command != "postgres" || top.Container != "store-check" || top.WriteMiBs != 50 || top.WriteMiB != 100 {
|
|
t.Errorf("the busiest process, named by its container: %+v", top)
|
|
}
|
|
if j := l.Processes[2]; j.Unit != "systemd-journald.service" || j.Container != "" || j.WriteMiBs != 1 {
|
|
t.Errorf("a service's process, named by its unit: %+v", j)
|
|
}
|
|
if len(l.Units) != 3 || l.Units[0].WriteIOPS != 500 || l.Units[2].Unit != "systemd-journald.service" {
|
|
t.Errorf("units %+v", l.Units)
|
|
}
|
|
if l.Unreadable != 1 || !strings.Contains(l.Note, "1 processes' I/O could not be read") {
|
|
t.Errorf("an unreadable process is counted and said: %d %q", l.Unreadable, l.Note)
|
|
}
|
|
|
|
said := strings.Join(l.Said, "\n")
|
|
for _, s := range []string{
|
|
"CPU: 50.0 % waiting on I/O", "HIGH: the processors sat idle waiting on storage",
|
|
"I/O pressure: some task stalled 45.0 % of the window", "every task stalled 30.0 %",
|
|
"nvme0n1: busy 98.0 %, read 10.0 MiB/s (100.0 IOPS, 2.0 ms wait), write 50.0 MiB/s (500.0 IOPS, 25.0 ms wait), queue depth 12.7, holds /boot (nvme0n1p1) — SATURATED",
|
|
"dm-0 (root): busy 99.0 %", "holds / — SATURATED",
|
|
"/: 94.7 % full (90.0 GiB of 100.0 GiB, 5.0 GiB free)", "NEARLY FULL",
|
|
`/mnt/back up: not read (no such filesystem)`,
|
|
"busiest process: postgres (pid 100, container store-check) read 0.00 MiB/s, wrote 50.00 MiB/s",
|
|
"busiest unit: container store-check read 5.00 MiB/s, wrote 50.00 MiB/s",
|
|
"busiest unit: editor.scope read 0.00 MiB/s, wrote 2.50 MiB/s",
|
|
} {
|
|
if !strings.Contains(said, s) {
|
|
t.Errorf("said lacks %q:\n%s", s, said)
|
|
}
|
|
}
|
|
}
|
|
|
|
func must(b []byte, err error) []byte {
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return b
|
|
}
|
|
|
|
func TestAllSaysEveryDeviceAndPartition(t *testing.T) {
|
|
l, err := aServerUnderWrites(t).machine().LoadOver(2*time.Second, 0, true, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var names []string
|
|
for _, d := range l.Devices {
|
|
names = append(names, d.Device)
|
|
}
|
|
sort.Strings(names)
|
|
if strings.Join(names, ",") != "dm-0,loop0,nvme0n1,nvme0n1p1,nvme0n1p2,sda,zram0" {
|
|
t.Errorf("all: %v", names)
|
|
}
|
|
if l.Processes != nil || l.Filesystems != nil {
|
|
t.Errorf("processes=0 names none and reads none: %+v %+v", l.Processes, l.Filesystems)
|
|
}
|
|
}
|
|
|
|
func TestAKernelWithoutPressureStallSaysSo(t *testing.T) {
|
|
f := aServerUnderWrites(t)
|
|
os.Remove(f.root + "/proc/pressure/io")
|
|
delete(f.later, "/proc/pressure/io")
|
|
l, err := f.machine().LoadOver(time.Second, 0, false, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if s, ok := l.Pressure.(string); !ok || !strings.HasPrefix(s, "not available") {
|
|
t.Errorf("pressure %+v", l.Pressure)
|
|
}
|
|
}
|
|
|
|
func TestACounterThatWentBackwardsReadsAsNothingNotAsAnEnormousNumber(t *testing.T) {
|
|
a := diskCounters{reads: 100, readSectors: 1000, busyMs: 5000}
|
|
b := diskCounters{reads: 10, readSectors: 100, busyMs: 10}
|
|
d := deviceOver("sdx", a, b, time.Second)
|
|
if d.ReadIOPS != 0 || d.ReadMiBs != 0 || d.BusyPct != 0 {
|
|
t.Errorf("a reset counter: %+v", d)
|
|
}
|
|
if c := cpuShare(cpuTimes{iowait: 10, idle: 10}, cpuTimes{iowait: 5, idle: 10}); c.IOWaitPct != 0 {
|
|
t.Errorf("cpu over no time: %+v", c)
|
|
}
|
|
}
|
|
|
|
func TestBusyIsHeldAtAHundred(t *testing.T) {
|
|
d := deviceOver("sdx", diskCounters{}, diskCounters{busyMs: 1500}, time.Second)
|
|
if d.BusyPct != 100 {
|
|
t.Errorf("busy %v", d.BusyPct)
|
|
}
|
|
}
|
|
|
|
func TestAStatWithoutACPULineIsAnError(t *testing.T) {
|
|
if _, err := parseCPU("intr 1\n"); err == nil {
|
|
t.Error("no cpu line was accepted")
|
|
}
|
|
if _, err := parseCPU("cpu 1 2 3\n"); err == nil {
|
|
t.Error("a short cpu line was accepted")
|
|
}
|
|
f := newFake(t)
|
|
if _, err := f.machine().LoadOver(time.Second, 0, false, false); err == nil {
|
|
t.Error("a machine without /proc/stat answered")
|
|
}
|
|
}
|
|
|
|
func TestDevicesSayWhatEachDiskIsAndHasDoneSinceBoot(t *testing.T) {
|
|
ds, up := aServerUnderWrites(t).machine().DevicesNow(false)
|
|
if up != 10 {
|
|
t.Errorf("up %v hours", up)
|
|
}
|
|
by := map[string]DeviceInfo{}
|
|
for _, d := range ds {
|
|
by[d.Device] = d
|
|
}
|
|
n := by["nvme0n1"]
|
|
if n.Model != "Example NVMe 512GB" || n.Rotational || n.Scheduler != "none" || n.QueueLimit != 1023 || n.SizeGiB != 476.9 {
|
|
t.Errorf("nvme0n1 %+v", n)
|
|
}
|
|
if n.WrittenGiB != 0.2 || n.ReadWaitMs != 0.5 || n.WriteWaitMs != 2 || n.BusySincePct != 0 {
|
|
t.Errorf("nvme0n1 since boot %+v", n)
|
|
}
|
|
if s := by["sda"]; !s.Rotational || s.Scheduler != "bfq" || s.SizeGiB != 3726 || s.BusySincePct != 0 {
|
|
t.Errorf("sda %+v", s)
|
|
}
|
|
if r := by["dm-0"]; r.Name != "root" || strings.Join(r.On, ",") != "nvme0n1p2" {
|
|
t.Errorf("dm-0 %+v", r)
|
|
}
|
|
if _, ok := by["zram0"]; ok {
|
|
t.Error("zram is memory, not a disk")
|
|
}
|
|
}
|
|
|
|
func TestAPartitionsSizeIsReadUnderItsDisk(t *testing.T) {
|
|
f := aServerUnderWrites(t)
|
|
f.file("/sys/block/nvme0n1/nvme0n1p1/size", "2097152\n")
|
|
ds, _ := f.machine().DevicesNow(true)
|
|
for _, d := range ds {
|
|
if d.Device == "nvme0n1p1" && d.SizeGiB != 1 {
|
|
t.Errorf("nvme0n1p1 %+v", d)
|
|
}
|
|
}
|
|
}
|