package main import ( "fmt" "sort" "strconv" "strings" "time" ) // The lines past which a reading is said as a finding, not only a number. const ( busyLine = 90.0 // a device with a request in flight this share of the window is saturated iowaitLine = 20.0 // the processors idle on I/O this share of their time stallLine = 10.0 // some task stalled on I/O this share of the window fullLine = 90.0 // a filesystem this full, or this short of inodes ) // reading is everything read at one moment of the window. type reading struct { cpu cpuTimes disks map[string]diskCounters psi PSI psiOK bool procs map[int]procIO cantRead int units map[string]cgroupIO } func (m *Machine) readNow(withProcesses bool, counted map[string]bool) (reading, error) { var r reading var err error if r.cpu, err = parseCPU(m.read("/proc/stat")); err != nil { return r, err } r.disks = parseDiskstats(m.read("/proc/diskstats")) r.psi, r.psiOK = parsePSI(m.read("/proc/pressure/io")) if withProcesses { r.procs, r.cantRead = m.processIO() r.units = m.unitIO(counted) } return r, nil } // Load is the machine's disk load over one window. type Load struct { WindowS float64 `json:"window_s"` Said []string `json:"said"` CPU CPU `json:"cpu"` Pressure any `json:"io_pressure"` Devices []Device `json:"devices"` Filesystems []Filesystem `json:"filesystems,omitempty"` Units []Process `json:"units,omitempty"` Processes []Process `json:"processes,omitempty"` Unreadable int `json:"processes_unreadable,omitempty"` Note string `json:"note,omitempty"` } // LoadOver reads the machine, waits the window, reads it again and says what happened between. // processes is how many of the busiest processes to name (0 for none); all keeps every device and // partition, not only the whole disks. func (m *Machine) LoadOver(window time.Duration, processes int, all, filesystems bool) (Load, error) { counted := m.bottomDevices() a, err := m.readNow(processes > 0, counted) if err != nil { return Load{}, err } m.Wait(window) b, err := m.readNow(processes > 0, counted) if err != nil { return Load{}, err } l := Load{WindowS: window.Seconds(), CPU: cpuShare(a.cpu, b.cpu)} mounts := map[string][]string{} if filesystems { l.Filesystems = m.Filesystems() for _, fs := range l.Filesystems { if fs.Device == "" { continue } mounts[fs.Device] = append(mounts[fs.Device], fs.Mount) if p := m.parentOf(fs.Device); p != fs.Device { mounts[p] = append(mounts[p], fs.Mount+" ("+fs.Device+")") } } } l.Devices = []Device{} for _, name := range m.blockDevices(b.disks, all) { start, ok := a.disks[name] if !ok { continue } d := deviceOver(name, start, b.disks[name], window) d.Name = m.mapperName(name) d.On = m.below(name) d.Mounts = mounts[name] l.Devices = append(l.Devices, d) } sort.SliceStable(l.Devices, func(i, j int) bool { return l.Devices[i].BusyPct > l.Devices[j].BusyPct }) if a.psiOK && b.psiOK { l.Pressure = psiOver(a.psi, b.psi, window) } else { l.Pressure = "not available: this kernel has no /proc/pressure/io (pressure stall information is off)" } if processes > 0 { l.Units = m.topUnits(a.units, b.units, window, "unit", processes) l.Processes = m.topProcesses(a.procs, b.procs, window, processes) l.Unreadable = b.cantRead if b.cantRead > 0 { l.Note = unreadableNote(b.cantRead) } } l.Said = say(l) return l, nil } func say(l Load) []string { var out []string c := l.CPU line := fmt.Sprintf("CPU: %.1f %% waiting on I/O, %.1f %% user, %.1f %% system, %.1f %% idle, over %g s on %d CPUs", c.IOWaitPct, c.UserPct, c.SystemPct, c.IdlePct, l.WindowS, c.CPUs) if c.IOWaitPct >= iowaitLine { line += " — HIGH: the processors sat idle waiting on storage" } out = append(out, line) if p, ok := l.Pressure.(PSI); ok { line := fmt.Sprintf("I/O pressure: some task stalled %.1f %% of the window (avg10 %.1f %%, avg60 %.1f %%, avg300 %.1f %%)", p.Some.WindowPct, p.Some.Avg10, p.Some.Avg60, p.Some.Avg300) if p.Full != nil { line += fmt.Sprintf("; every task stalled %.1f %%", p.Full.WindowPct) } if p.Some.WindowPct >= stallLine || p.Some.Avg60 >= stallLine { line += " — HIGH" } out = append(out, line) } for _, d := range l.Devices { name := d.Device if d.Name != "" { name += " (" + d.Name + ")" } line := fmt.Sprintf("%s: busy %.1f %%, read %.1f MiB/s (%.1f IOPS, %.1f ms wait), write %.1f MiB/s (%.1f IOPS, %.1f ms wait), queue depth %.1f", name, d.BusyPct, d.ReadMiBs, d.ReadIOPS, d.ReadWaitMs, d.WriteMiBs, d.WriteIOPS, d.WriteWaitMs, d.QueueDepth) if len(d.Mounts) > 0 { line += ", holds " + strings.Join(d.Mounts, ", ") } if d.BusyPct >= busyLine { line += " — SATURATED" } out = append(out, line) } for _, fs := range l.Filesystems { if fs.Unanswered != "" { out = append(out, fmt.Sprintf("%s: not read (%s)", fs.Mount, fs.Unanswered)) continue } if fs.UsedPct >= fullLine || fs.InodesUsed >= fullLine { out = append(out, fmt.Sprintf("%s: %.1f %% full (%.1f GiB of %.1f GiB, %.1f GiB free), inodes %.1f %% used — NEARLY FULL", fs.Mount, fs.UsedPct, fs.UsedGiB, fs.SizeGiB, fs.AvailGiB, fs.InodesUsed)) } } for i, u := range l.Units { if i == 3 { break } name := u.Unit if u.Container != "" { name = "container " + u.Container } out = append(out, fmt.Sprintf("busiest unit: %s read %.2f MiB/s, wrote %.2f MiB/s", name, u.ReadMiBs, u.WriteMiBs)) } for i, p := range l.Processes { if i == 3 { break } where := p.Unit if p.Container != "" { where = "container " + p.Container } out = append(out, fmt.Sprintf("busiest process: %s (pid %d, %s) read %.2f MiB/s, wrote %.2f MiB/s", p.Command, p.PID, orNone(where), p.ReadMiBs, p.WriteMiBs)) } return out } // unreadableNote says what a list of processes misses when the tool runner is not root. func unreadableNote(n int) string { return fmt.Sprintf("%d processes' I/O could not be read: they are another account's, and the tool runner is not root, "+ "so a busy one may be missing from the processes; the units, read from their cgroups, miss none", n) } func orNone(s string) string { if s == "" { return "no unit" } return s } // DeviceInfo is one disk as it is, and what it has done since the machine started. type DeviceInfo struct { Device string `json:"device"` Name string `json:"name,omitempty"` On []string `json:"on,omitempty"` Model string `json:"model,omitempty"` SizeGiB float64 `json:"size_gib"` Rotational bool `json:"rotational"` Scheduler string `json:"scheduler,omitempty"` QueueLimit int `json:"queue_limit,omitempty"` ReadGiB float64 `json:"read_since_boot_gib"` WrittenGiB float64 `json:"written_since_boot_gib"` BusySincePct float64 `json:"busy_since_boot_pct"` ReadWaitMs float64 `json:"read_wait_since_boot_ms"` WriteWaitMs float64 `json:"write_wait_since_boot_ms"` } // DevicesNow is every disk with its kind, its size, its scheduler and its totals since boot: the // question "is this a spinning disk" and "how much has it written" without a window. func (m *Machine) DevicesNow(all bool) ([]DeviceInfo, float64) { stats := parseDiskstats(m.read("/proc/diskstats")) uptime := 0.0 if f := strings.Fields(m.read("/proc/uptime")); len(f) > 0 { uptime, _ = strconv.ParseFloat(f[0], 64) } out := []DeviceInfo{} for _, name := range m.blockDevices(stats, all) { c := stats[name] base := "/sys/block/" + name if !m.exists(base) { base = "/sys/block/" + m.parentOf(name) + "/" + name } sectors, _ := strconv.ParseFloat(strings.TrimSpace(m.read(base+"/size")), 64) d := DeviceInfo{ Device: name, Name: m.mapperName(name), On: m.below(name), Model: strings.TrimSpace(m.read(base + "/device/model")), SizeGiB: round1(sectors * 512 / gib), Rotational: strings.TrimSpace(m.read(base+"/queue/rotational")) == "1", Scheduler: activeScheduler(m.read(base + "/queue/scheduler")), ReadGiB: round1(float64(c.readSectors) * 512 / gib), WrittenGiB: round1(float64(c.writeSectors) * 512 / gib), } d.QueueLimit, _ = strconv.Atoi(strings.TrimSpace(m.read(base + "/queue/nr_requests"))) if uptime > 0 { d.BusySincePct = round1(float64(c.busyMs) / (uptime * 1000) * 100) } if c.reads > 0 { d.ReadWaitMs = round1(float64(c.readMs) / float64(c.reads)) } if c.writes > 0 { d.WriteWaitMs = round1(float64(c.writeMs) / float64(c.writes)) } out = append(out, d) } return out, round1(uptime / 3600) } // activeScheduler is the bracketed one of "mq-deadline kyber [bfq] none". func activeScheduler(s string) string { if i := strings.Index(s, "["); i >= 0 { if j := strings.Index(s[i:], "]"); j > 0 { return s[i+1 : i+j] } } return strings.TrimSpace(s) }