265 lines
8.8 KiB
Go
265 lines
8.8 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// The lines past which a reading is said as a finding, not only a number.
|
|
const (
|
|
busyLine = 90.0 // a device with a request in flight this share of the window is saturated
|
|
iowaitLine = 20.0 // the processors idle on I/O this share of their time
|
|
stallLine = 10.0 // some task stalled on I/O this share of the window
|
|
fullLine = 90.0 // a filesystem this full, or this short of inodes
|
|
)
|
|
|
|
// reading is everything read at one moment of the window.
|
|
type reading struct {
|
|
cpu cpuTimes
|
|
disks map[string]diskCounters
|
|
psi PSI
|
|
psiOK bool
|
|
procs map[int]procIO
|
|
cantRead int
|
|
units map[string]cgroupIO
|
|
}
|
|
|
|
func (m *Machine) readNow(withProcesses bool, counted map[string]bool) (reading, error) {
|
|
var r reading
|
|
var err error
|
|
if r.cpu, err = parseCPU(m.read("/proc/stat")); err != nil {
|
|
return r, err
|
|
}
|
|
r.disks = parseDiskstats(m.read("/proc/diskstats"))
|
|
r.psi, r.psiOK = parsePSI(m.read("/proc/pressure/io"))
|
|
if withProcesses {
|
|
r.procs, r.cantRead = m.processIO()
|
|
r.units = m.unitIO(counted)
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
// Load is the machine's disk load over one window.
|
|
type Load struct {
|
|
WindowS float64 `json:"window_s"`
|
|
Said []string `json:"said"`
|
|
CPU CPU `json:"cpu"`
|
|
Pressure any `json:"io_pressure"`
|
|
Devices []Device `json:"devices"`
|
|
Filesystems []Filesystem `json:"filesystems,omitempty"`
|
|
Units []Process `json:"units,omitempty"`
|
|
Processes []Process `json:"processes,omitempty"`
|
|
Unreadable int `json:"processes_unreadable,omitempty"`
|
|
Note string `json:"note,omitempty"`
|
|
}
|
|
|
|
// LoadOver reads the machine, waits the window, reads it again and says what happened between.
|
|
// processes is how many of the busiest processes to name (0 for none); all keeps every device and
|
|
// partition, not only the whole disks.
|
|
func (m *Machine) LoadOver(window time.Duration, processes int, all, filesystems bool) (Load, error) {
|
|
counted := m.bottomDevices()
|
|
a, err := m.readNow(processes > 0, counted)
|
|
if err != nil {
|
|
return Load{}, err
|
|
}
|
|
m.Wait(window)
|
|
b, err := m.readNow(processes > 0, counted)
|
|
if err != nil {
|
|
return Load{}, err
|
|
}
|
|
l := Load{WindowS: window.Seconds(), CPU: cpuShare(a.cpu, b.cpu)}
|
|
|
|
mounts := map[string][]string{}
|
|
if filesystems {
|
|
l.Filesystems = m.Filesystems()
|
|
for _, fs := range l.Filesystems {
|
|
if fs.Device == "" {
|
|
continue
|
|
}
|
|
mounts[fs.Device] = append(mounts[fs.Device], fs.Mount)
|
|
if p := m.parentOf(fs.Device); p != fs.Device {
|
|
mounts[p] = append(mounts[p], fs.Mount+" ("+fs.Device+")")
|
|
}
|
|
}
|
|
}
|
|
l.Devices = []Device{}
|
|
for _, name := range m.blockDevices(b.disks, all) {
|
|
start, ok := a.disks[name]
|
|
if !ok {
|
|
continue
|
|
}
|
|
d := deviceOver(name, start, b.disks[name], window)
|
|
d.Name = m.mapperName(name)
|
|
d.On = m.below(name)
|
|
d.Mounts = mounts[name]
|
|
l.Devices = append(l.Devices, d)
|
|
}
|
|
sort.SliceStable(l.Devices, func(i, j int) bool { return l.Devices[i].BusyPct > l.Devices[j].BusyPct })
|
|
|
|
if a.psiOK && b.psiOK {
|
|
l.Pressure = psiOver(a.psi, b.psi, window)
|
|
} else {
|
|
l.Pressure = "not available: this kernel has no /proc/pressure/io (pressure stall information is off)"
|
|
}
|
|
if processes > 0 {
|
|
l.Units = m.topUnits(a.units, b.units, window, "unit", processes)
|
|
l.Processes = m.topProcesses(a.procs, b.procs, window, processes)
|
|
l.Unreadable = b.cantRead
|
|
if b.cantRead > 0 {
|
|
l.Note = unreadableNote(b.cantRead)
|
|
}
|
|
}
|
|
l.Said = say(l)
|
|
return l, nil
|
|
}
|
|
|
|
func say(l Load) []string {
|
|
var out []string
|
|
c := l.CPU
|
|
line := fmt.Sprintf("CPU: %.1f %% waiting on I/O, %.1f %% user, %.1f %% system, %.1f %% idle, over %g s on %d CPUs", c.IOWaitPct, c.UserPct, c.SystemPct, c.IdlePct, l.WindowS, c.CPUs)
|
|
if c.IOWaitPct >= iowaitLine {
|
|
line += " — HIGH: the processors sat idle waiting on storage"
|
|
}
|
|
out = append(out, line)
|
|
if p, ok := l.Pressure.(PSI); ok {
|
|
line := fmt.Sprintf("I/O pressure: some task stalled %.1f %% of the window (avg10 %.1f %%, avg60 %.1f %%, avg300 %.1f %%)", p.Some.WindowPct, p.Some.Avg10, p.Some.Avg60, p.Some.Avg300)
|
|
if p.Full != nil {
|
|
line += fmt.Sprintf("; every task stalled %.1f %%", p.Full.WindowPct)
|
|
}
|
|
if p.Some.WindowPct >= stallLine || p.Some.Avg60 >= stallLine {
|
|
line += " — HIGH"
|
|
}
|
|
out = append(out, line)
|
|
}
|
|
for _, d := range l.Devices {
|
|
name := d.Device
|
|
if d.Name != "" {
|
|
name += " (" + d.Name + ")"
|
|
}
|
|
line := fmt.Sprintf("%s: busy %.1f %%, read %.1f MiB/s (%.1f IOPS, %.1f ms wait), write %.1f MiB/s (%.1f IOPS, %.1f ms wait), queue depth %.1f",
|
|
name, d.BusyPct, d.ReadMiBs, d.ReadIOPS, d.ReadWaitMs, d.WriteMiBs, d.WriteIOPS, d.WriteWaitMs, d.QueueDepth)
|
|
if len(d.Mounts) > 0 {
|
|
line += ", holds " + strings.Join(d.Mounts, ", ")
|
|
}
|
|
if d.BusyPct >= busyLine {
|
|
line += " — SATURATED"
|
|
}
|
|
out = append(out, line)
|
|
}
|
|
for _, fs := range l.Filesystems {
|
|
if fs.Unanswered != "" {
|
|
out = append(out, fmt.Sprintf("%s: not read (%s)", fs.Mount, fs.Unanswered))
|
|
continue
|
|
}
|
|
if fs.UsedPct >= fullLine || fs.InodesUsed >= fullLine {
|
|
out = append(out, fmt.Sprintf("%s: %.1f %% full (%.1f GiB of %.1f GiB, %.1f GiB free), inodes %.1f %% used — NEARLY FULL", fs.Mount, fs.UsedPct, fs.UsedGiB, fs.SizeGiB, fs.AvailGiB, fs.InodesUsed))
|
|
}
|
|
}
|
|
for i, u := range l.Units {
|
|
if i == 3 {
|
|
break
|
|
}
|
|
name := u.Unit
|
|
if u.Container != "" {
|
|
name = "container " + u.Container
|
|
}
|
|
out = append(out, fmt.Sprintf("busiest unit: %s read %.2f MiB/s, wrote %.2f MiB/s", name, u.ReadMiBs, u.WriteMiBs))
|
|
}
|
|
for i, p := range l.Processes {
|
|
if i == 3 {
|
|
break
|
|
}
|
|
where := p.Unit
|
|
if p.Container != "" {
|
|
where = "container " + p.Container
|
|
}
|
|
out = append(out, fmt.Sprintf("busiest process: %s (pid %d, %s) read %.2f MiB/s, wrote %.2f MiB/s", p.Command, p.PID, orNone(where), p.ReadMiBs, p.WriteMiBs))
|
|
}
|
|
return out
|
|
}
|
|
|
|
// unreadableNote says what a list of processes misses when the tool runner is not root.
|
|
func unreadableNote(n int) string {
|
|
return fmt.Sprintf("%d processes' I/O could not be read: they are another account's, and the tool runner is not root, "+
|
|
"so a busy one may be missing from the processes; the units, read from their cgroups, miss none", n)
|
|
}
|
|
|
|
func orNone(s string) string {
|
|
if s == "" {
|
|
return "no unit"
|
|
}
|
|
return s
|
|
}
|
|
|
|
// DeviceInfo is one disk as it is, and what it has done since the machine started.
|
|
type DeviceInfo struct {
|
|
Device string `json:"device"`
|
|
Name string `json:"name,omitempty"`
|
|
On []string `json:"on,omitempty"`
|
|
Model string `json:"model,omitempty"`
|
|
SizeGiB float64 `json:"size_gib"`
|
|
Rotational bool `json:"rotational"`
|
|
Scheduler string `json:"scheduler,omitempty"`
|
|
QueueLimit int `json:"queue_limit,omitempty"`
|
|
ReadGiB float64 `json:"read_since_boot_gib"`
|
|
WrittenGiB float64 `json:"written_since_boot_gib"`
|
|
BusySincePct float64 `json:"busy_since_boot_pct"`
|
|
ReadWaitMs float64 `json:"read_wait_since_boot_ms"`
|
|
WriteWaitMs float64 `json:"write_wait_since_boot_ms"`
|
|
}
|
|
|
|
// DevicesNow is every disk with its kind, its size, its scheduler and its totals since boot: the
|
|
// question "is this a spinning disk" and "how much has it written" without a window.
|
|
func (m *Machine) DevicesNow(all bool) ([]DeviceInfo, float64) {
|
|
stats := parseDiskstats(m.read("/proc/diskstats"))
|
|
uptime := 0.0
|
|
if f := strings.Fields(m.read("/proc/uptime")); len(f) > 0 {
|
|
uptime, _ = strconv.ParseFloat(f[0], 64)
|
|
}
|
|
out := []DeviceInfo{}
|
|
for _, name := range m.blockDevices(stats, all) {
|
|
c := stats[name]
|
|
base := "/sys/block/" + name
|
|
if !m.exists(base) {
|
|
base = "/sys/block/" + m.parentOf(name) + "/" + name
|
|
}
|
|
sectors, _ := strconv.ParseFloat(strings.TrimSpace(m.read(base+"/size")), 64)
|
|
d := DeviceInfo{
|
|
Device: name,
|
|
Name: m.mapperName(name),
|
|
On: m.below(name),
|
|
Model: strings.TrimSpace(m.read(base + "/device/model")),
|
|
SizeGiB: round1(sectors * 512 / gib),
|
|
Rotational: strings.TrimSpace(m.read(base+"/queue/rotational")) == "1",
|
|
Scheduler: activeScheduler(m.read(base + "/queue/scheduler")),
|
|
ReadGiB: round1(float64(c.readSectors) * 512 / gib),
|
|
WrittenGiB: round1(float64(c.writeSectors) * 512 / gib),
|
|
}
|
|
d.QueueLimit, _ = strconv.Atoi(strings.TrimSpace(m.read(base + "/queue/nr_requests")))
|
|
if uptime > 0 {
|
|
d.BusySincePct = round1(float64(c.busyMs) / (uptime * 1000) * 100)
|
|
}
|
|
if c.reads > 0 {
|
|
d.ReadWaitMs = round1(float64(c.readMs) / float64(c.reads))
|
|
}
|
|
if c.writes > 0 {
|
|
d.WriteWaitMs = round1(float64(c.writeMs) / float64(c.writes))
|
|
}
|
|
out = append(out, d)
|
|
}
|
|
return out, round1(uptime / 3600)
|
|
}
|
|
|
|
// activeScheduler is the bracketed one of "mq-deadline kyber [bfq] none".
|
|
func activeScheduler(s string) string {
|
|
if i := strings.Index(s, "["); i >= 0 {
|
|
if j := strings.Index(s[i:], "]"); j > 0 {
|
|
return s[i+1 : i+j]
|
|
}
|
|
}
|
|
return strings.TrimSpace(s)
|
|
}
|