Add disk-load: no tool said a machine's iowait, disk load or who does the I/O (hq issue 312)
This commit is contained in:
jochen
@@ -0,0 +1,281 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// cpuTimes is the first line of /proc/stat, in clock ticks: what every CPU together spent its time on.
|
||||
type cpuTimes struct {
|
||||
user, nice, system, idle, iowait, irq, softirq, steal uint64
|
||||
cpus int
|
||||
}
|
||||
|
||||
// total leaves guest time out: the kernel already counts it in user and nice.
|
||||
func (c cpuTimes) total() uint64 {
|
||||
return c.user + c.nice + c.system + c.idle + c.iowait + c.irq + c.softirq + c.steal
|
||||
}
|
||||
|
||||
func parseCPU(stat string) (cpuTimes, error) {
|
||||
var c cpuTimes
|
||||
found := false
|
||||
for _, line := range strings.Split(stat, "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) == 0 || !strings.HasPrefix(f[0], "cpu") {
|
||||
continue
|
||||
}
|
||||
if f[0] != "cpu" {
|
||||
c.cpus++
|
||||
continue
|
||||
}
|
||||
if len(f) < 9 {
|
||||
return c, fmt.Errorf("/proc/stat: the cpu line has %d fields, not at least 9", len(f))
|
||||
}
|
||||
v := make([]uint64, 8)
|
||||
for i := range v {
|
||||
n, err := strconv.ParseUint(f[i+1], 10, 64)
|
||||
if err != nil {
|
||||
return c, fmt.Errorf("/proc/stat: %q is not a count", f[i+1])
|
||||
}
|
||||
v[i] = n
|
||||
}
|
||||
c.user, c.nice, c.system, c.idle, c.iowait, c.irq, c.softirq, c.steal = v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7]
|
||||
found = true
|
||||
}
|
||||
if !found {
|
||||
return c, fmt.Errorf("/proc/stat has no cpu line")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// CPU is how all the processors together spent the window, in percent of their time.
|
||||
type CPU struct {
|
||||
CPUs int `json:"cpus"`
|
||||
IOWaitPct float64 `json:"iowait_pct"`
|
||||
UserPct float64 `json:"user_pct"`
|
||||
SystemPct float64 `json:"system_pct"`
|
||||
IdlePct float64 `json:"idle_pct"`
|
||||
StealPct float64 `json:"steal_pct"`
|
||||
}
|
||||
|
||||
func cpuShare(a, b cpuTimes) CPU {
|
||||
total := float64(b.total() - a.total())
|
||||
pct := func(x, y uint64) float64 {
|
||||
if total <= 0 || y < x {
|
||||
return 0
|
||||
}
|
||||
return round1(float64(y-x) / total * 100)
|
||||
}
|
||||
return CPU{
|
||||
CPUs: b.cpus,
|
||||
IOWaitPct: pct(a.iowait, b.iowait),
|
||||
UserPct: pct(a.user+a.nice, b.user+b.nice),
|
||||
SystemPct: pct(a.system+a.irq+a.softirq, b.system+b.irq+b.softirq),
|
||||
IdlePct: pct(a.idle, b.idle),
|
||||
StealPct: pct(a.steal, b.steal),
|
||||
}
|
||||
}
|
||||
|
||||
// diskCounters is one line of /proc/diskstats (Documentation/admin-guide/iostats.rst). Sectors are
|
||||
// always 512 bytes there, whatever the device's own sector size.
|
||||
type diskCounters struct {
|
||||
reads, readSectors, readMs uint64
|
||||
writes, writeSectors, writeMs uint64
|
||||
inFlight, busyMs, weightedMs uint64
|
||||
}
|
||||
|
||||
func parseDiskstats(s string) map[string]diskCounters {
|
||||
out := map[string]diskCounters{}
|
||||
for _, line := range strings.Split(s, "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) < 14 {
|
||||
continue
|
||||
}
|
||||
n := make([]uint64, 11)
|
||||
ok := true
|
||||
for i := range n {
|
||||
v, err := strconv.ParseUint(f[i+3], 10, 64)
|
||||
if err != nil {
|
||||
ok = false
|
||||
break
|
||||
}
|
||||
n[i] = v
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
out[f[2]] = diskCounters{
|
||||
reads: n[0], readSectors: n[2], readMs: n[3],
|
||||
writes: n[4], writeSectors: n[6], writeMs: n[7],
|
||||
inFlight: n[8], busyMs: n[9], weightedMs: n[10],
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Device is one block device over the window.
|
||||
type Device struct {
|
||||
Device string `json:"device"`
|
||||
Name string `json:"name,omitempty"`
|
||||
On []string `json:"on,omitempty"`
|
||||
Mounts []string `json:"mounts,omitempty"`
|
||||
BusyPct float64 `json:"busy_pct"`
|
||||
ReadMiBs float64 `json:"read_mib_s"`
|
||||
WriteMiBs float64 `json:"write_mib_s"`
|
||||
ReadIOPS float64 `json:"read_iops"`
|
||||
WriteIOPS float64 `json:"write_iops"`
|
||||
ReadWaitMs float64 `json:"read_wait_ms"`
|
||||
WriteWaitMs float64 `json:"write_wait_ms"`
|
||||
QueueDepth float64 `json:"queue_depth"`
|
||||
InFlight uint64 `json:"in_flight"`
|
||||
}
|
||||
|
||||
func sub(b, a uint64) float64 {
|
||||
if b < a {
|
||||
return 0
|
||||
}
|
||||
return float64(b - a)
|
||||
}
|
||||
|
||||
// deviceOver is iostat's arithmetic: busy is the share of the window with any request in flight, wait
|
||||
// the time a completed request took from queueing to completion, queue depth the requests in flight
|
||||
// on average.
|
||||
func deviceOver(name string, a, b diskCounters, window time.Duration) Device {
|
||||
ms := float64(window.Milliseconds())
|
||||
s := window.Seconds()
|
||||
reads, writes := sub(b.reads, a.reads), sub(b.writes, a.writes)
|
||||
wait := func(spent, n float64) float64 {
|
||||
if n == 0 {
|
||||
return 0
|
||||
}
|
||||
return round1(spent / n)
|
||||
}
|
||||
busy := sub(b.busyMs, a.busyMs) / ms * 100
|
||||
if busy > 100 {
|
||||
busy = 100
|
||||
}
|
||||
return Device{
|
||||
Device: name,
|
||||
BusyPct: round1(busy),
|
||||
ReadMiBs: round1(sub(b.readSectors, a.readSectors) * 512 / mib / s),
|
||||
WriteMiBs: round1(sub(b.writeSectors, a.writeSectors) * 512 / mib / s),
|
||||
ReadIOPS: round1(reads / s),
|
||||
WriteIOPS: round1(writes / s),
|
||||
ReadWaitMs: wait(sub(b.readMs, a.readMs), reads),
|
||||
WriteWaitMs: wait(sub(b.writeMs, a.writeMs), writes),
|
||||
QueueDepth: round1(sub(b.weightedMs, a.weightedMs) / ms),
|
||||
InFlight: b.inFlight,
|
||||
}
|
||||
}
|
||||
|
||||
// blockDevices are the whole devices in /sys/block, without the ones that are not a disk: loop devices,
|
||||
// RAM disks and compressed swap in RAM (zram, the memory-pressure module's). all keeps every one, and
|
||||
// every partition /proc/diskstats names.
|
||||
func (m *Machine) blockDevices(stats map[string]diskCounters, all bool) []string {
|
||||
var out []string
|
||||
if all {
|
||||
for name := range stats {
|
||||
out = append(out, name)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
for _, name := range m.names("/sys/block") {
|
||||
if strings.HasPrefix(name, "loop") || strings.HasPrefix(name, "ram") || strings.HasPrefix(name, "zram") {
|
||||
continue
|
||||
}
|
||||
if _, ok := stats[name]; ok {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mapperName is a device-mapper device's own name (dm-0 is "root"), "" for any other device.
|
||||
func (m *Machine) mapperName(dev string) string {
|
||||
return strings.TrimSpace(m.read("/sys/block/" + dev + "/dm/name"))
|
||||
}
|
||||
|
||||
// below is what a device-mapper or RAID device is built on.
|
||||
func (m *Machine) below(dev string) []string { return m.names("/sys/block/" + dev + "/slaves") }
|
||||
|
||||
// parentOf is the whole device a partition is on; the device itself when it is whole.
|
||||
func (m *Machine) parentOf(dev string) string {
|
||||
if m.exists("/sys/block/" + dev) {
|
||||
return dev
|
||||
}
|
||||
for _, disk := range m.names("/sys/block") {
|
||||
if m.exists("/sys/block/" + disk + "/" + dev) {
|
||||
return disk
|
||||
}
|
||||
}
|
||||
return dev
|
||||
}
|
||||
|
||||
// PSI is the kernel's pressure stall information for I/O: the share of time some (or all) runnable
|
||||
// tasks were stalled waiting on it.
|
||||
type PSI struct {
|
||||
Some PSILine `json:"some"`
|
||||
Full *PSILine `json:"full,omitempty"`
|
||||
}
|
||||
|
||||
// PSILine is one line, its running averages and its share of the sampled window.
|
||||
type PSILine struct {
|
||||
WindowPct float64 `json:"window_pct"`
|
||||
Avg10 float64 `json:"avg10_pct"`
|
||||
Avg60 float64 `json:"avg60_pct"`
|
||||
Avg300 float64 `json:"avg300_pct"`
|
||||
totalUs uint64
|
||||
}
|
||||
|
||||
func parsePSI(s string) (PSI, bool) {
|
||||
var p PSI
|
||||
some := false
|
||||
for _, line := range strings.Split(s, "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) < 5 {
|
||||
continue
|
||||
}
|
||||
var l PSILine
|
||||
for _, kv := range f[1:] {
|
||||
k, v, _ := strings.Cut(kv, "=")
|
||||
switch k {
|
||||
case "avg10":
|
||||
l.Avg10, _ = strconv.ParseFloat(v, 64)
|
||||
case "avg60":
|
||||
l.Avg60, _ = strconv.ParseFloat(v, 64)
|
||||
case "avg300":
|
||||
l.Avg300, _ = strconv.ParseFloat(v, 64)
|
||||
case "total":
|
||||
l.totalUs, _ = strconv.ParseUint(v, 10, 64)
|
||||
}
|
||||
}
|
||||
switch f[0] {
|
||||
case "some":
|
||||
p.Some, some = l, true
|
||||
case "full":
|
||||
full := l
|
||||
p.Full = &full
|
||||
}
|
||||
}
|
||||
return p, some
|
||||
}
|
||||
|
||||
func psiOver(a, b PSI, window time.Duration) PSI {
|
||||
share := func(x, y PSILine) PSILine {
|
||||
y.WindowPct = round1(sub(y.totalUs, x.totalUs) / float64(window.Microseconds()) * 100)
|
||||
if y.WindowPct > 100 {
|
||||
y.WindowPct = 100
|
||||
}
|
||||
return y
|
||||
}
|
||||
out := PSI{Some: share(a.Some, b.Some)}
|
||||
if a.Full != nil && b.Full != nil {
|
||||
full := share(*a.Full, *b.Full)
|
||||
out.Full = &full
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user