282 lines
7.6 KiB
Go
282 lines
7.6 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// cpuTimes is the first line of /proc/stat, in clock ticks: what every CPU together spent its time on.
|
|
type cpuTimes struct {
|
|
user, nice, system, idle, iowait, irq, softirq, steal uint64
|
|
cpus int
|
|
}
|
|
|
|
// total leaves guest time out: the kernel already counts it in user and nice.
|
|
func (c cpuTimes) total() uint64 {
|
|
return c.user + c.nice + c.system + c.idle + c.iowait + c.irq + c.softirq + c.steal
|
|
}
|
|
|
|
func parseCPU(stat string) (cpuTimes, error) {
|
|
var c cpuTimes
|
|
found := false
|
|
for _, line := range strings.Split(stat, "\n") {
|
|
f := strings.Fields(line)
|
|
if len(f) == 0 || !strings.HasPrefix(f[0], "cpu") {
|
|
continue
|
|
}
|
|
if f[0] != "cpu" {
|
|
c.cpus++
|
|
continue
|
|
}
|
|
if len(f) < 9 {
|
|
return c, fmt.Errorf("/proc/stat: the cpu line has %d fields, not at least 9", len(f))
|
|
}
|
|
v := make([]uint64, 8)
|
|
for i := range v {
|
|
n, err := strconv.ParseUint(f[i+1], 10, 64)
|
|
if err != nil {
|
|
return c, fmt.Errorf("/proc/stat: %q is not a count", f[i+1])
|
|
}
|
|
v[i] = n
|
|
}
|
|
c.user, c.nice, c.system, c.idle, c.iowait, c.irq, c.softirq, c.steal = v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7]
|
|
found = true
|
|
}
|
|
if !found {
|
|
return c, fmt.Errorf("/proc/stat has no cpu line")
|
|
}
|
|
return c, nil
|
|
}
|
|
|
|
// CPU is how all the processors together spent the window, in percent of their time.
|
|
type CPU struct {
|
|
CPUs int `json:"cpus"`
|
|
IOWaitPct float64 `json:"iowait_pct"`
|
|
UserPct float64 `json:"user_pct"`
|
|
SystemPct float64 `json:"system_pct"`
|
|
IdlePct float64 `json:"idle_pct"`
|
|
StealPct float64 `json:"steal_pct"`
|
|
}
|
|
|
|
func cpuShare(a, b cpuTimes) CPU {
|
|
total := float64(b.total() - a.total())
|
|
pct := func(x, y uint64) float64 {
|
|
if total <= 0 || y < x {
|
|
return 0
|
|
}
|
|
return round1(float64(y-x) / total * 100)
|
|
}
|
|
return CPU{
|
|
CPUs: b.cpus,
|
|
IOWaitPct: pct(a.iowait, b.iowait),
|
|
UserPct: pct(a.user+a.nice, b.user+b.nice),
|
|
SystemPct: pct(a.system+a.irq+a.softirq, b.system+b.irq+b.softirq),
|
|
IdlePct: pct(a.idle, b.idle),
|
|
StealPct: pct(a.steal, b.steal),
|
|
}
|
|
}
|
|
|
|
// diskCounters is one line of /proc/diskstats (Documentation/admin-guide/iostats.rst). Sectors are
|
|
// always 512 bytes there, whatever the device's own sector size.
|
|
type diskCounters struct {
|
|
reads, readSectors, readMs uint64
|
|
writes, writeSectors, writeMs uint64
|
|
inFlight, busyMs, weightedMs uint64
|
|
}
|
|
|
|
func parseDiskstats(s string) map[string]diskCounters {
|
|
out := map[string]diskCounters{}
|
|
for _, line := range strings.Split(s, "\n") {
|
|
f := strings.Fields(line)
|
|
if len(f) < 14 {
|
|
continue
|
|
}
|
|
n := make([]uint64, 11)
|
|
ok := true
|
|
for i := range n {
|
|
v, err := strconv.ParseUint(f[i+3], 10, 64)
|
|
if err != nil {
|
|
ok = false
|
|
break
|
|
}
|
|
n[i] = v
|
|
}
|
|
if !ok {
|
|
continue
|
|
}
|
|
out[f[2]] = diskCounters{
|
|
reads: n[0], readSectors: n[2], readMs: n[3],
|
|
writes: n[4], writeSectors: n[6], writeMs: n[7],
|
|
inFlight: n[8], busyMs: n[9], weightedMs: n[10],
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Device is one block device over the window.
|
|
type Device struct {
|
|
Device string `json:"device"`
|
|
Name string `json:"name,omitempty"`
|
|
On []string `json:"on,omitempty"`
|
|
Mounts []string `json:"mounts,omitempty"`
|
|
BusyPct float64 `json:"busy_pct"`
|
|
ReadMiBs float64 `json:"read_mib_s"`
|
|
WriteMiBs float64 `json:"write_mib_s"`
|
|
ReadIOPS float64 `json:"read_iops"`
|
|
WriteIOPS float64 `json:"write_iops"`
|
|
ReadWaitMs float64 `json:"read_wait_ms"`
|
|
WriteWaitMs float64 `json:"write_wait_ms"`
|
|
QueueDepth float64 `json:"queue_depth"`
|
|
InFlight uint64 `json:"in_flight"`
|
|
}
|
|
|
|
func sub(b, a uint64) float64 {
|
|
if b < a {
|
|
return 0
|
|
}
|
|
return float64(b - a)
|
|
}
|
|
|
|
// deviceOver is iostat's arithmetic: busy is the share of the window with any request in flight, wait
|
|
// the time a completed request took from queueing to completion, queue depth the requests in flight
|
|
// on average.
|
|
func deviceOver(name string, a, b diskCounters, window time.Duration) Device {
|
|
ms := float64(window.Milliseconds())
|
|
s := window.Seconds()
|
|
reads, writes := sub(b.reads, a.reads), sub(b.writes, a.writes)
|
|
wait := func(spent, n float64) float64 {
|
|
if n == 0 {
|
|
return 0
|
|
}
|
|
return round1(spent / n)
|
|
}
|
|
busy := sub(b.busyMs, a.busyMs) / ms * 100
|
|
if busy > 100 {
|
|
busy = 100
|
|
}
|
|
return Device{
|
|
Device: name,
|
|
BusyPct: round1(busy),
|
|
ReadMiBs: round1(sub(b.readSectors, a.readSectors) * 512 / mib / s),
|
|
WriteMiBs: round1(sub(b.writeSectors, a.writeSectors) * 512 / mib / s),
|
|
ReadIOPS: round1(reads / s),
|
|
WriteIOPS: round1(writes / s),
|
|
ReadWaitMs: wait(sub(b.readMs, a.readMs), reads),
|
|
WriteWaitMs: wait(sub(b.writeMs, a.writeMs), writes),
|
|
QueueDepth: round1(sub(b.weightedMs, a.weightedMs) / ms),
|
|
InFlight: b.inFlight,
|
|
}
|
|
}
|
|
|
|
// blockDevices are the whole devices in /sys/block, without the ones that are not a disk: loop devices,
|
|
// RAM disks and compressed swap in RAM (zram, the memory-pressure module's). all keeps every one, and
|
|
// every partition /proc/diskstats names.
|
|
func (m *Machine) blockDevices(stats map[string]diskCounters, all bool) []string {
|
|
var out []string
|
|
if all {
|
|
for name := range stats {
|
|
out = append(out, name)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
for _, name := range m.names("/sys/block") {
|
|
if strings.HasPrefix(name, "loop") || strings.HasPrefix(name, "ram") || strings.HasPrefix(name, "zram") {
|
|
continue
|
|
}
|
|
if _, ok := stats[name]; ok {
|
|
out = append(out, name)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// mapperName is a device-mapper device's own name (dm-0 is "root"), "" for any other device.
|
|
func (m *Machine) mapperName(dev string) string {
|
|
return strings.TrimSpace(m.read("/sys/block/" + dev + "/dm/name"))
|
|
}
|
|
|
|
// below is what a device-mapper or RAID device is built on.
|
|
func (m *Machine) below(dev string) []string { return m.names("/sys/block/" + dev + "/slaves") }
|
|
|
|
// parentOf is the whole device a partition is on; the device itself when it is whole.
|
|
func (m *Machine) parentOf(dev string) string {
|
|
if m.exists("/sys/block/" + dev) {
|
|
return dev
|
|
}
|
|
for _, disk := range m.names("/sys/block") {
|
|
if m.exists("/sys/block/" + disk + "/" + dev) {
|
|
return disk
|
|
}
|
|
}
|
|
return dev
|
|
}
|
|
|
|
// PSI is the kernel's pressure stall information for I/O: the share of time some (or all) runnable
|
|
// tasks were stalled waiting on it.
|
|
type PSI struct {
|
|
Some PSILine `json:"some"`
|
|
Full *PSILine `json:"full,omitempty"`
|
|
}
|
|
|
|
// PSILine is one line, its running averages and its share of the sampled window.
|
|
type PSILine struct {
|
|
WindowPct float64 `json:"window_pct"`
|
|
Avg10 float64 `json:"avg10_pct"`
|
|
Avg60 float64 `json:"avg60_pct"`
|
|
Avg300 float64 `json:"avg300_pct"`
|
|
totalUs uint64
|
|
}
|
|
|
|
func parsePSI(s string) (PSI, bool) {
|
|
var p PSI
|
|
some := false
|
|
for _, line := range strings.Split(s, "\n") {
|
|
f := strings.Fields(line)
|
|
if len(f) < 5 {
|
|
continue
|
|
}
|
|
var l PSILine
|
|
for _, kv := range f[1:] {
|
|
k, v, _ := strings.Cut(kv, "=")
|
|
switch k {
|
|
case "avg10":
|
|
l.Avg10, _ = strconv.ParseFloat(v, 64)
|
|
case "avg60":
|
|
l.Avg60, _ = strconv.ParseFloat(v, 64)
|
|
case "avg300":
|
|
l.Avg300, _ = strconv.ParseFloat(v, 64)
|
|
case "total":
|
|
l.totalUs, _ = strconv.ParseUint(v, 10, 64)
|
|
}
|
|
}
|
|
switch f[0] {
|
|
case "some":
|
|
p.Some, some = l, true
|
|
case "full":
|
|
full := l
|
|
p.Full = &full
|
|
}
|
|
}
|
|
return p, some
|
|
}
|
|
|
|
func psiOver(a, b PSI, window time.Duration) PSI {
|
|
share := func(x, y PSILine) PSILine {
|
|
y.WindowPct = round1(sub(y.totalUs, x.totalUs) / float64(window.Microseconds()) * 100)
|
|
if y.WindowPct > 100 {
|
|
y.WindowPct = 100
|
|
}
|
|
return y
|
|
}
|
|
out := PSI{Some: share(a.Some, b.Some)}
|
|
if a.Full != nil && b.Full != nil {
|
|
full := share(*a.Full, *b.Full)
|
|
out.Full = &full
|
|
}
|
|
return out
|
|
}
|