Add disk-load's cpu_top tool: CPU by systemd unit and container (issue 741)
No tool named which units or processes used CPU outside containers during the novox overload (issue 739). cpu_top reads each cgroup's cpu.stat (no root needed), ranks units or containers busiest first, and says the machine's CPU in use and the busy time no unit accounts for.
This commit is contained in:
jochen
@@ -0,0 +1,164 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// cgroupCPU is a cgroup's processor time, from its cpu.stat.
|
||||
type cgroupCPU struct {
|
||||
usageUsec uint64
|
||||
}
|
||||
|
||||
// parseCPUStat reads a cgroup's cpu.stat ("usage_usec 123\nuser_usec …\nsystem_usec …"), readable by
|
||||
// every account, unlike /proc/<pid>/stat for another account's process.
|
||||
func parseCPUStat(s string) cgroupCPU {
|
||||
var c cgroupCPU
|
||||
for _, line := range strings.Split(s, "\n") {
|
||||
k, v, ok := strings.Cut(line, " ")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if k == "usage_usec" {
|
||||
c.usageUsec, _ = strconv.ParseUint(strings.TrimSpace(v), 10, 64)
|
||||
}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// unitCPU is every unit's processor time, by its cgroup's path. A cgroup whose cpu.stat cannot be
|
||||
// read is left out, never an error for the whole answer.
|
||||
func (m *Machine) unitCPU() map[string]cgroupCPU {
|
||||
out := map[string]cgroupCPU{}
|
||||
for _, u := range m.unitPaths() {
|
||||
text := m.read("/sys/fs/cgroup/" + u + "/cpu.stat")
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
out[u] = parseCPUStat(text)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// MachineCPU is the machine's CPU in use over the window, from /proc/stat's cpu line.
|
||||
type MachineCPU struct {
|
||||
CPUs int `json:"cpus"`
|
||||
BusyPct float64 `json:"busy_pct"` // % of all CPUs together
|
||||
CPUsBusy float64 `json:"cpus_busy"` // as a count of CPUs, e.g. 7.5 of 12
|
||||
}
|
||||
|
||||
// machineCPUOver is the machine's CPU in use between two readings of /proc/stat: everything but idle
|
||||
// and iowait, counted against the window's total ticks over every CPU.
|
||||
func machineCPUOver(a, b cpuTimes) MachineCPU {
|
||||
total := float64(b.total() - a.total())
|
||||
if total <= 0 {
|
||||
return MachineCPU{CPUs: b.cpus}
|
||||
}
|
||||
idle := sub(b.idle, a.idle) + sub(b.iowait, a.iowait)
|
||||
busy := total - idle
|
||||
if busy < 0 {
|
||||
busy = 0
|
||||
}
|
||||
pct := round1(busy / total * 100)
|
||||
return MachineCPU{CPUs: b.cpus, BusyPct: pct, CPUsBusy: round1(pct / 100 * float64(b.cpus))}
|
||||
}
|
||||
|
||||
// CPUUsage is a unit's or a container's share of CPU between two readings of its cgroup's cpu.stat.
|
||||
type CPUUsage struct {
|
||||
Unit string `json:"unit,omitempty"`
|
||||
Container string `json:"container,omitempty"`
|
||||
CPUPct float64 `json:"cpu_pct"` // % of one CPU: 100 = one CPU busy the whole window
|
||||
SharePct float64 `json:"share_pct"` // this unit's share of the machine's busy time
|
||||
raw float64
|
||||
}
|
||||
|
||||
// topCPU ranks the units, or the containers, with the most CPU time between two readings of
|
||||
// cpu.stat, busiest first. unaccountedPct is the machine's busy time no unit accounts for, as a
|
||||
// share of it (never below 0): the sum is always taken over every unit, whichever by ranks by, so a
|
||||
// limit on the list never changes it.
|
||||
func (m *Machine) topCPU(a, b map[string]cgroupCPU, window time.Duration, by string, limit int, cpusBusy float64) ([]CPUUsage, float64) {
|
||||
windowUsec := window.Seconds() * 1e6
|
||||
type entry struct {
|
||||
unit, container string
|
||||
pct float64
|
||||
}
|
||||
var entries []entry
|
||||
var names map[string]string
|
||||
var sumPct float64
|
||||
for path, end := range b {
|
||||
start, ok := a[path]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
d := sub(end.usageUsec, start.usageUsec)
|
||||
if d == 0 {
|
||||
continue
|
||||
}
|
||||
pct := d / windowUsec * 100
|
||||
sumPct += pct
|
||||
unit := filepath.Base(path)
|
||||
var container string
|
||||
if id := containerID.FindStringSubmatch("/" + path); id != nil {
|
||||
if names == nil {
|
||||
names = m.containerNames()
|
||||
}
|
||||
container = id[1][:12]
|
||||
if n, ok := names[id[1]]; ok {
|
||||
container = n
|
||||
}
|
||||
}
|
||||
entries = append(entries, entry{unit: unit, container: container, pct: pct})
|
||||
}
|
||||
|
||||
groups := map[string]*CPUUsage{}
|
||||
for _, e := range entries {
|
||||
key := e.unit
|
||||
if by == "container" {
|
||||
if e.container == "" {
|
||||
continue
|
||||
}
|
||||
key = e.container
|
||||
}
|
||||
g, ok := groups[key]
|
||||
if !ok {
|
||||
g = &CPUUsage{Unit: e.unit, Container: e.container}
|
||||
if by == "container" {
|
||||
g.Unit = ""
|
||||
}
|
||||
groups[key] = g
|
||||
}
|
||||
g.raw += e.pct
|
||||
}
|
||||
out := make([]CPUUsage, 0, len(groups))
|
||||
for _, g := range groups {
|
||||
out = append(out, *g)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].raw != out[j].raw {
|
||||
return out[i].raw > out[j].raw
|
||||
}
|
||||
return out[i].Unit+out[i].Container < out[j].Unit+out[j].Container
|
||||
})
|
||||
if len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
for i := range out {
|
||||
out[i].CPUPct = round1(out[i].raw)
|
||||
if cpusBusy > 0 {
|
||||
out[i].SharePct = round1(out[i].raw / (cpusBusy * 100) * 100)
|
||||
}
|
||||
}
|
||||
|
||||
unaccounted := cpusBusy*100 - sumPct
|
||||
if unaccounted < 0 {
|
||||
unaccounted = 0
|
||||
}
|
||||
unaccountedPct := 0.0
|
||||
if cpusBusy > 0 {
|
||||
unaccountedPct = round1(unaccounted / (cpusBusy * 100) * 100)
|
||||
}
|
||||
return out, unaccountedPct
|
||||
}
|
||||
Reference in New Issue
Block a user