From a8d84d1730bc9c9de49afd6215d574c7a7366260 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 15 Oct 2026 14:51:08 +0200 Subject: [PATCH] Add disk-load's cpu_top tool: CPU by systemd unit and container (issue 741) No tool named which units or processes used CPU outside containers during the novox overload (issue 739). cpu_top reads each cgroup's cpu.stat (no root needed), ranks units or containers busiest first, and says the machine's CPU in use and the busy time no unit accounts for. --- modules/disk-load/README.md | 7 +- modules/disk-load/cmd/disk-load/cgroups.go | 21 ++- modules/disk-load/cmd/disk-load/cpu.go | 164 ++++++++++++++++++++ modules/disk-load/cmd/disk-load/cpu_test.go | 107 +++++++++++++ modules/disk-load/cmd/disk-load/tools.go | 51 ++++++ modules/disk-load/module.json | 3 +- 6 files changed, 344 insertions(+), 9 deletions(-) create mode 100644 modules/disk-load/cmd/disk-load/cpu.go create mode 100644 modules/disk-load/cmd/disk-load/cpu_test.go diff --git a/modules/disk-load/README.md b/modules/disk-load/README.md index 4ba20e5e..4d4f764e 100644 --- a/modules/disk-load/README.md +++ b/modules/disk-load/README.md @@ -3,7 +3,9 @@ How loaded a machine's storage is, read from `/proc` and `/sys` and nothing else: the share of CPU time spent waiting on I/O, I/O pressure stall, each disk's busy share, throughput, IOPS, wait and queue depth, the filesystems' space, and the units, containers and processes doing the I/O (novox/hq issue -312). It changes nothing and declares no resources, so it can be assigned to every machine. +312). It also reads a machine's CPU use by systemd unit and by container, from each cgroup's cpu.stat, +so an overload outside every container can be named (novox/hq issue 741). It changes nothing and +declares no resources, so it can be assigned to every machine. ## Why it exists, and why here @@ -22,7 +24,7 @@ machine's resources a seat, these tools become its verbs. ## Tools -All four only read. +All five only read. | tool | what | replaces | |---|---|---| @@ -30,6 +32,7 @@ All four only read. | `disk_top` | Over a window. `by=unit` (the default) ranks systemd units and containers by bytes moved, from their cgroups. `by=container` ranks containers by name, and `by=process` ranks processes | `iotop` | | `disk_filesystems` | Each filesystem that stores data, said once however many places it is mounted: size, used and free GiB, used % as `df` counts it, inodes used % | `df -h`, `df -i` | | `disk_devices` | Each disk without sampling: model, size, whether it spins, scheduler, queue limit, what it is built on, and since boot the GiB read and written, the busy share and the wait per read and per write | `lsblk`, `cat /proc/diskstats` | +| `cpu_top` | Samples a window (`seconds`, default 3, at most 10). It answers the machine's CPU in use (% of all CPUs and as CPUs busy, from `/proc/stat`), then `by=unit` (the default) ranks systemd units and containers by CPU, from their cgroups' cpu.stat, busiest first (`limit`, default 15), with the busy time no unit accounts for; `by=container` ranks containers by name | `top`, `systemd-cgtop` | ### How each number is read diff --git a/modules/disk-load/cmd/disk-load/cgroups.go b/modules/disk-load/cmd/disk-load/cgroups.go index 79af261b..84f46e89 100644 --- a/modules/disk-load/cmd/disk-load/cgroups.go +++ b/modules/disk-load/cmd/disk-load/cgroups.go @@ -68,9 +68,10 @@ func (m *Machine) bottomDevices() map[string]bool { return out } -// unitIO is every unit's storage I/O, by its cgroup's path: the innermost .service or .scope, because -// io.stat counts a cgroup's descendants too and an outer unit would count them twice. -func (m *Machine) unitIO(counted map[string]bool) map[string]cgroupIO { +// unitPaths is every systemd unit's cgroup path under /sys/fs/cgroup: the innermost .service or +// .scope, because a cgroup's own stat file counts its descendants too and an outer unit would count +// them twice. Shared by every reader that walks the cgroup tree (unitIO here, unitCPU in cpu.go). +func (m *Machine) unitPaths() []string { base := m.path("/sys/fs/cgroup") var units []string filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error { @@ -89,11 +90,19 @@ func (m *Machine) unitIO(counted map[string]bool) map[string]cgroupIO { outer[dir] = true } } - out := map[string]cgroupIO{} + out := make([]string, 0, len(units)) for _, u := range units { - if outer[u] { - continue + if !outer[u] { + out = append(out, u) } + } + return out +} + +// unitIO is every unit's storage I/O, by its cgroup's path. +func (m *Machine) unitIO(counted map[string]bool) map[string]cgroupIO { + out := map[string]cgroupIO{} + for _, u := range m.unitPaths() { text := m.read("/sys/fs/cgroup/" + u + "/io.stat") if text == "" { continue diff --git a/modules/disk-load/cmd/disk-load/cpu.go b/modules/disk-load/cmd/disk-load/cpu.go new file mode 100644 index 00000000..d5d3122f --- /dev/null +++ b/modules/disk-load/cmd/disk-load/cpu.go @@ -0,0 +1,164 @@ +package main + +import ( + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +// cgroupCPU is a cgroup's processor time, from its cpu.stat. +type cgroupCPU struct { + usageUsec uint64 +} + +// parseCPUStat reads a cgroup's cpu.stat ("usage_usec 123\nuser_usec …\nsystem_usec …"), readable by +// every account, unlike /proc//stat for another account's process. +func parseCPUStat(s string) cgroupCPU { + var c cgroupCPU + for _, line := range strings.Split(s, "\n") { + k, v, ok := strings.Cut(line, " ") + if !ok { + continue + } + if k == "usage_usec" { + c.usageUsec, _ = strconv.ParseUint(strings.TrimSpace(v), 10, 64) + } + } + return c +} + +// unitCPU is every unit's processor time, by its cgroup's path. A cgroup whose cpu.stat cannot be +// read is left out, never an error for the whole answer. +func (m *Machine) unitCPU() map[string]cgroupCPU { + out := map[string]cgroupCPU{} + for _, u := range m.unitPaths() { + text := m.read("/sys/fs/cgroup/" + u + "/cpu.stat") + if text == "" { + continue + } + out[u] = parseCPUStat(text) + } + return out +} + +// MachineCPU is the machine's CPU in use over the window, from /proc/stat's cpu line. +type MachineCPU struct { + CPUs int `json:"cpus"` + BusyPct float64 `json:"busy_pct"` // % of all CPUs together + CPUsBusy float64 `json:"cpus_busy"` // as a count of CPUs, e.g. 7.5 of 12 +} + +// machineCPUOver is the machine's CPU in use between two readings of /proc/stat: everything but idle +// and iowait, counted against the window's total ticks over every CPU. +func machineCPUOver(a, b cpuTimes) MachineCPU { + total := float64(b.total() - a.total()) + if total <= 0 { + return MachineCPU{CPUs: b.cpus} + } + idle := sub(b.idle, a.idle) + sub(b.iowait, a.iowait) + busy := total - idle + if busy < 0 { + busy = 0 + } + pct := round1(busy / total * 100) + return MachineCPU{CPUs: b.cpus, BusyPct: pct, CPUsBusy: round1(pct / 100 * float64(b.cpus))} +} + +// CPUUsage is a unit's or a container's share of CPU between two readings of its cgroup's cpu.stat. +type CPUUsage struct { + Unit string `json:"unit,omitempty"` + Container string `json:"container,omitempty"` + CPUPct float64 `json:"cpu_pct"` // % of one CPU: 100 = one CPU busy the whole window + SharePct float64 `json:"share_pct"` // this unit's share of the machine's busy time + raw float64 +} + +// topCPU ranks the units, or the containers, with the most CPU time between two readings of +// cpu.stat, busiest first. unaccountedPct is the machine's busy time no unit accounts for, as a +// share of it (never below 0): the sum is always taken over every unit, whichever by ranks by, so a +// limit on the list never changes it. +func (m *Machine) topCPU(a, b map[string]cgroupCPU, window time.Duration, by string, limit int, cpusBusy float64) ([]CPUUsage, float64) { + windowUsec := window.Seconds() * 1e6 + type entry struct { + unit, container string + pct float64 + } + var entries []entry + var names map[string]string + var sumPct float64 + for path, end := range b { + start, ok := a[path] + if !ok { + continue + } + d := sub(end.usageUsec, start.usageUsec) + if d == 0 { + continue + } + pct := d / windowUsec * 100 + sumPct += pct + unit := filepath.Base(path) + var container string + if id := containerID.FindStringSubmatch("/" + path); id != nil { + if names == nil { + names = m.containerNames() + } + container = id[1][:12] + if n, ok := names[id[1]]; ok { + container = n + } + } + entries = append(entries, entry{unit: unit, container: container, pct: pct}) + } + + groups := map[string]*CPUUsage{} + for _, e := range entries { + key := e.unit + if by == "container" { + if e.container == "" { + continue + } + key = e.container + } + g, ok := groups[key] + if !ok { + g = &CPUUsage{Unit: e.unit, Container: e.container} + if by == "container" { + g.Unit = "" + } + groups[key] = g + } + g.raw += e.pct + } + out := make([]CPUUsage, 0, len(groups)) + for _, g := range groups { + out = append(out, *g) + } + sort.Slice(out, func(i, j int) bool { + if out[i].raw != out[j].raw { + return out[i].raw > out[j].raw + } + return out[i].Unit+out[i].Container < out[j].Unit+out[j].Container + }) + if len(out) > limit { + out = out[:limit] + } + for i := range out { + out[i].CPUPct = round1(out[i].raw) + if cpusBusy > 0 { + out[i].SharePct = round1(out[i].raw / (cpusBusy * 100) * 100) + } + } + + unaccounted := cpusBusy*100 - sumPct + if unaccounted < 0 { + unaccounted = 0 + } + unaccountedPct := 0.0 + if cpusBusy > 0 { + unaccountedPct = round1(unaccounted / (cpusBusy * 100) * 100) + } + return out, unaccountedPct +} diff --git a/modules/disk-load/cmd/disk-load/cpu_test.go b/modules/disk-load/cmd/disk-load/cpu_test.go new file mode 100644 index 00000000..f0886fb8 --- /dev/null +++ b/modules/disk-load/cmd/disk-load/cpu_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// aCPUHog is a machine whose CPU is 75 % busy (1.5 of 2 CPUs) over a 2 s window: a unit, another unit, +// and a container, each with a known usage_usec delta, leaving part of the machine's busy time +// unaccounted — the shape of issue 739, where 750 % of 12 CPUs went to processes outside containers. +func aCPUHog(t *testing.T) *fake { + f := newFake(t) + f.file("/proc/stat", "cpu 0 0 0 0 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\nintr 1\n") + f.then("/proc/stat", "cpu 100 0 50 50 0 0 0 0 0 0\ncpu0 50 0 25 25 0 0 0 0 0 0\ncpu1 50 0 25 25 0 0 0 0 0 0\nintr 2\n") + + zero := "usage_usec 0\nuser_usec 0\nsystem_usec 0\n" + cg := func(path string, usec int) { + f.file("/sys/fs/cgroup/"+path+"/cpu.stat", zero) + f.then("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec "+itoa(usec)+"\nuser_usec "+itoa(usec)+"\nsystem_usec 0\n") + } + cg("system.slice/alpha.service", 1000000) // 50 % of one CPU over 2 s + cg("system.slice/beta.service", 400000) // 20 % + cg("system.slice/docker-"+store+".scope", 600000) // 30 %, named "cpu-hog" by the runtime + f.answers["docker ps --no-trunc --format {{.ID}}\t{{.Names}}"] = store + "\tcpu-hog\n" + return f +} + +func TestCPUTopRanksUnitsAndContainersByCPUOverTheWindow(t *testing.T) { + f := aCPUHog(t) + out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0}) + if err != nil { + t.Fatal(err) + } + m := out.(map[string]any) + if m["by"] != "unit" || f.waited != 2*time.Second { + t.Errorf("the default: %+v, waited %s", m, f.waited) + } + cpu := m["cpu"].(MachineCPU) + if cpu.CPUs != 2 || cpu.BusyPct != 75 || cpu.CPUsBusy != 1.5 { + t.Errorf("machine cpu %+v", cpu) + } + top := m["top"].([]CPUUsage) + if len(top) != 3 { + t.Fatalf("three units moved: %+v", top) + } + if top[0].Unit != "alpha.service" || top[0].CPUPct != 50 || top[0].SharePct != 33.3 { + t.Errorf("busiest first: %+v", top[0]) + } + if top[1].Container != "cpu-hog" || top[1].CPUPct != 30 || top[1].SharePct != 20 { + t.Errorf("the container, named by the runtime: %+v", top[1]) + } + if top[2].Unit != "beta.service" || top[2].CPUPct != 20 || top[2].SharePct != 13.3 { + t.Errorf("the least busy: %+v", top[2]) + } + if m["unaccounted_pct"] != 33.3 { + t.Errorf("unaccounted: %+v (machine busy 150%% of one CPU, units sum to 100%%, 50%% of 150%% left over)", m["unaccounted_pct"]) + } +} + +func TestCPUTopByContainerNamesOnlyContainers(t *testing.T) { + out, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"by": "container", "seconds": 2.0}) + if err != nil { + t.Fatal(err) + } + top := out.(map[string]any)["top"].([]CPUUsage) + if len(top) != 1 || top[0].Container != "cpu-hog" || top[0].Unit != "" || top[0].CPUPct != 30 { + t.Errorf("by container: %+v", top) + } + if _, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"by": "process"}); err == nil { + t.Error("by=process was accepted; cpu_top only ranks units and containers") + } +} + +func TestCPUTopLimitsHowManyAreNamed(t *testing.T) { + out, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"limit": 1.0, "seconds": 2.0}) + if err != nil { + t.Fatal(err) + } + m := out.(map[string]any) + top := m["top"].([]CPUUsage) + if len(top) != 1 || top[0].Unit != "alpha.service" { + t.Errorf("limit 1: %+v", top) + } + // The limit narrows the list, never the unaccounted share, which is taken over every unit. + if m["unaccounted_pct"] != 33.3 { + t.Errorf("unaccounted with limit 1: %+v", m["unaccounted_pct"]) + } +} + +func TestCPUTopLeavesOutACgroupThatAppearsOrVanishesWithinTheWindow(t *testing.T) { + f := aCPUHog(t) + f.then("/sys/fs/cgroup/system.slice/short-lived.service/cpu.stat", "usage_usec 500000\n") + out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0}) + if err != nil { + t.Fatal(err) + } + top := out.(map[string]any)["top"].([]CPUUsage) + for _, u := range top { + if u.Unit == "short-lived.service" { + t.Errorf("a unit only in the second reading was counted: %+v", top) + } + } + if !strings.Contains(strings.Join(f.calls, ","), "docker ps") { + t.Errorf("the container runtime was never asked: %v", f.calls) + } +} diff --git a/modules/disk-load/cmd/disk-load/tools.go b/modules/disk-load/cmd/disk-load/tools.go index b54bf21e..0d9c48c5 100644 --- a/modules/disk-load/cmd/disk-load/tools.go +++ b/modules/disk-load/cmd/disk-load/tools.go @@ -85,6 +85,57 @@ func Tools(m *Machine) []stdio.Tool { return out, nil }, }, + { + Name: "cpu_top", + Description: "What used the most CPU over a few seconds, from each unit's cgroup cpu.stat, which needs no root: the machine's " + + "CPU in use (as % of all CPUs and as CPUs busy, from /proc/stat), then by=unit (the default) each systemd unit and " + + "container ranked by CPU, busiest first, with the busy time no unit accounts for; by=container only the containers, " + + "by name. Replaces top, systemd-cgtop. (r)", + Input: map[string]any{ + "seconds": map[string]any{"type": "integer", "description": "how long to sample, in seconds (default 3, at most 10)"}, + "limit": map[string]any{"type": "integer", "description": "how many units or containers to name (default 15, at most 50)"}, + "by": map[string]any{"type": "string", "enum": []string{"unit", "container"}, "description": "what to rank (default unit)"}, + }, + Run: func(args map[string]any) (any, error) { + n, err := count(args, "seconds", 3, 1, 10) + if err != nil { + return nil, err + } + w := time.Duration(n) * time.Second + limit, err := count(args, "limit", 15, 1, 50) + if err != nil { + return nil, err + } + by := text(args, "by") + switch by { + case "": + by = "unit" + case "unit", "container": + default: + return nil, fmt.Errorf("by is unit or container, not %q", by) + } + ac, err := parseCPU(m.read("/proc/stat")) + if err != nil { + return nil, err + } + a := m.unitCPU() + m.Wait(w) + bc, err := parseCPU(m.read("/proc/stat")) + if err != nil { + return nil, err + } + b := m.unitCPU() + machine := machineCPUOver(ac, bc) + top, unaccounted := m.topCPU(a, b, w, by, limit, machine.CPUsBusy) + return map[string]any{ + "window_s": w.Seconds(), + "by": by, + "cpu": machine, + "top": top, + "unaccounted_pct": unaccounted, + }, nil + }, + }, { Name: "disk_filesystems", Description: "The space of every filesystem that stores data (not proc, tmpfs or overlay layers): size, used and free in GiB, " + diff --git a/modules/disk-load/module.json b/modules/disk-load/module.json index 3b7ce911..e50253e6 100644 --- a/modules/disk-load/module.json +++ b/modules/disk-load/module.json @@ -5,7 +5,8 @@ "disk_load", "disk_top", "disk_filesystems", - "disk_devices" + "disk_devices", + "cpu_top" ], "build": { "artifacts": [