Add disk-load's cpu_top tool: CPU by systemd unit and container (issue 741)

No tool named which units or processes used CPU outside containers during
the novox overload (issue 739). cpu_top reads each cgroup's cpu.stat (no
root needed), ranks units or containers busiest first, and says the
machine's CPU in use and the busy time no unit accounts for.

modules/disk-load/cmd/disk-load/cpu_test.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+// aCPUHog is a machine whose CPU is 75 % busy (1.5 of 2 CPUs) over a 2 s window: a unit, another unit,
+// and a container, each with a known usage_usec delta, leaving part of the machine's busy time
+// unaccounted — the shape of issue 739, where 750 % of 12 CPUs went to processes outside containers.
+func aCPUHog(t *testing.T) *fake {
+	f := newFake(t)
+	f.file("/proc/stat", "cpu  0 0 0 0 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\nintr 1\n")
+	f.then("/proc/stat", "cpu  100 0 50 50 0 0 0 0 0 0\ncpu0 50 0 25 25 0 0 0 0 0 0\ncpu1 50 0 25 25 0 0 0 0 0 0\nintr 2\n")
+
+	zero := "usage_usec 0\nuser_usec 0\nsystem_usec 0\n"
+	cg := func(path string, usec int) {
+		f.file("/sys/fs/cgroup/"+path+"/cpu.stat", zero)
+		f.then("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec "+itoa(usec)+"\nuser_usec "+itoa(usec)+"\nsystem_usec 0\n")
+	}
+	cg("system.slice/alpha.service", 1000000)         // 50 % of one CPU over 2 s
+	cg("system.slice/beta.service", 400000)           // 20 %
+	cg("system.slice/docker-"+store+".scope", 600000) // 30 %, named "cpu-hog" by the runtime
+	f.answers["docker ps --no-trunc --format {{.ID}}\t{{.Names}}"] = store + "\tcpu-hog\n"
+	return f
+}
+
+func TestCPUTopRanksUnitsAndContainersByCPUOverTheWindow(t *testing.T) {
+	f := aCPUHog(t)
+	out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0})
+	if err != nil {
+		t.Fatal(err)
+	}
+	m := out.(map[string]any)
+	if m["by"] != "unit" || f.waited != 2*time.Second {
+		t.Errorf("the default: %+v, waited %s", m, f.waited)
+	}
+	cpu := m["cpu"].(MachineCPU)
+	if cpu.CPUs != 2 || cpu.BusyPct != 75 || cpu.CPUsBusy != 1.5 {
+		t.Errorf("machine cpu %+v", cpu)
+	}
+	top := m["top"].([]CPUUsage)
+	if len(top) != 3 {
+		t.Fatalf("three units moved: %+v", top)
+	}
+	if top[0].Unit != "alpha.service" || top[0].CPUPct != 50 || top[0].SharePct != 33.3 {
+		t.Errorf("busiest first: %+v", top[0])
+	}
+	if top[1].Container != "cpu-hog" || top[1].CPUPct != 30 || top[1].SharePct != 20 {
+		t.Errorf("the container, named by the runtime: %+v", top[1])
+	}
+	if top[2].Unit != "beta.service" || top[2].CPUPct != 20 || top[2].SharePct != 13.3 {
+		t.Errorf("the least busy: %+v", top[2])
+	}
+	if m["unaccounted_pct"] != 33.3 {
+		t.Errorf("unaccounted: %+v (machine busy 150%% of one CPU, units sum to 100%%, 50%% of 150%% left over)", m["unaccounted_pct"])
+	}
+}
+
+func TestCPUTopByContainerNamesOnlyContainers(t *testing.T) {
+	out, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"by": "container", "seconds": 2.0})
+	if err != nil {
+		t.Fatal(err)
+	}
+	top := out.(map[string]any)["top"].([]CPUUsage)
+	if len(top) != 1 || top[0].Container != "cpu-hog" || top[0].Unit != "" || top[0].CPUPct != 30 {
+		t.Errorf("by container: %+v", top)
+	}
+	if _, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"by": "process"}); err == nil {
+		t.Error("by=process was accepted; cpu_top only ranks units and containers")
+	}
+}
+
+func TestCPUTopLimitsHowManyAreNamed(t *testing.T) {
+	out, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"limit": 1.0, "seconds": 2.0})
+	if err != nil {
+		t.Fatal(err)
+	}
+	m := out.(map[string]any)
+	top := m["top"].([]CPUUsage)
+	if len(top) != 1 || top[0].Unit != "alpha.service" {
+		t.Errorf("limit 1: %+v", top)
+	}
+	// The limit narrows the list, never the unaccounted share, which is taken over every unit.
+	if m["unaccounted_pct"] != 33.3 {
+		t.Errorf("unaccounted with limit 1: %+v", m["unaccounted_pct"])
+	}
+}
+
+func TestCPUTopLeavesOutACgroupThatAppearsOrVanishesWithinTheWindow(t *testing.T) {
+	f := aCPUHog(t)
+	f.then("/sys/fs/cgroup/system.slice/short-lived.service/cpu.stat", "usage_usec 500000\n")
+	out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0})
+	if err != nil {
+		t.Fatal(err)
+	}
+	top := out.(map[string]any)["top"].([]CPUUsage)
+	for _, u := range top {
+		if u.Unit == "short-lived.service" {
+			t.Errorf("a unit only in the second reading was counted: %+v", top)
+		}
+	}
+	if !strings.Contains(strings.Join(f.calls, ","), "docker ps") {
+		t.Errorf("the container runtime was never asked: %v", f.calls)
+	}
+}