package main import ( "strings" "testing" "time" ) // aCPUHog is a machine whose CPU is 75 % busy (1.5 of 2 CPUs) over a 2 s window: a unit, another unit, // and a container, each with a known usage_usec delta, leaving part of the machine's busy time // unaccounted — the shape of issue 739, where 750 % of 12 CPUs went to processes outside containers. func aCPUHog(t *testing.T) *fake { f := newFake(t) f.file("/proc/stat", "cpu 0 0 0 0 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\nintr 1\n") f.then("/proc/stat", "cpu 100 0 50 50 0 0 0 0 0 0\ncpu0 50 0 25 25 0 0 0 0 0 0\ncpu1 50 0 25 25 0 0 0 0 0 0\nintr 2\n") zero := "usage_usec 0\nuser_usec 0\nsystem_usec 0\n" cg := func(path string, usec int) { f.file("/sys/fs/cgroup/"+path+"/cpu.stat", zero) f.then("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec "+itoa(usec)+"\nuser_usec "+itoa(usec)+"\nsystem_usec 0\n") } cg("system.slice/alpha.service", 1000000) // 50 % of one CPU over 2 s cg("system.slice/beta.service", 400000) // 20 % cg("system.slice/docker-"+store+".scope", 600000) // 30 %, named "cpu-hog" by the runtime f.answers["docker ps --no-trunc --format {{.ID}}\t{{.Names}}"] = store + "\tcpu-hog\n" return f } func TestCPUTopRanksUnitsAndContainersByCPUOverTheWindow(t *testing.T) { f := aCPUHog(t) out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0}) if err != nil { t.Fatal(err) } m := out.(map[string]any) if m["by"] != "unit" || f.waited != 2*time.Second { t.Errorf("the default: %+v, waited %s", m, f.waited) } cpu := m["cpu"].(MachineCPU) if cpu.CPUs != 2 || cpu.BusyPct != 75 || cpu.CPUsBusy != 1.5 { t.Errorf("machine cpu %+v", cpu) } top := m["top"].([]CPUUsage) if len(top) != 3 { t.Fatalf("three units moved: %+v", top) } if top[0].Unit != "alpha.service" || top[0].CPUPct != 50 || top[0].SharePct != 33.3 { t.Errorf("busiest first: %+v", top[0]) } if top[1].Container != "cpu-hog" || top[1].CPUPct != 30 || top[1].SharePct != 20 { t.Errorf("the container, named by the runtime: %+v", top[1]) } if top[2].Unit != "beta.service" || top[2].CPUPct != 20 || top[2].SharePct != 13.3 { t.Errorf("the least busy: %+v", top[2]) } if m["unaccounted_pct"] != 33.3 { t.Errorf("unaccounted: %+v (machine busy 150%% of one CPU, units sum to 100%%, 50%% of 150%% left over)", m["unaccounted_pct"]) } } func TestCPUTopByContainerNamesOnlyContainers(t *testing.T) { out, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"by": "container", "seconds": 2.0}) if err != nil { t.Fatal(err) } top := out.(map[string]any)["top"].([]CPUUsage) if len(top) != 1 || top[0].Container != "cpu-hog" || top[0].Unit != "" || top[0].CPUPct != 30 { t.Errorf("by container: %+v", top) } if _, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"by": "process"}); err == nil { t.Error("by=process was accepted; cpu_top only ranks units and containers") } } func TestCPUTopLimitsHowManyAreNamed(t *testing.T) { out, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"limit": 1.0, "seconds": 2.0}) if err != nil { t.Fatal(err) } m := out.(map[string]any) top := m["top"].([]CPUUsage) if len(top) != 1 || top[0].Unit != "alpha.service" { t.Errorf("limit 1: %+v", top) } // The limit narrows the list, never the unaccounted share, which is taken over every unit. if m["unaccounted_pct"] != 33.3 { t.Errorf("unaccounted with limit 1: %+v", m["unaccounted_pct"]) } } func TestCPUTopLeavesOutACgroupThatAppearsOrVanishesWithinTheWindow(t *testing.T) { f := aCPUHog(t) f.then("/sys/fs/cgroup/system.slice/short-lived.service/cpu.stat", "usage_usec 500000\n") out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0}) if err != nil { t.Fatal(err) } top := out.(map[string]any)["top"].([]CPUUsage) for _, u := range top { if u.Unit == "short-lived.service" { t.Errorf("a unit only in the second reading was counted: %+v", top) } } if !strings.Contains(strings.Join(f.calls, ","), "docker ps") { t.Errorf("the container runtime was never asked: %v", f.calls) } } // aTinyLoad is a machine with 12 CPUs, only 0.04 of them busy over a 2 s window (busy 1 of 300 total // ticks): rounded to one decimal for display, CPUsBusy reads 0.0, but a unit using half of that busy // time still has a non-zero share — review round 1's decision that share_pct and unaccounted_pct are // computed from the unrounded busy time, never from the rounded CPUsBusy. func aTinyLoad(t *testing.T) *fake { f := newFake(t) cpus := "cpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\ncpu2 0 0 0 0 0 0 0 0 0 0\ncpu3 0 0 0 0 0 0 0 0 0 0\n" + "cpu4 0 0 0 0 0 0 0 0 0 0\ncpu5 0 0 0 0 0 0 0 0 0 0\ncpu6 0 0 0 0 0 0 0 0 0 0\ncpu7 0 0 0 0 0 0 0 0 0 0\n" + "cpu8 0 0 0 0 0 0 0 0 0 0\ncpu9 0 0 0 0 0 0 0 0 0 0\ncpu10 0 0 0 0 0 0 0 0 0 0\ncpu11 0 0 0 0 0 0 0 0 0 0\n" f.file("/proc/stat", "cpu 0 0 0 0 0 0 0 0 0 0\n"+cpus) f.then("/proc/stat", "cpu 1 0 0 299 0 0 0 0 0 0\n"+cpus) f.file("/sys/fs/cgroup/system.slice/tiny.service/cpu.stat", "usage_usec 0\n") f.then("/sys/fs/cgroup/system.slice/tiny.service/cpu.stat", "usage_usec 60000\n") // 3 % of one CPU over 2 s return f } func TestCPUTopSharesAreNonZeroEvenWhenRoundedCPUsBusyIsZero(t *testing.T) { out, err := tool(t, aTinyLoad(t).machine(), "cpu_top").Run(map[string]any{"seconds": 2.0}) if err != nil { t.Fatal(err) } m := out.(map[string]any) cpu := m["cpu"].(MachineCPU) if cpu.CPUs != 12 || cpu.CPUsBusy != 0 { t.Fatalf("the machine 0.04 CPUs busy, rounded for display: %+v", cpu) } top := m["top"].([]CPUUsage) if len(top) != 1 || top[0].Unit != "tiny.service" || top[0].CPUPct != 3 || top[0].SharePct != 75 { t.Fatalf("a non-zero share from the unrounded busy time: %+v", top) } if m["unaccounted_pct"] != 25.0 { t.Fatalf("unaccounted from the unrounded busy time: %+v (want 25, so shares plus unaccounted add to 100)", m["unaccounted_pct"]) } } // TestCPUTopKeepsCgroupsWithTheSameUnitNameAsSeparateRows is review round 1's other decision: units are keyed // by their cgroup path, never by filepath.Base, so two cgroups that both end in "init.scope" — one at the // root, one under a user's session — are two rows, not one merged by their shared leaf name. func TestCPUTopKeepsCgroupsWithTheSameUnitNameAsSeparateRows(t *testing.T) { f := aCPUHog(t) cg := func(path string, usec int) { f.file("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec 0\n") f.then("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec "+itoa(usec)+"\n") } cg("init.scope", 1000000) // the root init, 50 % of one CPU over 2 s cg("user.slice/user@1000.service/init.scope", 400000) // a user session's init, 20 % out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0, "limit": 10.0}) if err != nil { t.Fatal(err) } top := out.(map[string]any)["top"].([]CPUUsage) var inits []CPUUsage for _, u := range top { if u.Unit == "init.scope" { inits = append(inits, u) } } if len(inits) != 2 { t.Fatalf("two cgroups named init.scope, merged into %d row(s): %+v", len(inits), top) } byCgroup := map[string]CPUUsage{} for _, u := range inits { byCgroup[u.Cgroup] = u } if u, ok := byCgroup["init.scope"]; !ok || u.CPUPct != 50 { t.Errorf("the root init.scope: %+v", byCgroup) } if u, ok := byCgroup["user.slice/user@1000.service/init.scope"]; !ok || u.CPUPct != 20 { t.Errorf("the user session's init.scope: %+v", byCgroup) } } // TestCPUTopRanksAUnitsProcesses is the unit argument: a session scope with a child cgroup, ranked by // the utime+stime each process used over a 2 s window; one process ends, one starts, one comm holds ")". func TestCPUTopRanksAUnitsProcesses(t *testing.T) { f := aCPUHog(t) f.file("/sys/fs/cgroup/user.slice/session-c1.scope/cgroup.procs", "10\n11\n12\n") f.file("/sys/fs/cgroup/user.slice/session-c1.scope/app/cgroup.procs", "13\n") st := func(pid int, comm string, ticks, threads int) { f.then("/proc/"+itoa(pid)+"/stat", itoa(pid)+" ("+comm+") S 1 1 1 0 -1 0 0 0 0 0 "+itoa(ticks)+" 0 0 0 20 0 "+itoa(threads)+" 0 1 1 1\n") } for pid, c := range map[int][2]string{10: {"firefox", "1"}, 11: {"sh (x)", "1"}, 12: {"gone", "1"}} { f.file("/proc/"+itoa(pid)+"/stat", itoa(pid)+" ("+c[0]+") S 1 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 "+c[1]+" 0 1 1 1\n") } f.file("/proc/13/stat", "13 (deep) S 1 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 4 0 1 1 1\n") f.file("/proc/10/cmdline", "/usr/bin/firefox\x00--new-window\x00") f.file("/proc/11/cmdline", "mysql\x00--password\x00hunter2\x00") f.file("/proc/13/cmdline", "/usr/bin/tool --password hunter2 -x") // a rewritten title: one string, spaces, no NUL st(10, "firefox", 150, 40) // 75 % of one CPU over 2 s st(11, "sh (x)", 40, 1) // 20 % st(13, "deep", 10, 4) // 5 %, in the child cgroup f.then("/sys/fs/cgroup/user.slice/session-c1.scope/cgroup.procs", "10\n11\n14\n") f.file("/sys/fs/cgroup/user.slice/session-c1.scope/app/cgroup.procs", "13\n15\n") f.file("/proc/15/stat", "15 (old) S 1 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 1 1 1\n") f.then("/proc/15/stat", "15 (reused) S 1 1 1 0 -1 0 0 0 0 0 900 0 0 0 20 0 1 0 77 1 1\n") // new starttime: another process st(14, "newborn", 500, 1) // not in the first reading: skipped out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"unit": "session-c1.scope", "seconds": 2.0, "limit": 3.0}) if err != nil { t.Fatal(err) } top := out.(map[string]any)["top"].([]ProcessCPU) if len(top) != 3 || top[2].PID != 13 || top[2].CPUPct != 5 || top[0].PID != 10 || top[0].CPUPct != 75 || top[0].SharePct != 75 || top[0].Threads != 40 || top[0].Command != "firefox: /usr/bin/firefox" || top[1].PID != 11 || top[1].CPUPct != 20 || top[1].Command != "sh (x): mysql" || top[2].Command != "deep: /usr/bin/tool" { t.Errorf("processes of the unit: %+v", top) } for _, p := range top { if p.PID == 15 || strings.Contains(p.Command, "hunter2") { t.Errorf("a reused pid was counted, or an argument leaked: %+v", p) } } for _, u := range []string{"../../etc", "/sys/fs/cgroup/../../proc", "user.slice/../.."} { if _, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"unit": u}); err == nil { t.Errorf("cgroup path %q escaping the root was accepted", u) } } if _, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"unit": "nothing.scope"}); err == nil { t.Error("an unknown unit was accepted") } }