jschoubben
3112692def
mesh/delivery merged: source.main moved to novox/mesh-catalog@1d1d5ec443d0
mesh/check being carried to 3112692defdf on 1d1d5ec443d0 | from 4915ff5d293f
232 lines
10 KiB
Go
232 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// aCPUHog is a machine whose CPU is 75 % busy (1.5 of 2 CPUs) over a 2 s window: a unit, another unit,
|
|
// and a container, each with a known usage_usec delta, leaving part of the machine's busy time
|
|
// unaccounted — the shape of issue 739, where 750 % of 12 CPUs went to processes outside containers.
|
|
func aCPUHog(t *testing.T) *fake {
|
|
f := newFake(t)
|
|
f.file("/proc/stat", "cpu 0 0 0 0 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\nintr 1\n")
|
|
f.then("/proc/stat", "cpu 100 0 50 50 0 0 0 0 0 0\ncpu0 50 0 25 25 0 0 0 0 0 0\ncpu1 50 0 25 25 0 0 0 0 0 0\nintr 2\n")
|
|
|
|
zero := "usage_usec 0\nuser_usec 0\nsystem_usec 0\n"
|
|
cg := func(path string, usec int) {
|
|
f.file("/sys/fs/cgroup/"+path+"/cpu.stat", zero)
|
|
f.then("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec "+itoa(usec)+"\nuser_usec "+itoa(usec)+"\nsystem_usec 0\n")
|
|
}
|
|
cg("system.slice/alpha.service", 1000000) // 50 % of one CPU over 2 s
|
|
cg("system.slice/beta.service", 400000) // 20 %
|
|
cg("system.slice/docker-"+store+".scope", 600000) // 30 %, named "cpu-hog" by the runtime
|
|
f.answers["docker ps --no-trunc --format {{.ID}}\t{{.Names}}"] = store + "\tcpu-hog\n"
|
|
return f
|
|
}
|
|
|
|
func TestCPUTopRanksUnitsAndContainersByCPUOverTheWindow(t *testing.T) {
|
|
f := aCPUHog(t)
|
|
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m := out.(map[string]any)
|
|
if m["by"] != "unit" || f.waited != 2*time.Second {
|
|
t.Errorf("the default: %+v, waited %s", m, f.waited)
|
|
}
|
|
cpu := m["cpu"].(MachineCPU)
|
|
if cpu.CPUs != 2 || cpu.BusyPct != 75 || cpu.CPUsBusy != 1.5 {
|
|
t.Errorf("machine cpu %+v", cpu)
|
|
}
|
|
top := m["top"].([]CPUUsage)
|
|
if len(top) != 3 {
|
|
t.Fatalf("three units moved: %+v", top)
|
|
}
|
|
if top[0].Unit != "alpha.service" || top[0].CPUPct != 50 || top[0].SharePct != 33.3 {
|
|
t.Errorf("busiest first: %+v", top[0])
|
|
}
|
|
if top[1].Container != "cpu-hog" || top[1].CPUPct != 30 || top[1].SharePct != 20 {
|
|
t.Errorf("the container, named by the runtime: %+v", top[1])
|
|
}
|
|
if top[2].Unit != "beta.service" || top[2].CPUPct != 20 || top[2].SharePct != 13.3 {
|
|
t.Errorf("the least busy: %+v", top[2])
|
|
}
|
|
if m["unaccounted_pct"] != 33.3 {
|
|
t.Errorf("unaccounted: %+v (machine busy 150%% of one CPU, units sum to 100%%, 50%% of 150%% left over)", m["unaccounted_pct"])
|
|
}
|
|
}
|
|
|
|
func TestCPUTopByContainerNamesOnlyContainers(t *testing.T) {
|
|
out, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"by": "container", "seconds": 2.0})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
top := out.(map[string]any)["top"].([]CPUUsage)
|
|
if len(top) != 1 || top[0].Container != "cpu-hog" || top[0].Unit != "" || top[0].CPUPct != 30 {
|
|
t.Errorf("by container: %+v", top)
|
|
}
|
|
if _, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"by": "process"}); err == nil {
|
|
t.Error("by=process was accepted; cpu_top only ranks units and containers")
|
|
}
|
|
}
|
|
|
|
func TestCPUTopLimitsHowManyAreNamed(t *testing.T) {
|
|
out, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"limit": 1.0, "seconds": 2.0})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m := out.(map[string]any)
|
|
top := m["top"].([]CPUUsage)
|
|
if len(top) != 1 || top[0].Unit != "alpha.service" {
|
|
t.Errorf("limit 1: %+v", top)
|
|
}
|
|
// The limit narrows the list, never the unaccounted share, which is taken over every unit.
|
|
if m["unaccounted_pct"] != 33.3 {
|
|
t.Errorf("unaccounted with limit 1: %+v", m["unaccounted_pct"])
|
|
}
|
|
}
|
|
|
|
func TestCPUTopLeavesOutACgroupThatAppearsOrVanishesWithinTheWindow(t *testing.T) {
|
|
f := aCPUHog(t)
|
|
f.then("/sys/fs/cgroup/system.slice/short-lived.service/cpu.stat", "usage_usec 500000\n")
|
|
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
top := out.(map[string]any)["top"].([]CPUUsage)
|
|
for _, u := range top {
|
|
if u.Unit == "short-lived.service" {
|
|
t.Errorf("a unit only in the second reading was counted: %+v", top)
|
|
}
|
|
}
|
|
if !strings.Contains(strings.Join(f.calls, ","), "docker ps") {
|
|
t.Errorf("the container runtime was never asked: %v", f.calls)
|
|
}
|
|
}
|
|
|
|
// aTinyLoad is a machine with 12 CPUs, only 0.04 of them busy over a 2 s window (busy 1 of 300 total
|
|
// ticks): rounded to one decimal for display, CPUsBusy reads 0.0, but a unit using half of that busy
|
|
// time still has a non-zero share — review round 1's decision that share_pct and unaccounted_pct are
|
|
// computed from the unrounded busy time, never from the rounded CPUsBusy.
|
|
func aTinyLoad(t *testing.T) *fake {
|
|
f := newFake(t)
|
|
cpus := "cpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\ncpu2 0 0 0 0 0 0 0 0 0 0\ncpu3 0 0 0 0 0 0 0 0 0 0\n" +
|
|
"cpu4 0 0 0 0 0 0 0 0 0 0\ncpu5 0 0 0 0 0 0 0 0 0 0\ncpu6 0 0 0 0 0 0 0 0 0 0\ncpu7 0 0 0 0 0 0 0 0 0 0\n" +
|
|
"cpu8 0 0 0 0 0 0 0 0 0 0\ncpu9 0 0 0 0 0 0 0 0 0 0\ncpu10 0 0 0 0 0 0 0 0 0 0\ncpu11 0 0 0 0 0 0 0 0 0 0\n"
|
|
f.file("/proc/stat", "cpu 0 0 0 0 0 0 0 0 0 0\n"+cpus)
|
|
f.then("/proc/stat", "cpu 1 0 0 299 0 0 0 0 0 0\n"+cpus)
|
|
|
|
f.file("/sys/fs/cgroup/system.slice/tiny.service/cpu.stat", "usage_usec 0\n")
|
|
f.then("/sys/fs/cgroup/system.slice/tiny.service/cpu.stat", "usage_usec 60000\n") // 3 % of one CPU over 2 s
|
|
return f
|
|
}
|
|
|
|
func TestCPUTopSharesAreNonZeroEvenWhenRoundedCPUsBusyIsZero(t *testing.T) {
|
|
out, err := tool(t, aTinyLoad(t).machine(), "cpu_top").Run(map[string]any{"seconds": 2.0})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m := out.(map[string]any)
|
|
cpu := m["cpu"].(MachineCPU)
|
|
if cpu.CPUs != 12 || cpu.CPUsBusy != 0 {
|
|
t.Fatalf("the machine 0.04 CPUs busy, rounded for display: %+v", cpu)
|
|
}
|
|
top := m["top"].([]CPUUsage)
|
|
if len(top) != 1 || top[0].Unit != "tiny.service" || top[0].CPUPct != 3 || top[0].SharePct != 75 {
|
|
t.Fatalf("a non-zero share from the unrounded busy time: %+v", top)
|
|
}
|
|
if m["unaccounted_pct"] != 25.0 {
|
|
t.Fatalf("unaccounted from the unrounded busy time: %+v (want 25, so shares plus unaccounted add to 100)",
|
|
m["unaccounted_pct"])
|
|
}
|
|
}
|
|
|
|
// TestCPUTopKeepsCgroupsWithTheSameUnitNameAsSeparateRows is review round 1's other decision: units are keyed
|
|
// by their cgroup path, never by filepath.Base, so two cgroups that both end in "init.scope" — one at the
|
|
// root, one under a user's session — are two rows, not one merged by their shared leaf name.
|
|
func TestCPUTopKeepsCgroupsWithTheSameUnitNameAsSeparateRows(t *testing.T) {
|
|
f := aCPUHog(t)
|
|
cg := func(path string, usec int) {
|
|
f.file("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec 0\n")
|
|
f.then("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec "+itoa(usec)+"\n")
|
|
}
|
|
cg("init.scope", 1000000) // the root init, 50 % of one CPU over 2 s
|
|
cg("user.slice/user@1000.service/init.scope", 400000) // a user session's init, 20 %
|
|
|
|
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0, "limit": 10.0})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
top := out.(map[string]any)["top"].([]CPUUsage)
|
|
var inits []CPUUsage
|
|
for _, u := range top {
|
|
if u.Unit == "init.scope" {
|
|
inits = append(inits, u)
|
|
}
|
|
}
|
|
if len(inits) != 2 {
|
|
t.Fatalf("two cgroups named init.scope, merged into %d row(s): %+v", len(inits), top)
|
|
}
|
|
byCgroup := map[string]CPUUsage{}
|
|
for _, u := range inits {
|
|
byCgroup[u.Cgroup] = u
|
|
}
|
|
if u, ok := byCgroup["init.scope"]; !ok || u.CPUPct != 50 {
|
|
t.Errorf("the root init.scope: %+v", byCgroup)
|
|
}
|
|
if u, ok := byCgroup["user.slice/user@1000.service/init.scope"]; !ok || u.CPUPct != 20 {
|
|
t.Errorf("the user session's init.scope: %+v", byCgroup)
|
|
}
|
|
}
|
|
|
|
// TestCPUTopRanksAUnitsProcesses is the unit argument: a session scope with a child cgroup, ranked by
|
|
// the utime+stime each process used over a 2 s window; one process ends, one starts, one comm holds ")".
|
|
func TestCPUTopRanksAUnitsProcesses(t *testing.T) {
|
|
f := aCPUHog(t)
|
|
f.file("/sys/fs/cgroup/user.slice/session-c1.scope/cgroup.procs", "10\n11\n12\n")
|
|
f.file("/sys/fs/cgroup/user.slice/session-c1.scope/app/cgroup.procs", "13\n")
|
|
st := func(pid int, comm string, ticks, threads int) {
|
|
f.then("/proc/"+itoa(pid)+"/stat", itoa(pid)+" ("+comm+") S 1 1 1 0 -1 0 0 0 0 0 "+itoa(ticks)+" 0 0 0 20 0 "+itoa(threads)+" 0 1 1 1\n")
|
|
}
|
|
for pid, c := range map[int][2]string{10: {"firefox", "1"}, 11: {"sh (x)", "1"}, 12: {"gone", "1"}} {
|
|
f.file("/proc/"+itoa(pid)+"/stat", itoa(pid)+" ("+c[0]+") S 1 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 "+c[1]+" 0 1 1 1\n")
|
|
}
|
|
f.file("/proc/13/stat", "13 (deep) S 1 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 4 0 1 1 1\n")
|
|
f.file("/proc/10/cmdline", "/usr/bin/firefox\x00--new-window\x00")
|
|
f.file("/proc/11/cmdline", "mysql\x00--password\x00hunter2\x00")
|
|
f.file("/proc/13/cmdline", "/usr/bin/tool --password hunter2 -x") // a rewritten title: one string, spaces, no NUL
|
|
st(10, "firefox", 150, 40) // 75 % of one CPU over 2 s
|
|
st(11, "sh (x)", 40, 1) // 20 %
|
|
st(13, "deep", 10, 4) // 5 %, in the child cgroup
|
|
f.then("/sys/fs/cgroup/user.slice/session-c1.scope/cgroup.procs", "10\n11\n14\n")
|
|
f.file("/sys/fs/cgroup/user.slice/session-c1.scope/app/cgroup.procs", "13\n15\n")
|
|
f.file("/proc/15/stat", "15 (old) S 1 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 1 1 1\n")
|
|
f.then("/proc/15/stat", "15 (reused) S 1 1 1 0 -1 0 0 0 0 0 900 0 0 0 20 0 1 0 77 1 1\n") // new starttime: another process
|
|
st(14, "newborn", 500, 1) // not in the first reading: skipped
|
|
|
|
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"unit": "session-c1.scope", "seconds": 2.0, "limit": 3.0})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
top := out.(map[string]any)["top"].([]ProcessCPU)
|
|
if len(top) != 3 || top[2].PID != 13 || top[2].CPUPct != 5 || top[0].PID != 10 || top[0].CPUPct != 75 || top[0].SharePct != 75 || top[0].Threads != 40 ||
|
|
top[0].Command != "firefox: /usr/bin/firefox" || top[1].PID != 11 || top[1].CPUPct != 20 || top[1].Command != "sh (x): mysql" || top[2].Command != "deep: /usr/bin/tool" {
|
|
t.Errorf("processes of the unit: %+v", top)
|
|
}
|
|
for _, p := range top {
|
|
if p.PID == 15 || strings.Contains(p.Command, "hunter2") {
|
|
t.Errorf("a reused pid was counted, or an argument leaked: %+v", p)
|
|
}
|
|
}
|
|
for _, u := range []string{"../../etc", "/sys/fs/cgroup/../../proc", "user.slice/../.."} {
|
|
if _, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"unit": u}); err == nil {
|
|
t.Errorf("cgroup path %q escaping the root was accepted", u)
|
|
}
|
|
}
|
|
if _, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"unit": "nothing.scope"}); err == nil {
|
|
t.Error("an unknown unit was accepted")
|
|
}
|
|
}
|