Merge pull request 'disk-load's cpu_top names the processes inside one unit' (#600) from fix/741-cpu-by-process into main
This commit was merged in pull request #600.
This commit is contained in:
@@ -33,7 +33,7 @@ All six only read.
|
||||
| `disk_filesystems` | Each filesystem that stores data, said once however many places it is mounted: size, used and free GiB, used % as `df` counts it, inodes used % | `df -h`, `df -i` |
|
||||
| `disk_devices` | Each disk without sampling: model, size, whether it spins, scheduler, queue limit, what it is built on, and since boot the GiB read and written, the busy share and the wait per read and per write | `lsblk`, `cat /proc/diskstats` |
|
||||
| `disk_folders` | Which folders fill a disk, like `du -x`: the largest folders under `path` (default `/`) to `depth` levels (default 2, at most 3), `limit` of them (default 20, at most 100), with bytes and GiB. Sizes are allocated bytes, a hard link once. It stays on the path's one filesystem: a mount below it is named, never entered. It reads every folder the tool runner can read (docker's included) and no content; folders it cannot read are counted and named. After 25 s (the tool runner gives a call 30 s) it answers what it has measured, marked `partial`, with `stopped_at` (novox/hq issue 1804) | `du -x`, `ncdu` |
|
||||
| `cpu_top` | Samples a window (`seconds`, default 3, at most 10). It answers the machine's CPU in use (% of all CPUs and as CPUs busy, from `/proc/stat`), then `by=unit` (the default) ranks systemd units and containers by CPU, from their cgroups' cpu.stat, busiest first (`limit`, default 15), with the busy time no unit accounts for; `by=container` ranks containers by name | `top`, `systemd-cgtop` |
|
||||
| `cpu_top` | Samples a window (`seconds`, default 3, at most 10). It answers the machine's CPU in use (% of all CPUs and as CPUs busy, from `/proc/stat`), then `by=unit` (the default) ranks systemd units and containers by CPU, from their cgroups' cpu.stat, busiest first (`limit`, default 15), with the busy time no unit accounts for; `by=container` ranks containers by name With `unit` (a unit name such as `session-c1.scope`, or a cgroup path) it ranks that unit's processes instead (pid, command, cpu_pct, share_pct of the unit's busy time, threads), from `/proc/<pid>/stat` | `top`, `systemd-cgtop` |
|
||||
|
||||
### How each number is read
|
||||
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// clockTicks is USER_HZ, the unit of utime and stime in /proc/<pid>/stat: 100 on every Linux the mesh runs.
|
||||
const clockTicks = 100.0
|
||||
|
||||
// ProcessCPU is one process of a unit and its share of CPU over the window (cpu_top with unit set).
|
||||
type ProcessCPU struct {
|
||||
PID int `json:"pid"`
|
||||
Command string `json:"command"` // comm, then the first 120 characters of the command line
|
||||
CPUPct float64 `json:"cpu_pct"` // % of one CPU: 100 = one CPU busy the whole window
|
||||
SharePct float64 `json:"share_pct"` // this process's share of the unit's busy time
|
||||
Threads int `json:"threads"`
|
||||
raw float64
|
||||
}
|
||||
|
||||
// procTicks is a process's utime+stime ticks and thread count from its /proc/<pid>/stat, whose comm
|
||||
// field may hold spaces and parentheses, so the fields are counted from the last ")".
|
||||
func parseProcStat(s string) (comm string, ticks uint64, threads int, start string, ok bool) {
|
||||
l, r := strings.Index(s, "("), strings.LastIndex(s, ")")
|
||||
if l < 0 || r < l {
|
||||
return "", 0, 0, "", false
|
||||
}
|
||||
f := strings.Fields(s[r+1:])
|
||||
if len(f) < 20 {
|
||||
return "", 0, 0, "", false
|
||||
}
|
||||
ut, e1 := strconv.ParseUint(f[11], 10, 64)
|
||||
st, e2 := strconv.ParseUint(f[12], 10, 64)
|
||||
th, e3 := strconv.Atoi(f[17])
|
||||
if e1 != nil || e2 != nil || e3 != nil {
|
||||
return "", 0, 0, "", false
|
||||
}
|
||||
return s[l+1 : r], ut + st, th, f[19], true
|
||||
}
|
||||
|
||||
// cgroupOfUnit is the cgroup path (relative to /sys/fs/cgroup) a unit name or a path names.
|
||||
func (m *Machine) cgroupOfUnit(unit string) (string, error) {
|
||||
base := m.path("/sys/fs/cgroup")
|
||||
if strings.Contains(unit, "/") {
|
||||
rel := strings.Trim(strings.TrimPrefix(unit, "/sys/fs/cgroup"), "/")
|
||||
if strings.Contains(rel, "..") || filepath.Clean("/"+rel) != "/"+rel {
|
||||
return "", fmt.Errorf("cgroup %q is not a path under /sys/fs/cgroup", unit)
|
||||
}
|
||||
if rel == "" || !m.exists("/sys/fs/cgroup/"+rel) {
|
||||
return "", fmt.Errorf("no cgroup %q", unit)
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
var found []string
|
||||
filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
|
||||
if err == nil && d.IsDir() && d.Name() == unit {
|
||||
rel, _ := filepath.Rel(base, p)
|
||||
found = append(found, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
switch len(found) {
|
||||
case 0:
|
||||
return "", fmt.Errorf("no unit %q in the cgroup tree", unit)
|
||||
case 1:
|
||||
return found[0], nil
|
||||
}
|
||||
return "", fmt.Errorf("unit %q is %d cgroups (%s): give the cgroup path", unit, len(found), strings.Join(found, ", "))
|
||||
}
|
||||
|
||||
type procReading struct {
|
||||
ticks uint64
|
||||
comm string
|
||||
threads int
|
||||
pid int
|
||||
start string // starttime, to tell a reused pid
|
||||
}
|
||||
|
||||
// unitProcs reads every process in a cgroup and its child cgroups: its ticks, name and threads.
|
||||
func (m *Machine) unitProcs(cgroup string) map[int]procReading {
|
||||
out := map[int]procReading{}
|
||||
base := m.path("/sys/fs/cgroup/" + cgroup)
|
||||
filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || d.Name() != "cgroup.procs" {
|
||||
return nil
|
||||
}
|
||||
for _, f := range strings.Fields(m.read(strings.TrimPrefix(p, filepath.Clean(m.Root)))) {
|
||||
pid, err := strconv.Atoi(f)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
comm, ticks, th, start, ok := parseProcStat(m.read("/proc/" + f + "/stat"))
|
||||
if ok {
|
||||
out[pid] = procReading{ticks: ticks, comm: comm, threads: th, pid: pid, start: start}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// commandLine is a process's program path (argv[0], cut to 120 characters), never its arguments:
|
||||
// they can carry a password.
|
||||
func (m *Machine) commandLine(pid int) string {
|
||||
s, _, _ := strings.Cut(m.read("/proc/"+strconv.Itoa(pid)+"/cmdline"), "\x00")
|
||||
s = strings.TrimSpace(s)
|
||||
if r := []rune(s); len(r) > 120 {
|
||||
s = string(r[:120])
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// topProcessCPU ranks a unit's processes by CPU between two readings; a process missing from either
|
||||
// reading (started or ended in the window) is skipped.
|
||||
func (m *Machine) topProcessCPU(a, b map[int]procReading, window time.Duration, limit int) []ProcessCPU {
|
||||
var out []ProcessCPU
|
||||
var sum float64
|
||||
for pid, end := range b {
|
||||
start, ok := a[pid]
|
||||
if !ok || end.start != start.start || end.ticks <= start.ticks {
|
||||
continue
|
||||
}
|
||||
pct := float64(end.ticks-start.ticks) / clockTicks / window.Seconds() * 100
|
||||
sum += pct
|
||||
out = append(out, ProcessCPU{PID: pid, Command: end.comm, Threads: end.threads, raw: pct})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].raw != out[j].raw {
|
||||
return out[i].raw > out[j].raw
|
||||
}
|
||||
return out[i].PID < out[j].PID
|
||||
})
|
||||
if len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
for i := range out {
|
||||
out[i].CPUPct = round1(out[i].raw)
|
||||
out[i].SharePct = round1(out[i].raw / sum * 100)
|
||||
if cl := m.commandLine(out[i].PID); cl != "" {
|
||||
out[i].Command += ": " + cl
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -180,3 +180,51 @@ func TestCPUTopKeepsCgroupsWithTheSameUnitNameAsSeparateRows(t *testing.T) {
|
||||
t.Errorf("the user session's init.scope: %+v", byCgroup)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCPUTopRanksAUnitsProcesses is the unit argument: a session scope with a child cgroup, ranked by
|
||||
// the utime+stime each process used over a 2 s window; one process ends, one starts, one comm holds ")".
|
||||
func TestCPUTopRanksAUnitsProcesses(t *testing.T) {
|
||||
f := aCPUHog(t)
|
||||
f.file("/sys/fs/cgroup/user.slice/session-c1.scope/cgroup.procs", "10\n11\n12\n")
|
||||
f.file("/sys/fs/cgroup/user.slice/session-c1.scope/app/cgroup.procs", "13\n")
|
||||
st := func(pid int, comm string, ticks, threads int) {
|
||||
f.then("/proc/"+itoa(pid)+"/stat", itoa(pid)+" ("+comm+") S 1 1 1 0 -1 0 0 0 0 0 "+itoa(ticks)+" 0 0 0 20 0 "+itoa(threads)+" 0 1 1 1\n")
|
||||
}
|
||||
for pid, c := range map[int][2]string{10: {"firefox", "1"}, 11: {"sh (x)", "1"}, 12: {"gone", "1"}} {
|
||||
f.file("/proc/"+itoa(pid)+"/stat", itoa(pid)+" ("+c[0]+") S 1 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 "+c[1]+" 0 1 1 1\n")
|
||||
}
|
||||
f.file("/proc/13/stat", "13 (deep) S 1 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 4 0 1 1 1\n")
|
||||
f.file("/proc/10/cmdline", "/usr/bin/firefox\x00--new-window\x00")
|
||||
f.file("/proc/11/cmdline", "mysql\x00--password\x00hunter2\x00")
|
||||
st(10, "firefox", 150, 40) // 75 % of one CPU over 2 s
|
||||
st(11, "sh (x)", 40, 1) // 20 %
|
||||
st(13, "deep", 10, 4) // 5 %, in the child cgroup
|
||||
f.then("/sys/fs/cgroup/user.slice/session-c1.scope/cgroup.procs", "10\n11\n14\n")
|
||||
f.file("/sys/fs/cgroup/user.slice/session-c1.scope/app/cgroup.procs", "13\n15\n")
|
||||
f.file("/proc/15/stat", "15 (old) S 1 1 1 0 -1 0 0 0 0 0 0 0 0 0 20 0 1 0 1 1 1\n")
|
||||
f.then("/proc/15/stat", "15 (reused) S 1 1 1 0 -1 0 0 0 0 0 900 0 0 0 20 0 1 0 77 1 1\n") // new starttime: another process
|
||||
st(14, "newborn", 500, 1) // not in the first reading: skipped
|
||||
|
||||
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"unit": "session-c1.scope", "seconds": 2.0, "limit": 3.0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
top := out.(map[string]any)["top"].([]ProcessCPU)
|
||||
if len(top) != 3 || top[2].PID != 13 || top[2].CPUPct != 5 || top[0].PID != 10 || top[0].CPUPct != 75 || top[0].SharePct != 75 || top[0].Threads != 40 ||
|
||||
top[0].Command != "firefox: /usr/bin/firefox" || top[1].PID != 11 || top[1].CPUPct != 20 || top[1].Command != "sh (x): mysql" {
|
||||
t.Errorf("processes of the unit: %+v", top)
|
||||
}
|
||||
for _, p := range top {
|
||||
if p.PID == 15 || strings.Contains(p.Command, "hunter2") {
|
||||
t.Errorf("a reused pid was counted, or an argument leaked: %+v", p)
|
||||
}
|
||||
}
|
||||
for _, u := range []string{"../../etc", "/sys/fs/cgroup/../../proc", "user.slice/../.."} {
|
||||
if _, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"unit": u}); err == nil {
|
||||
t.Errorf("cgroup path %q escaping the root was accepted", u)
|
||||
}
|
||||
}
|
||||
if _, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"unit": "nothing.scope"}); err == nil {
|
||||
t.Error("an unknown unit was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,11 +90,14 @@ func Tools(m *Machine) []stdio.Tool {
|
||||
Description: "What used the most CPU over a few seconds, from each unit's cgroup cpu.stat, which needs no root: the machine's " +
|
||||
"CPU in use (as % of all CPUs and as CPUs busy, from /proc/stat), then by=unit (the default) each systemd unit and " +
|
||||
"container ranked by CPU, busiest first, with the busy time no unit accounts for; by=container only the containers, " +
|
||||
"by name. Replaces top, systemd-cgtop. (r)",
|
||||
"by name. With unit (a systemd unit such as session-c1.scope, or its cgroup path) it ranks that unit's processes instead, CPU by process, the top process: " +
|
||||
"pid, command, cpu_pct, share_pct of the unit's busy time and threads, from /proc/<pid>/stat, needing no root; a desktop session is one unit, " +
|
||||
"and this names the program in it. Replaces top, systemd-cgtop. (r)",
|
||||
Input: map[string]any{
|
||||
"seconds": map[string]any{"type": "integer", "description": "how long to sample, in seconds (default 3, at most 10)"},
|
||||
"limit": map[string]any{"type": "integer", "description": "how many units or containers to name (default 15, at most 50)"},
|
||||
"by": map[string]any{"type": "string", "enum": []string{"unit", "container"}, "description": "what to rank (default unit)"},
|
||||
"unit": map[string]any{"type": "string", "description": "a systemd unit name (session-c1.scope) or cgroup path: rank that unit's processes by CPU instead"},
|
||||
},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
n, err := count(args, "seconds", 3, 1, 10)
|
||||
@@ -118,6 +121,25 @@ func Tools(m *Machine) []stdio.Tool {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if unit := text(args, "unit"); unit != "" {
|
||||
cg, err := m.cgroupOfUnit(unit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pa := m.unitProcs(cg)
|
||||
m.Wait(w)
|
||||
bc, err := parseCPU(m.read("/proc/stat"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{
|
||||
"window_s": w.Seconds(),
|
||||
"by": "process",
|
||||
"cgroup": cg,
|
||||
"cpu": machineCPUOver(ac, bc),
|
||||
"top": m.topProcessCPU(pa, m.unitProcs(cg), w, limit),
|
||||
}, nil
|
||||
}
|
||||
a := m.unitCPU()
|
||||
m.Wait(w)
|
||||
bc, err := parseCPU(m.read("/proc/stat"))
|
||||
|
||||
Reference in New Issue
Block a user