jschoubben
04902ff24a
The operator's desktop slows while it builds: the Go build runtime took 1650% of shanks' 24 processors with nothing holding it back. A build seat's holder now runs its runtime at idle priority under its node's cpu-cap share (declaration Yields, composed from Seat.Yields and the module's cpu-cap setting), and buildcpu/buildstep give each step its part of that cap in threads.
259 lines
9.1 KiB
Go
259 lines
9.1 KiB
Go
// Package buildcpu is how much of a build node's processors its builds may take, and how they yield to everything else
|
|
// there (novox/hq issue 1806; the operator, 2026-10-26: "limit CPU usage on our build nodes so the total cpu usage never
|
|
// impacts the normal workings of our nodes").
|
|
//
|
|
// Three parts, each with one owner:
|
|
//
|
|
// - **Priority.** A build seat's holder (catalogue.Seat.Yields) runs at the lowest CPU and IO priority there is: the
|
|
// composer gives its runtime process declaration.Yields, and the node-engine writes it into the unit as
|
|
// CPUWeight=idle, IOWeight=1, Nice=19 and the best-effort IO class at its lowest level. Interactive and service
|
|
// work always comes first; builds take the cycles nothing else wants.
|
|
// - **The cap.** The build modules' setting `cpu-cap` (Setting) gives each node the share of its processors its builds
|
|
// may take at most, as `<node>=<percent>` pairs. The composer turns this node's share into Yields.CPUShare, and the
|
|
// node-engine into CPUQuota for the machine's processor count: a hard bound, held by the kernel.
|
|
// - **Threads.** Inside that bound a step starts no more threads than its part of it (Threads): GOMAXPROCS, `go
|
|
// test -p`, the Node test runner's concurrency. A step with more runnable threads than its quota only waits in the
|
|
// kernel's throttle, and its timeouts with it.
|
|
package buildcpu
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"runtime"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// Setting is the build modules' setting of each node's processor cap: `<node>=<percent>` pairs separated by commas,
|
|
// such as `shanks=50,ace=75`; a percent sign after the number is allowed. Each build module declares it with Defaults
|
|
// as its default.
|
|
const Setting = "cpu-cap"
|
|
|
|
// Defaults is the setting's default: the operator's two desktops (shanks, g14) keep half their processors for
|
|
// themselves at all times, and ace, a server, builds on three quarters. A node it does not name has DefaultPercent.
|
|
const Defaults = "shanks=50,g14=50,ace=75"
|
|
|
|
// DefaultPercent is the cap of a node the setting does not name.
|
|
const DefaultPercent = 50
|
|
|
|
// Parse reads the setting's pairs. A percent outside 1 to 100 is refused: 0 would stop every build without pausing the
|
|
// seat (the capacity setting says that, with 0 asks), and more than 100 is no cap.
|
|
func Parse(s string) (map[string]int, error) {
|
|
out := map[string]int{}
|
|
for _, pair := range strings.Split(s, ",") {
|
|
pair = strings.TrimSpace(pair)
|
|
if pair == "" {
|
|
continue
|
|
}
|
|
node, value, ok := strings.Cut(pair, "=")
|
|
node = strings.TrimSpace(node)
|
|
value = strings.TrimSuffix(strings.TrimSpace(value), "%")
|
|
n, err := strconv.Atoi(strings.TrimSpace(value))
|
|
if !ok || node == "" || err != nil || n < 1 || n > 100 {
|
|
return nil, fmt.Errorf("the processor cap %q is not <node>=<percent of its processors, 1 to 100>", pair)
|
|
}
|
|
if _, dup := out[node]; dup {
|
|
return nil, fmt.Errorf("the processor cap names %s twice", node)
|
|
}
|
|
out[node] = n
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Of is node's cap in percent of its processors: DefaultPercent for a node the setting does not name.
|
|
func Of(setting, node string) (int, error) {
|
|
caps, err := Parse(setting)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if n, ok := caps[node]; ok {
|
|
return n, nil
|
|
}
|
|
return DefaultPercent, nil
|
|
}
|
|
|
|
// Format is a cap map in the setting's own words, sorted by node.
|
|
func Format(caps map[string]int) string {
|
|
nodes := make([]string, 0, len(caps))
|
|
for n := range caps {
|
|
nodes = append(nodes, n)
|
|
}
|
|
sort.Strings(nodes)
|
|
parts := make([]string, 0, len(nodes))
|
|
for _, n := range nodes {
|
|
parts = append(parts, fmt.Sprintf("%s=%d", n, caps[n]))
|
|
}
|
|
return strings.Join(parts, ",")
|
|
}
|
|
|
|
// Quota is the unit's CPUQuota for a share of a machine with cores processors: systemd counts a quota in percent of
|
|
// one processor, so half of 24 is 1200%. At least 100%, one processor, so a cap on a small machine never stops a step.
|
|
func Quota(percent, cores int) int {
|
|
q := percent * cores
|
|
if q < 100 {
|
|
q = 100
|
|
}
|
|
return q
|
|
}
|
|
|
|
// Threads is how many threads one step may run at once: its part of the cap when slots steps run beside it, at least
|
|
// one. Rounded down, so the steps together stay inside the quota.
|
|
func Threads(percent, cores, slots int) int {
|
|
if slots < 1 {
|
|
slots = 1
|
|
}
|
|
if percent < 1 || percent > 100 {
|
|
percent = 100
|
|
}
|
|
t := cores * percent / 100 / slots
|
|
if t < 1 {
|
|
t = 1
|
|
}
|
|
return t
|
|
}
|
|
|
|
// Limits is what one step is held to on this node, as a status tool says it.
|
|
type Limits struct {
|
|
// Percent is the node's cap in percent of its processors, and Setting the words it came from.
|
|
Percent int `json:"cap-percent"`
|
|
Setting string `json:"cap-setting"`
|
|
// Cores is the machine's processor count.
|
|
Cores int `json:"cores"`
|
|
// Slots is the node's build capacity: steps at once.
|
|
Slots int `json:"slots"`
|
|
// Threads is what each step is given (Threads).
|
|
Threads int `json:"threads-per-step"`
|
|
}
|
|
|
|
// LimitsFor is the limits of a step on node, from the cap setting and the node's capacity. A setting that cannot be read
|
|
// gives the default cap and says why in the error, so a step still runs with a bound rather than none.
|
|
func LimitsFor(setting, node string, slots int) (Limits, error) {
|
|
l := Limits{Setting: setting, Cores: runtime.NumCPU(), Slots: slots}
|
|
p, err := Of(setting, node)
|
|
if err != nil {
|
|
p = DefaultPercent
|
|
}
|
|
l.Percent = p
|
|
l.Threads = Threads(p, l.Cores, slots)
|
|
return l, err
|
|
}
|
|
|
|
// Env is the environment a step's commands are given so they start no more threads than they may: GOMAXPROCS for every
|
|
// Go program (the compiler, a test binary, esbuild, buildah), UV_THREADPOOL_SIZE for Node's worker pool. Returned as
|
|
// KEY=value words to append to a command's environment.
|
|
func Env(threads int) []string {
|
|
if threads < 1 {
|
|
threads = 1
|
|
}
|
|
t := strconv.Itoa(threads)
|
|
return []string{"GOMAXPROCS=" + t, "UV_THREADPOOL_SIZE=" + t}
|
|
}
|
|
|
|
// WithEnv is env with Env(threads) in place of any GOMAXPROCS or UV_THREADPOOL_SIZE it carried.
|
|
func WithEnv(env []string, threads int) []string {
|
|
out := make([]string, 0, len(env)+2)
|
|
for _, kv := range env {
|
|
if strings.HasPrefix(kv, "GOMAXPROCS=") || strings.HasPrefix(kv, "UV_THREADPOOL_SIZE=") {
|
|
continue
|
|
}
|
|
out = append(out, kv)
|
|
}
|
|
return append(out, Env(threads)...)
|
|
}
|
|
|
|
// InForce is what the kernel holds this process to, read rather than assumed (evidence over assertion): its nice
|
|
// level, its cgroup, the cgroup's CPU weight, idle mark and quota. A field the machine does not answer is left empty
|
|
// and Unread says which.
|
|
type InForce struct {
|
|
Nice int `json:"nice"`
|
|
Cgroup string `json:"cgroup,omitempty"`
|
|
CPUIdle string `json:"cpu-idle,omitempty"`
|
|
CPUWeight string `json:"cpu-weight,omitempty"`
|
|
// CPUMax is the cgroup's cpu.max: "<quota µs> <period µs>", or "max" with no quota.
|
|
CPUMax string `json:"cpu-max,omitempty"`
|
|
// Quota is CPUMax as a percent of one processor, 0 with none.
|
|
Quota int `json:"quota-percent,omitempty"`
|
|
IOWeight string `json:"io-weight,omitempty"`
|
|
Unread []string `json:"unread,omitempty"`
|
|
}
|
|
|
|
// cgroupRoot is where the unified hierarchy is mounted. A variable so a test can point it elsewhere.
|
|
var cgroupRoot = "/sys/fs/cgroup"
|
|
|
|
// procSelfCgroup is this process's cgroup file. A variable so a test can point it elsewhere.
|
|
var procSelfCgroup = "/proc/self/cgroup"
|
|
|
|
// ReadInForce reads what the kernel holds this process to.
|
|
func ReadInForce() InForce {
|
|
var f InForce
|
|
f.Nice = nice()
|
|
raw, err := os.ReadFile(procSelfCgroup)
|
|
if err != nil {
|
|
f.Unread = append(f.Unread, "cgroup: "+err.Error())
|
|
return f
|
|
}
|
|
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
|
|
if strings.HasPrefix(line, "0::") {
|
|
f.Cgroup = strings.TrimPrefix(line, "0::")
|
|
}
|
|
}
|
|
if f.Cgroup == "" {
|
|
f.Unread = append(f.Unread, "cgroup: not on the unified hierarchy")
|
|
return f
|
|
}
|
|
dir := cgroupRoot + f.Cgroup
|
|
read := func(name string) string {
|
|
b, err := os.ReadFile(dir + "/" + name)
|
|
if err != nil {
|
|
f.Unread = append(f.Unread, name+": "+err.Error())
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(string(b))
|
|
}
|
|
f.CPUIdle = read("cpu.idle")
|
|
f.CPUWeight = read("cpu.weight")
|
|
f.CPUMax = read("cpu.max")
|
|
f.IOWeight = read("io.weight")
|
|
f.Quota = quotaOf(f.CPUMax)
|
|
return f
|
|
}
|
|
|
|
// quotaOf is a cpu.max value as a percent of one processor: "1200000 100000" is 1200; "max ..." or nothing is 0.
|
|
func quotaOf(cpuMax string) int {
|
|
fields := strings.Fields(cpuMax)
|
|
if len(fields) != 2 || fields[0] == "max" {
|
|
return 0
|
|
}
|
|
q, err1 := strconv.Atoi(fields[0])
|
|
p, err2 := strconv.Atoi(fields[1])
|
|
if err1 != nil || err2 != nil || p <= 0 {
|
|
return 0
|
|
}
|
|
return q * 100 / p
|
|
}
|
|
|
|
// procSelfStat is this process's stat file. A variable so a test can point it elsewhere.
|
|
var procSelfStat = "/proc/self/stat"
|
|
|
|
// nice is this process's nice level from its stat file (the 19th field, counted after the command's closing
|
|
// parenthesis, which may itself hold spaces); 0 when it cannot be read.
|
|
func nice() int {
|
|
raw, err := os.ReadFile(procSelfStat)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
s := string(raw)
|
|
i := strings.LastIndex(s, ")")
|
|
if i < 0 {
|
|
return 0
|
|
}
|
|
fields := strings.Fields(s[i+1:])
|
|
// After the command: state is field 3, so nice (field 19) is index 16.
|
|
if len(fields) < 17 {
|
|
return 0
|
|
}
|
|
n, _ := strconv.Atoi(fields[16])
|
|
return n
|
|
}
|