jschoubben
852ba81c5a
Review round 1: the decoder type ships before any composer can emit it, so a routine SDK bump never sends yields to an engine that refuses them; the composer half moves to a later change. Every yielding process of a node goes into one top-level slice, meshyield.slice, so the three build modules share one quota instead of taking one each; the share is one node value, node-tools' build-cpu-share, and each step's threads divide it by every build slot of the node (words from the composer, a conservative fallback without them).
229 lines
9.0 KiB
Go
229 lines
9.0 KiB
Go
// Package buildcpu is how much of a build node's processors its builds may take, and how they yield to everything else
|
||
// there (novox/hq ADR 0553, issue 1806; the operator, 2026-10-26: "limit CPU usage on our build nodes so the total cpu
|
||
// usage never impacts the normal workings of our nodes ... We have 3 build agents so they should share the cpu power").
|
||
//
|
||
// Three parts, each with one owner:
|
||
//
|
||
// - **Priority.** A build seat's holder (catalogue.Seat.Yields) runs at the lowest priority there is: the composer
|
||
// gives its runtime process declaration.Yields, and the node-engine places every yielding process of the node in
|
||
// one slice (declaration.YieldSlice) whose CPU weight is idle, with Nice=19 for every process it starts.
|
||
// Interactive and service work always comes first; builds take the cycles nothing else wants.
|
||
// - **One cap per node, shared.** The node's share of its processors is one value, the node-tools runtime's setting
|
||
// `build-cpu-share` (ShareSetting), set per node through the node's settings layer. The node-engine writes it as the
|
||
// slice's CPUQuota: one bound the kernel holds for all the node's build modules together, never one per module.
|
||
// - **Threads.** Inside that bound a step starts no more threads than its part of it (Threads): the node's share
|
||
// divided by every build slot of the node, across all its build modules (the composer hands both as words,
|
||
// ShareWord and SlotsWord). GOMAXPROCS sets every go command's -p, the compiler's concurrency, a test binary's
|
||
// processors and -parallel.
|
||
package buildcpu
|
||
|
||
import (
|
||
"os"
|
||
"runtime"
|
||
"strconv"
|
||
"strings"
|
||
)
|
||
|
||
// ShareSetting is the node-tools runtime's setting of the node's processor share for its builds, in percent of the
|
||
// machine's processors, 1 to 100: one value per node, set through the node's settings layer (novox/hq ADR 0553 §3).
|
||
const ShareSetting = "build-cpu-share"
|
||
|
||
// DefaultPercent is the share where nothing sets one: half the machine, so a desktop keeps half its processors for
|
||
// itself whatever builds.
|
||
const DefaultPercent = 50
|
||
|
||
// BuildSeats is how many seats yield (catalogue.Seat.Yields): one build module per language. A step that is told no
|
||
// node-wide slot count assumes every one of them runs its full capacity beside it.
|
||
const BuildSeats = 3
|
||
|
||
// The words the composer gives every yielding module (novox/hq ADR 0553 §4): the node's share, and every build slot
|
||
// of the node across all its yielding modules.
|
||
const (
|
||
ShareWord = "MESH_BUILD_CPU_SHARE"
|
||
SlotsWord = "MESH_BUILD_SLOTS"
|
||
)
|
||
|
||
// Quota is a CPUQuota for a share of a machine with cores processors: systemd counts a quota in percent of one
|
||
// processor, so half of 24 is 1200%. At least 100%, one processor, so a share on a small machine never stops a step.
|
||
func Quota(percent, cores int) int {
|
||
q := percent * cores
|
||
if q < 100 {
|
||
q = 100
|
||
}
|
||
return q
|
||
}
|
||
|
||
// Threads is how many threads one step may run at once: its part of the share when slots steps may run on the node
|
||
// at once, at least one. Rounded down, so the steps together stay inside the quota.
|
||
func Threads(percent, cores, slots int) int {
|
||
if slots < 1 {
|
||
slots = 1
|
||
}
|
||
if percent < 1 || percent > 100 {
|
||
percent = DefaultPercent
|
||
}
|
||
t := cores * percent / 100 / slots
|
||
if t < 1 {
|
||
t = 1
|
||
}
|
||
return t
|
||
}
|
||
|
||
// Limits is what one step is held to on this node, as a status tool says it.
|
||
type Limits struct {
|
||
// Percent is the node's share in percent of its processors; From says where it was read: the composer's word,
|
||
// or the default.
|
||
Percent int `json:"share-percent"`
|
||
From string `json:"share-from"`
|
||
// Cores is the machine's processor count.
|
||
Cores int `json:"cores"`
|
||
// Slots is every build slot of the node, across all its build modules; SlotsFrom where it was read.
|
||
Slots int `json:"node-slots"`
|
||
SlotsFrom string `json:"slots-from"`
|
||
// Threads is what each step is given (Threads).
|
||
Threads int `json:"threads-per-step"`
|
||
}
|
||
|
||
// FromWords is the limits of a step from the words the composer gave this process (ShareWord, SlotsWord). Where a word
|
||
// is absent or not a number: the default share, and own × BuildSeats slots. A module told nothing assumes every build
|
||
// module of the node runs its full capacity beside it, so the steps together never take more than the default share.
|
||
func FromWords(getenv func(string) string, own int) Limits {
|
||
if own < 1 {
|
||
own = 1
|
||
}
|
||
l := Limits{Percent: DefaultPercent, From: "default", Cores: runtime.NumCPU(), Slots: own * BuildSeats,
|
||
SlotsFrom: "own capacity × " + strconv.Itoa(BuildSeats) + " build seats"}
|
||
if n, err := strconv.Atoi(strings.TrimSpace(getenv(ShareWord))); err == nil && n >= 1 && n <= 100 {
|
||
l.Percent, l.From = n, ShareWord
|
||
}
|
||
if n, err := strconv.Atoi(strings.TrimSpace(getenv(SlotsWord))); err == nil && n >= 1 {
|
||
l.Slots, l.SlotsFrom = n, SlotsWord
|
||
}
|
||
l.Threads = Threads(l.Percent, l.Cores, l.Slots)
|
||
return l
|
||
}
|
||
|
||
// Env is the environment a step's commands are given so they start no more threads than they may: GOMAXPROCS for every
|
||
// Go program (the compiler, a test binary, esbuild, buildah), UV_THREADPOOL_SIZE for Node's worker pool. Returned as
|
||
// KEY=value words to append to a command's environment.
|
||
func Env(threads int) []string {
|
||
if threads < 1 {
|
||
threads = 1
|
||
}
|
||
t := strconv.Itoa(threads)
|
||
return []string{"GOMAXPROCS=" + t, "UV_THREADPOOL_SIZE=" + t}
|
||
}
|
||
|
||
// InForce is what the kernel holds this process to, read rather than assumed (evidence over assertion): its nice
|
||
// level, its cgroup (the unit) and the cgroup's parent (the slice the node-engine placed it in), each one's CPU weight,
|
||
// idle mark and quota. A field the machine does not answer is left empty and Unread says which.
|
||
//
|
||
// The IO weight is read but binds only where the disk's scheduler honours it (bfq, or the io.cost controller); under
|
||
// the scheduler `none`, the usual one for NVMe, it does nothing, and the CPU bound is what holds.
|
||
type InForce struct {
|
||
Nice int `json:"nice"`
|
||
Cgroup string `json:"cgroup,omitempty"`
|
||
Unit Group `json:"unit"`
|
||
Slice Group `json:"slice"`
|
||
// Quota is the tighter of the two groups' cpu.max as a percent of one processor, 0 with none.
|
||
Quota int `json:"quota-percent,omitempty"`
|
||
Unread []string `json:"unread,omitempty"`
|
||
}
|
||
|
||
// Group is one cgroup's controls.
|
||
type Group struct {
|
||
Path string `json:"path,omitempty"`
|
||
CPUIdle string `json:"cpu-idle,omitempty"`
|
||
CPUWeight string `json:"cpu-weight,omitempty"`
|
||
// CPUMax is the cgroup's cpu.max: "<quota µs> <period µs>", or "max" with no quota.
|
||
CPUMax string `json:"cpu-max,omitempty"`
|
||
IOWeight string `json:"io-weight,omitempty"`
|
||
}
|
||
|
||
// cgroupRoot is where the unified hierarchy is mounted. A variable so a test can point it elsewhere.
|
||
var cgroupRoot = "/sys/fs/cgroup"
|
||
|
||
// procSelfCgroup is this process's cgroup file. A variable so a test can point it elsewhere.
|
||
var procSelfCgroup = "/proc/self/cgroup"
|
||
|
||
// procSelfStat is this process's stat file. A variable so a test can point it elsewhere.
|
||
var procSelfStat = "/proc/self/stat"
|
||
|
||
// ReadInForce reads what the kernel holds this process to.
|
||
func ReadInForce() InForce {
|
||
var f InForce
|
||
f.Nice = nice()
|
||
raw, err := os.ReadFile(procSelfCgroup)
|
||
if err != nil {
|
||
f.Unread = append(f.Unread, "cgroup: "+err.Error())
|
||
return f
|
||
}
|
||
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
|
||
if strings.HasPrefix(line, "0::") {
|
||
f.Cgroup = strings.TrimPrefix(line, "0::")
|
||
}
|
||
}
|
||
if f.Cgroup == "" {
|
||
f.Unread = append(f.Unread, "cgroup: not on the unified hierarchy")
|
||
return f
|
||
}
|
||
read := func(path string) Group {
|
||
g := Group{Path: path}
|
||
value := func(name string) string {
|
||
b, err := os.ReadFile(cgroupRoot + path + "/" + name)
|
||
if err != nil {
|
||
f.Unread = append(f.Unread, path+"/"+name+": "+err.Error())
|
||
return ""
|
||
}
|
||
return strings.TrimSpace(string(b))
|
||
}
|
||
g.CPUIdle, g.CPUWeight, g.CPUMax, g.IOWeight = value("cpu.idle"), value("cpu.weight"), value("cpu.max"), value("io.weight")
|
||
return g
|
||
}
|
||
f.Unit = read(f.Cgroup)
|
||
if i := strings.LastIndex(f.Cgroup, "/"); i > 0 {
|
||
f.Slice = read(f.Cgroup[:i])
|
||
}
|
||
for _, q := range []int{quotaOf(f.Unit.CPUMax), quotaOf(f.Slice.CPUMax)} {
|
||
if q > 0 && (f.Quota == 0 || q < f.Quota) {
|
||
f.Quota = q
|
||
}
|
||
}
|
||
return f
|
||
}
|
||
|
||
// quotaOf is a cpu.max value as a percent of one processor: "1200000 100000" is 1200; "max ..." or nothing is 0.
|
||
func quotaOf(cpuMax string) int {
|
||
fields := strings.Fields(cpuMax)
|
||
if len(fields) != 2 || fields[0] == "max" {
|
||
return 0
|
||
}
|
||
q, err1 := strconv.Atoi(fields[0])
|
||
p, err2 := strconv.Atoi(fields[1])
|
||
if err1 != nil || err2 != nil || p <= 0 {
|
||
return 0
|
||
}
|
||
return q * 100 / p
|
||
}
|
||
|
||
// nice is this process's nice level from its stat file (the 19th field, counted after the command's closing
|
||
// parenthesis, which may itself hold spaces); 0 when it cannot be read.
|
||
func nice() int {
|
||
raw, err := os.ReadFile(procSelfStat)
|
||
if err != nil {
|
||
return 0
|
||
}
|
||
s := string(raw)
|
||
i := strings.LastIndex(s, ")")
|
||
if i < 0 {
|
||
return 0
|
||
}
|
||
fields := strings.Fields(s[i+1:])
|
||
// After the command: state is field 3, so nice (field 19) is index 16.
|
||
if len(fields) < 17 {
|
||
return 0
|
||
}
|
||
n, _ := strconv.Atoi(fields[16])
|
||
return n
|
||
}
|