diff --git a/go/buildcpu/buildcpu.go b/go/buildcpu/buildcpu.go new file mode 100644 index 0000000..dcd2f36 --- /dev/null +++ b/go/buildcpu/buildcpu.go @@ -0,0 +1,215 @@ +// Package buildcpu is how much of a build node's processors its builds may take, and how they yield to everything else +// there (novox/hq ADR 0553, issue 1806; the operator, 2026-10-26: "limit CPU usage on our build nodes so the total cpu +// usage never impacts the normal workings of our nodes ... We have 3 build agents so they should share the cpu power"). +// +// Three parts, each with one owner: +// +// - **Priority.** A build seat's holder (catalogue.Seat.Yields) runs at the lowest priority there is: the composer +// gives its runtime process declaration.Yields, and the node-engine places every yielding process of the node in +// one slice (declaration.YieldSlice) whose CPU weight is idle, with Nice=19 for every process it starts. +// Interactive and service work always comes first; builds take the cycles nothing else wants. +// - **One cap per node, shared.** The node's share of its processors is one value, the node-engine's own config +// `build-cpu-share` (ShareSetting), set per node through the node's settings layer. The node-engine writes it as the +// slice's CPUQuota: one bound the kernel holds for all the node's build modules together, never one per module. +// - **Threads.** The quota bounds the total; threads only size each step. A step gets the share divided by its own +// module's capacity (Threads), from the share the composer hands as a word (ShareWord): GOMAXPROCS sets every go +// command's -p, the compiler's concurrency, a test binary's processors and -parallel. A module given no word sets +// no thread limit at all: without the slice there is nothing to size a step to, and a limit then would only slow +// builds. +package buildcpu + +import ( + "os" + "runtime" + "strconv" + "strings" +) + +// ShareSetting is the node-engine's config value of the node's processor share for its builds, in percent of the +// machine's processors, 1 to 100: one value per node, set through the node's settings layer (novox/hq ADR 0553 §3). +const ShareSetting = "build-cpu-share" + +// DefaultPercent is the share where nothing sets one: half the machine, so a desktop keeps half its processors for +// itself whatever builds. +const DefaultPercent = 50 + +// ShareWord is the word the composer gives every yielding module (novox/hq ADR 0553 §4): the node's share, in percent. +const ShareWord = "MESH_BUILD_CPU_SHARE" + +// Quota is a CPUQuota for a share of a machine with cores processors: systemd counts a quota in percent of one +// processor, so half of 24 is 1200%. At least 100%, one processor, so a share on a small machine never stops a step. +func Quota(percent, cores int) int { + q := percent * cores + if q < 100 { + q = 100 + } + return q +} + +// Threads is how many threads one step may run at once: its part of the share when its own module runs slots steps at +// once, at least one. The slice's quota bounds the steps of all the node's build modules together; this only sizes one. +func Threads(percent, cores, slots int) int { + if slots < 1 { + slots = 1 + } + if percent < 1 || percent > 100 { + percent = DefaultPercent + } + t := cores * percent / 100 / slots + if t < 1 { + t = 1 + } + return t +} + +// Limits is what one step is held to on this node, as a status tool says it. +type Limits struct { + // Percent is the node's share in percent of its processors, and From where it was read: ShareWord, or "none" when + // the composer gave no share and a step runs without a thread limit. + Percent int `json:"share-percent,omitempty"` + From string `json:"share-from"` + // Cores is the machine's processor count. + Cores int `json:"cores"` + // Slots is this module's own capacity on the node: steps at once. + Slots int `json:"slots"` + // Threads is what each step is given (Threads); 0 is no limit. + Threads int `json:"threads-per-step"` +} + +// FromWords is the limits of a step from the share the composer gave this process (ShareWord) and the module's own +// capacity. Without a usable share there is no limit: Threads is 0. +func FromWords(getenv func(string) string, own int) Limits { + if own < 1 { + own = 1 + } + l := Limits{From: "none", Cores: runtime.NumCPU(), Slots: own} + if n, err := strconv.Atoi(strings.TrimSpace(getenv(ShareWord))); err == nil && n >= 1 && n <= 100 { + l.Percent, l.From = n, ShareWord + l.Threads = Threads(n, l.Cores, own) + } + return l +} + +// Env is the environment a step's commands are given so they start no more threads than they may: GOMAXPROCS for every +// Go program (the compiler, a test binary, esbuild, buildah), UV_THREADPOOL_SIZE for Node's worker pool. Returned as +// KEY=value words to append to a command's environment; nothing for 0 threads, no limit. +func Env(threads int) []string { + if threads < 1 { + return nil + } + t := strconv.Itoa(threads) + return []string{"GOMAXPROCS=" + t, "UV_THREADPOOL_SIZE=" + t} +} + +// InForce is what the kernel holds this process to, read rather than assumed (evidence over assertion): its nice +// level, its cgroup (the unit) and the cgroup's parent (the slice the node-engine placed it in), each one's CPU weight, +// idle mark and quota. A field the machine does not answer is left empty and Unread says which. +// +// The IO weight is read but binds only where the disk's scheduler honours it (bfq, or the io.cost controller); under +// the scheduler `none`, the usual one for NVMe, it does nothing, and the CPU bound is what holds. +type InForce struct { + Nice int `json:"nice"` + Cgroup string `json:"cgroup,omitempty"` + Unit Group `json:"unit"` + Slice Group `json:"slice"` + // Quota is the tighter of the two groups' cpu.max as a percent of one processor, 0 with none. + Quota int `json:"quota-percent,omitempty"` + Unread []string `json:"unread,omitempty"` +} + +// Group is one cgroup's controls. +type Group struct { + Path string `json:"path,omitempty"` + CPUIdle string `json:"cpu-idle,omitempty"` + CPUWeight string `json:"cpu-weight,omitempty"` + // CPUMax is the cgroup's cpu.max: " ", or "max" with no quota. + CPUMax string `json:"cpu-max,omitempty"` + IOWeight string `json:"io-weight,omitempty"` +} + +// cgroupRoot is where the unified hierarchy is mounted. A variable so a test can point it elsewhere. +var cgroupRoot = "/sys/fs/cgroup" + +// procSelfCgroup is this process's cgroup file. A variable so a test can point it elsewhere. +var procSelfCgroup = "/proc/self/cgroup" + +// procSelfStat is this process's stat file. A variable so a test can point it elsewhere. +var procSelfStat = "/proc/self/stat" + +// ReadInForce reads what the kernel holds this process to. +func ReadInForce() InForce { + var f InForce + f.Nice = nice() + raw, err := os.ReadFile(procSelfCgroup) + if err != nil { + f.Unread = append(f.Unread, "cgroup: "+err.Error()) + return f + } + for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") { + if strings.HasPrefix(line, "0::") { + f.Cgroup = strings.TrimPrefix(line, "0::") + } + } + if f.Cgroup == "" { + f.Unread = append(f.Unread, "cgroup: not on the unified hierarchy") + return f + } + read := func(path string) Group { + g := Group{Path: path} + value := func(name string) string { + b, err := os.ReadFile(cgroupRoot + path + "/" + name) + if err != nil { + f.Unread = append(f.Unread, path+"/"+name+": "+err.Error()) + return "" + } + return strings.TrimSpace(string(b)) + } + g.CPUIdle, g.CPUWeight, g.CPUMax, g.IOWeight = value("cpu.idle"), value("cpu.weight"), value("cpu.max"), value("io.weight") + return g + } + f.Unit = read(f.Cgroup) + if i := strings.LastIndex(f.Cgroup, "/"); i > 0 { + f.Slice = read(f.Cgroup[:i]) + } + for _, q := range []int{quotaOf(f.Unit.CPUMax), quotaOf(f.Slice.CPUMax)} { + if q > 0 && (f.Quota == 0 || q < f.Quota) { + f.Quota = q + } + } + return f +} + +// quotaOf is a cpu.max value as a percent of one processor: "1200000 100000" is 1200; "max ..." or nothing is 0. +func quotaOf(cpuMax string) int { + fields := strings.Fields(cpuMax) + if len(fields) != 2 || fields[0] == "max" { + return 0 + } + q, err1 := strconv.Atoi(fields[0]) + p, err2 := strconv.Atoi(fields[1]) + if err1 != nil || err2 != nil || p <= 0 { + return 0 + } + return q * 100 / p +} + +// nice is this process's nice level from its stat file (the 19th field, counted after the command's closing +// parenthesis, which may itself hold spaces); 0 when it cannot be read. +func nice() int { + raw, err := os.ReadFile(procSelfStat) + if err != nil { + return 0 + } + s := string(raw) + i := strings.LastIndex(s, ")") + if i < 0 { + return 0 + } + fields := strings.Fields(s[i+1:]) + // After the command: state is field 3, so nice (field 19) is index 16. + if len(fields) < 17 { + return 0 + } + n, _ := strconv.Atoi(fields[16]) + return n +} diff --git a/go/buildcpu/buildcpu_test.go b/go/buildcpu/buildcpu_test.go new file mode 100644 index 0000000..6740687 --- /dev/null +++ b/go/buildcpu/buildcpu_test.go @@ -0,0 +1,90 @@ +package buildcpu + +import ( + "os" + "path/filepath" + "reflect" + "runtime" + "testing" +) + +// The node's share as a quota, the threads one step is given for its own module's capacity, and the word the composer +// hands, in one table (ADR 0553, issue 1806). +func TestShareQuotaAndThreads(t *testing.T) { + for _, c := range []struct { + name string + percent, cores, slots int + quota, threads int + }{ + {"shanks: half of 24, three slots", 50, 24, 3, 1200, 4}, + {"ace: three quarters of 24, two slots", 75, 24, 2, 1800, 9}, + {"g14: half of 16, one slot", 50, 16, 1, 800, 8}, + {"one processor at least", 10, 2, 4, 100, 1}, + {"no slots counts as one", 100, 4, 0, 400, 4}, + {"a share out of range is the default", 0, 8, 1, 0, 4}, + } { + if c.quota != 0 { + if q := Quota(c.percent, c.cores); q != c.quota { + t.Errorf("%s: Quota = %d, want %d", c.name, q, c.quota) + } + } + if th := Threads(c.percent, c.cores, c.slots); th != c.threads { + t.Errorf("%s: Threads = %d, want %d", c.name, th, c.threads) + } + } + l := FromWords(func(k string) string { return map[string]string{ShareWord: "75"}[k] }, 2) + if l.Percent != 75 || l.From != ShareWord || l.Slots != 2 || l.Threads != Threads(75, runtime.NumCPU(), 2) { + t.Errorf("from the word: %+v", l) + } + // Told nothing usable, a step has no thread limit: nothing in its environment. + for _, word := range []string{"", "250", "lots"} { + l := FromWords(func(string) string { return word }, 3) + if l.Threads != 0 || l.From != "none" || Env(l.Threads) != nil { + t.Errorf("share %q: %+v, env %v", word, l, Env(l.Threads)) + } + } + if got := Env(3); !reflect.DeepEqual(got, []string{"GOMAXPROCS=3", "UV_THREADPOOL_SIZE=3"}) { + t.Errorf("Env = %q", got) + } +} + +// What the kernel holds a process to is read from its unit's and its slice's cgroup and its stat file, not assumed. +func TestReadInForce(t *testing.T) { + dir := t.TempDir() + cg := filepath.Join(dir, "cg") + slice := filepath.Join(cg, "meshyield.slice") + unit := filepath.Join(slice, "mesh-build-go-runtime.service") + if err := os.MkdirAll(unit, 0o755); err != nil { + t.Fatal(err) + } + for at, files := range map[string]map[string]string{ + slice: {"cpu.idle": "1\n", "cpu.weight": "0\n", "cpu.max": "1200000 100000\n", "io.weight": "default 1\n"}, + unit: {"cpu.idle": "0\n", "cpu.weight": "100\n", "cpu.max": "max 100000\n", "io.weight": "default 100\n"}, + } { + for name, v := range files { + if err := os.WriteFile(filepath.Join(at, name), []byte(v), 0o644); err != nil { + t.Fatal(err) + } + } + } + self := filepath.Join(dir, "cgroup") + stat := filepath.Join(dir, "stat") + _ = os.WriteFile(self, []byte("0::/meshyield.slice/mesh-build-go-runtime.service\n"), 0o644) + // A command with a space and a parenthesis in it, then nice 19 in field 19. + _ = os.WriteFile(stat, []byte("42 (mesh (build) go) S 1 42 42 0 -1 4194560 1 0 0 0 0 0 0 0 39 19 8 0 1 2 3\n"), 0o644) + cgroupRoot, procSelfCgroup, procSelfStat = cg, self, stat + t.Cleanup(func() { + cgroupRoot, procSelfCgroup, procSelfStat = "/sys/fs/cgroup", "/proc/self/cgroup", "/proc/self/stat" + }) + + got := ReadInForce() + want := InForce{Nice: 19, Cgroup: "/meshyield.slice/mesh-build-go-runtime.service", + Unit: Group{Path: "/meshyield.slice/mesh-build-go-runtime.service", CPUIdle: "0", CPUWeight: "100", + CPUMax: "max 100000", IOWeight: "default 100"}, + Slice: Group{Path: "/meshyield.slice", CPUIdle: "1", CPUWeight: "0", CPUMax: "1200000 100000", + IOWeight: "default 1"}, + Quota: 1200} + if !reflect.DeepEqual(got, want) { + t.Fatalf("ReadInForce = %+v\nwant %+v", got, want) + } +} diff --git a/go/buildstep/capacity_test.go b/go/buildstep/capacity_test.go index 5334226..a9d1a60 100644 --- a/go/buildstep/capacity_test.go +++ b/go/buildstep/capacity_test.go @@ -36,3 +36,15 @@ func TestNodeCapacity(t *testing.T) { t.Errorf("a missing file = %d, want 0", got) } } + +// ADR 0553 §4: a step's threads are the composer's share over the module's own capacity; without the word, none. +func TestNodeLimits(t *testing.T) { + t.Setenv("MESH_BUILD_CPU_SHARE", "") + if l := NodeLimits(3); l.Threads != 0 || l.Slots != 3 { + t.Errorf("without the word: %+v", l) + } + t.Setenv("MESH_BUILD_CPU_SHARE", "75") + if l := NodeLimits(3); l.Percent != 75 || l.Slots != 3 || l.Threads < 1 { + t.Errorf("from the word: %+v", l) + } +} diff --git a/go/buildstep/cpu.go b/go/buildstep/cpu.go new file mode 100644 index 0000000..39b5e3e --- /dev/null +++ b/go/buildstep/cpu.go @@ -0,0 +1,14 @@ +package buildstep + +import ( + "os" + + "git.novox.be/novox/mesh-sdk/go/buildcpu" +) + +// NodeLimits is what one step of a build module is held to (novox/hq ADR 0553 §4): the node's processor share from the +// word the composer gives the module (buildcpu.ShareWord), divided by the module's own capacity, own, over the machine's +// processors. Without the word, no thread limit: Threads is 0. +func NodeLimits(own int) buildcpu.Limits { + return buildcpu.FromWords(os.Getenv, own) +} diff --git a/go/catalogue/holdings_test.go b/go/catalogue/holdings_test.go index c8a15b5..dedc566 100644 --- a/go/catalogue/holdings_test.go +++ b/go/catalogue/holdings_test.go @@ -172,7 +172,7 @@ func TestAStoreRowWithoutAProtocolKeepsTheCompiledOne(t *testing.T) { t.Cleanup(func() { UseSeats(was) }) UseSeats([]Seat{{Name: "node-build-go", Scope: ScopeNode, Decision: "row"}}) got, ok := SeatNamed("node-build-go") - if !ok || len(got.Accepts) == 0 || !got.PulledByKind { + if !ok || len(got.Accepts) == 0 || !got.PulledByKind || !got.Yields { t.Fatalf("the Go build seat lost what it accepts when loaded from the store: %+v", got) } if got.Decision != "row" { diff --git a/go/catalogue/seats.go b/go/catalogue/seats.go index 7337d8a..805d0be 100644 --- a/go/catalogue/seats.go +++ b/go/catalogue/seats.go @@ -49,6 +49,11 @@ type Seat struct { // order Accepts lists them (novox/hq ADR 0525 §2, ADR 0536 §4): the build seats'. Compiled, never stored, like // Receives. PulledByKind bool + // Yields says the seat's holder gives way to everything else on its node (novox/hq ADR 0553): its runtime runs at + // idle priority in the node's one yielding slice, under the node's one processor share (buildcpu.ShareSetting) that + // every yielding holder of the node shares. The build seats': a build must never slow the operator's desktop. + // Compiled, never stored, like Receives. + Yields bool // Serves carries each verb in full — name, description, schema — because a role's tools are the // mesh's to define and an agent's to call (novox/hq ADR 0132, design 33 §2). Serves []Verb @@ -256,11 +261,11 @@ var defaultSeats = append([]Seat{ // its holder taking nothing, kept on the module's own pause key (issue 1887's design limit 2). They replace // node-build-agent, retired with build-agent (ADR 0524 §5). {Name: "node-build-go", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("Go"), - Decision: "novox/hq ADR 0525, ADR 0536"}, + Yields: true, Decision: "novox/hq ADR 0525, ADR 0536, ADR 0553"}, {Name: "node-build-node", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("Node"), - Decision: "novox/hq ADR 0525, ADR 0536"}, + Yields: true, Decision: "novox/hq ADR 0525, ADR 0536, ADR 0553"}, {Name: "node-build-image", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("image"), - Decision: "novox/hq ADR 0525, ADR 0536"}, + Yields: true, Decision: "novox/hq ADR 0525, ADR 0536, ADR 0553"}, // **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and // every node and container asks them and nothing else. Replicated since ADR 0223: held on more // than one machine, each answering the same names from the same roster, and every machine's @@ -497,6 +502,7 @@ func UseSeats(s []Seat) { row.Paths = d.Paths row.Replicated = d.Replicated row.PulledByKind = d.PulledByKind + row.Yields = d.Yields // never stored (ADR 0553) row.Serves = optionalAsCompiled(row.Serves, d.Serves) } merged = append(merged, row) diff --git a/go/catalogue/seats_test.go b/go/catalogue/seats_test.go index caa5a25..62043e1 100644 --- a/go/catalogue/seats_test.go +++ b/go/catalogue/seats_test.go @@ -324,6 +324,9 @@ func TestNodeBuildAgentIsRetiredForTheBuildSeats(t *testing.T) { if !ok || s.Scope != ScopeNode || !s.PulledByKind || strings.Join(s.Accepts, ",") != "delivery,check,recheck" { t.Fatalf("%s = %+v", name, s) } + if !s.Yields { + t.Errorf("%s does not yield (ADR 0553)", name) + } var verbs []string for _, v := range s.Serves { verbs = append(verbs, v.Name) @@ -332,4 +335,14 @@ func TestNodeBuildAgentIsRetiredForTheBuildSeats(t *testing.T) { t.Errorf("%s serves %v", name, verbs) } } + // The build seats are the seats that yield, and no other (ADR 0553 §1). + var yielding []string + for _, s := range Seats() { + if s.Yields { + yielding = append(yielding, s.Name) + } + } + if strings.Join(yielding, ",") != "node-build-go,node-build-node,node-build-image" { + t.Errorf("the seats that yield are %v", yielding) + } } diff --git a/go/declaration/declaration.go b/go/declaration/declaration.go index cec208d..3d70e00 100644 --- a/go/declaration/declaration.go +++ b/go/declaration/declaration.go @@ -916,6 +916,56 @@ type Process struct { // Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its // timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more. Health *Health `json:"health,omitempty"` + + // Yields says this process gives way to everything else on its machine (novox/hq ADR 0553, issue 1806): the + // runtime of a build seat's holder, whose builds must never slow the operator's desktop. Absent: the machine's + // defaults. For a process that stays up. + Yields *Yields `json:"yields,omitempty"` +} + +// YieldIdle is the one priority a process yields at: the lowest there is. +const YieldIdle = "idle" + +// YieldSlice is the one slice the node-engine places every yielding process of its node in (novox/hq ADR 0553 §2): a +// slice at the top of the hierarchy, beside the operator's user slice and the system's, so its idle weight and its +// quota bind against everything else on the machine. No dash in its name: a dash makes a slice a child of the slice +// named before it, where its weight would count only against its siblings. +const YieldSlice = "meshyield.slice" + +// Yields is how a process gives way, written by the node-engine (novox/hq ADR 0553). +// +// **Priority** "idle": the process runs in YieldSlice, whose CPUWeight is idle (it runs only on cycles nothing else +// wants) and whose IOWeight is 1, with Nice=19 and the lowest best-effort IO level for every process it starts. The IO +// controls bind only where the disk's scheduler honours them (bfq, or io.cost for the weight); under `none`, the +// usual one for NVMe, they do nothing, and the CPU bound is what holds. **CPUShare** is the most of the machine's +// processors the slice may take, in percent, shared by every yielding process of the node together: the node-engine +// writes it as the slice's CPUQuota for the processors the machine has, so the bound is the kernel's. Every yielding +// process of one node carries the same share: the composer gives the node's one value (ADR 0553 §3). 0 is no cap. +type Yields struct { + Priority string `json:"priority,omitempty"` + CPUShare int `json:"cpu-share,omitempty"` +} + +func (y *Yields) problems(where string, staysUp bool) []string { + if y == nil { + return nil + } + var problems []string + if !staysUp { + problems = append(problems, where+": yields is for a process that stays up") + } + switch y.Priority { + case "", YieldIdle: + default: + problems = append(problems, fmt.Sprintf("%s: yields priority %q is not one this host knows: idle", where, y.Priority)) + } + if y.CPUShare < 0 || y.CPUShare > 100 { + problems = append(problems, fmt.Sprintf("%s: yields cpu-share %d is not a percent of the machine, 0 to 100", where, y.CPUShare)) + } + if y.Priority == "" && y.CPUShare == 0 { + problems = append(problems, where+": yields says neither a priority nor a cpu-share") + } + return problems } func (d *Process) Identity() string { return d.ID } @@ -948,6 +998,7 @@ func ProcessNameProblem(name string) string { func (d *Process) validate(where string, _ bool) []string { problems := d.Health.problems(where, false, !d.RunOnce && d.Schedule == "") + problems = append(problems, d.Yields.problems(where, !d.RunOnce && d.Schedule == "")...) if problem := ProcessNameProblem(d.Name); problem != "" { problems = append(problems, where+": "+problem) } diff --git a/go/declaration/process_test.go b/go/declaration/process_test.go index 9a7c7e9..02eaff1 100644 --- a/go/declaration/process_test.go +++ b/go/declaration/process_test.go @@ -92,6 +92,20 @@ func TestTheModesAreExclusive(t *testing.T) { if problems := watching.validate("a process", false); len(problems) == 0 { t.Fatal("a scheduled process was given something to restart on, and it is never running") } + + // A process yields only while it stays up, at a priority the host knows, within the machine (issue 1806). + for name, bad := range map[string]func(*Process){ + "a step that yields": func(p *Process) { p.RunOnce = true; p.Yields = &Yields{Priority: YieldIdle} }, + "an unknown priority": func(p *Process) { p.Yields = &Yields{Priority: "low"} }, + "a share over the whole": func(p *Process) { p.Yields = &Yields{CPUShare: 150} }, + "a yield saying nothing": func(p *Process) { p.Yields = &Yields{} }, + } { + p := aProcess() + bad(p) + if problems := p.validate("a process", false); len(problems) == 0 { + t.Errorf("%s was accepted", name) + } + } } // A cadence that is not a cadence is refused near its author, rather than by a machine at the far @@ -118,6 +132,11 @@ func TestEachModeOnItsOwnIsAccepted(t *testing.T) { if problems := every.validate("a process", false); len(problems) != 0 { t.Fatalf("a scheduled process was refused: %v", problems) } + yielding := aProcess() + yielding.Yields = &Yields{Priority: YieldIdle, CPUShare: 50} + if problems := yielding.validate("a process", false); len(problems) != 0 { + t.Fatalf("a daemon yielding at idle under half the machine was refused: %v", problems) + } } // **The one that cannot be escaped, only refused.**