Merge remote-tracking branch 'origin/main' into fix/1790-state-appears
This commit is contained in:
@@ -0,0 +1,215 @@
|
|||||||
|
// Package buildcpu is how much of a build node's processors its builds may take, and how they yield to everything else
|
||||||
|
// there (novox/hq ADR 0553, issue 1806; the operator, 2026-10-26: "limit CPU usage on our build nodes so the total cpu
|
||||||
|
// usage never impacts the normal workings of our nodes ... We have 3 build agents so they should share the cpu power").
|
||||||
|
//
|
||||||
|
// Three parts, each with one owner:
|
||||||
|
//
|
||||||
|
// - **Priority.** A build seat's holder (catalogue.Seat.Yields) runs at the lowest priority there is: the composer
|
||||||
|
// gives its runtime process declaration.Yields, and the node-engine places every yielding process of the node in
|
||||||
|
// one slice (declaration.YieldSlice) whose CPU weight is idle, with Nice=19 for every process it starts.
|
||||||
|
// Interactive and service work always comes first; builds take the cycles nothing else wants.
|
||||||
|
// - **One cap per node, shared.** The node's share of its processors is one value, the node-engine's own config
|
||||||
|
// `build-cpu-share` (ShareSetting), set per node through the node's settings layer. The node-engine writes it as the
|
||||||
|
// slice's CPUQuota: one bound the kernel holds for all the node's build modules together, never one per module.
|
||||||
|
// - **Threads.** The quota bounds the total; threads only size each step. A step gets the share divided by its own
|
||||||
|
// module's capacity (Threads), from the share the composer hands as a word (ShareWord): GOMAXPROCS sets every go
|
||||||
|
// command's -p, the compiler's concurrency, a test binary's processors and -parallel. A module given no word sets
|
||||||
|
// no thread limit at all: without the slice there is nothing to size a step to, and a limit then would only slow
|
||||||
|
// builds.
|
||||||
|
package buildcpu
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ShareSetting is the node-engine's config value of the node's processor share for its builds, in percent of the
|
||||||
|
// machine's processors, 1 to 100: one value per node, set through the node's settings layer (novox/hq ADR 0553 §3).
|
||||||
|
const ShareSetting = "build-cpu-share"
|
||||||
|
|
||||||
|
// DefaultPercent is the share where nothing sets one: half the machine, so a desktop keeps half its processors for
|
||||||
|
// itself whatever builds.
|
||||||
|
const DefaultPercent = 50
|
||||||
|
|
||||||
|
// ShareWord is the word the composer gives every yielding module (novox/hq ADR 0553 §4): the node's share, in percent.
|
||||||
|
const ShareWord = "MESH_BUILD_CPU_SHARE"
|
||||||
|
|
||||||
|
// Quota is a CPUQuota for a share of a machine with cores processors: systemd counts a quota in percent of one
|
||||||
|
// processor, so half of 24 is 1200%. At least 100%, one processor, so a share on a small machine never stops a step.
|
||||||
|
func Quota(percent, cores int) int {
|
||||||
|
q := percent * cores
|
||||||
|
if q < 100 {
|
||||||
|
q = 100
|
||||||
|
}
|
||||||
|
return q
|
||||||
|
}
|
||||||
|
|
||||||
|
// Threads is how many threads one step may run at once: its part of the share when its own module runs slots steps at
|
||||||
|
// once, at least one. The slice's quota bounds the steps of all the node's build modules together; this only sizes one.
|
||||||
|
func Threads(percent, cores, slots int) int {
|
||||||
|
if slots < 1 {
|
||||||
|
slots = 1
|
||||||
|
}
|
||||||
|
if percent < 1 || percent > 100 {
|
||||||
|
percent = DefaultPercent
|
||||||
|
}
|
||||||
|
t := cores * percent / 100 / slots
|
||||||
|
if t < 1 {
|
||||||
|
t = 1
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
// Limits is what one step is held to on this node, as a status tool says it.
|
||||||
|
type Limits struct {
|
||||||
|
// Percent is the node's share in percent of its processors, and From where it was read: ShareWord, or "none" when
|
||||||
|
// the composer gave no share and a step runs without a thread limit.
|
||||||
|
Percent int `json:"share-percent,omitempty"`
|
||||||
|
From string `json:"share-from"`
|
||||||
|
// Cores is the machine's processor count.
|
||||||
|
Cores int `json:"cores"`
|
||||||
|
// Slots is this module's own capacity on the node: steps at once.
|
||||||
|
Slots int `json:"slots"`
|
||||||
|
// Threads is what each step is given (Threads); 0 is no limit.
|
||||||
|
Threads int `json:"threads-per-step"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// FromWords is the limits of a step from the share the composer gave this process (ShareWord) and the module's own
|
||||||
|
// capacity. Without a usable share there is no limit: Threads is 0.
|
||||||
|
func FromWords(getenv func(string) string, own int) Limits {
|
||||||
|
if own < 1 {
|
||||||
|
own = 1
|
||||||
|
}
|
||||||
|
l := Limits{From: "none", Cores: runtime.NumCPU(), Slots: own}
|
||||||
|
if n, err := strconv.Atoi(strings.TrimSpace(getenv(ShareWord))); err == nil && n >= 1 && n <= 100 {
|
||||||
|
l.Percent, l.From = n, ShareWord
|
||||||
|
l.Threads = Threads(n, l.Cores, own)
|
||||||
|
}
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
|
||||||
|
// Env is the environment a step's commands are given so they start no more threads than they may: GOMAXPROCS for every
|
||||||
|
// Go program (the compiler, a test binary, esbuild, buildah), UV_THREADPOOL_SIZE for Node's worker pool. Returned as
|
||||||
|
// KEY=value words to append to a command's environment; nothing for 0 threads, no limit.
|
||||||
|
func Env(threads int) []string {
|
||||||
|
if threads < 1 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
t := strconv.Itoa(threads)
|
||||||
|
return []string{"GOMAXPROCS=" + t, "UV_THREADPOOL_SIZE=" + t}
|
||||||
|
}
|
||||||
|
|
||||||
|
// InForce is what the kernel holds this process to, read rather than assumed (evidence over assertion): its nice
|
||||||
|
// level, its cgroup (the unit) and the cgroup's parent (the slice the node-engine placed it in), each one's CPU weight,
|
||||||
|
// idle mark and quota. A field the machine does not answer is left empty and Unread says which.
|
||||||
|
//
|
||||||
|
// The IO weight is read but binds only where the disk's scheduler honours it (bfq, or the io.cost controller); under
|
||||||
|
// the scheduler `none`, the usual one for NVMe, it does nothing, and the CPU bound is what holds.
|
||||||
|
type InForce struct {
|
||||||
|
Nice int `json:"nice"`
|
||||||
|
Cgroup string `json:"cgroup,omitempty"`
|
||||||
|
Unit Group `json:"unit"`
|
||||||
|
Slice Group `json:"slice"`
|
||||||
|
// Quota is the tighter of the two groups' cpu.max as a percent of one processor, 0 with none.
|
||||||
|
Quota int `json:"quota-percent,omitempty"`
|
||||||
|
Unread []string `json:"unread,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Group is one cgroup's controls.
|
||||||
|
type Group struct {
|
||||||
|
Path string `json:"path,omitempty"`
|
||||||
|
CPUIdle string `json:"cpu-idle,omitempty"`
|
||||||
|
CPUWeight string `json:"cpu-weight,omitempty"`
|
||||||
|
// CPUMax is the cgroup's cpu.max: "<quota µs> <period µs>", or "max" with no quota.
|
||||||
|
CPUMax string `json:"cpu-max,omitempty"`
|
||||||
|
IOWeight string `json:"io-weight,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// cgroupRoot is where the unified hierarchy is mounted. A variable so a test can point it elsewhere.
|
||||||
|
var cgroupRoot = "/sys/fs/cgroup"
|
||||||
|
|
||||||
|
// procSelfCgroup is this process's cgroup file. A variable so a test can point it elsewhere.
|
||||||
|
var procSelfCgroup = "/proc/self/cgroup"
|
||||||
|
|
||||||
|
// procSelfStat is this process's stat file. A variable so a test can point it elsewhere.
|
||||||
|
var procSelfStat = "/proc/self/stat"
|
||||||
|
|
||||||
|
// ReadInForce reads what the kernel holds this process to.
|
||||||
|
func ReadInForce() InForce {
|
||||||
|
var f InForce
|
||||||
|
f.Nice = nice()
|
||||||
|
raw, err := os.ReadFile(procSelfCgroup)
|
||||||
|
if err != nil {
|
||||||
|
f.Unread = append(f.Unread, "cgroup: "+err.Error())
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
|
||||||
|
if strings.HasPrefix(line, "0::") {
|
||||||
|
f.Cgroup = strings.TrimPrefix(line, "0::")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if f.Cgroup == "" {
|
||||||
|
f.Unread = append(f.Unread, "cgroup: not on the unified hierarchy")
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
read := func(path string) Group {
|
||||||
|
g := Group{Path: path}
|
||||||
|
value := func(name string) string {
|
||||||
|
b, err := os.ReadFile(cgroupRoot + path + "/" + name)
|
||||||
|
if err != nil {
|
||||||
|
f.Unread = append(f.Unread, path+"/"+name+": "+err.Error())
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(b))
|
||||||
|
}
|
||||||
|
g.CPUIdle, g.CPUWeight, g.CPUMax, g.IOWeight = value("cpu.idle"), value("cpu.weight"), value("cpu.max"), value("io.weight")
|
||||||
|
return g
|
||||||
|
}
|
||||||
|
f.Unit = read(f.Cgroup)
|
||||||
|
if i := strings.LastIndex(f.Cgroup, "/"); i > 0 {
|
||||||
|
f.Slice = read(f.Cgroup[:i])
|
||||||
|
}
|
||||||
|
for _, q := range []int{quotaOf(f.Unit.CPUMax), quotaOf(f.Slice.CPUMax)} {
|
||||||
|
if q > 0 && (f.Quota == 0 || q < f.Quota) {
|
||||||
|
f.Quota = q
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
// quotaOf is a cpu.max value as a percent of one processor: "1200000 100000" is 1200; "max ..." or nothing is 0.
|
||||||
|
func quotaOf(cpuMax string) int {
|
||||||
|
fields := strings.Fields(cpuMax)
|
||||||
|
if len(fields) != 2 || fields[0] == "max" {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
q, err1 := strconv.Atoi(fields[0])
|
||||||
|
p, err2 := strconv.Atoi(fields[1])
|
||||||
|
if err1 != nil || err2 != nil || p <= 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return q * 100 / p
|
||||||
|
}
|
||||||
|
|
||||||
|
// nice is this process's nice level from its stat file (the 19th field, counted after the command's closing
|
||||||
|
// parenthesis, which may itself hold spaces); 0 when it cannot be read.
|
||||||
|
func nice() int {
|
||||||
|
raw, err := os.ReadFile(procSelfStat)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
s := string(raw)
|
||||||
|
i := strings.LastIndex(s, ")")
|
||||||
|
if i < 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
fields := strings.Fields(s[i+1:])
|
||||||
|
// After the command: state is field 3, so nice (field 19) is index 16.
|
||||||
|
if len(fields) < 17 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
n, _ := strconv.Atoi(fields[16])
|
||||||
|
return n
|
||||||
|
}
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package buildcpu
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The node's share as a quota, the threads one step is given for its own module's capacity, and the word the composer
|
||||||
|
// hands, in one table (ADR 0553, issue 1806).
|
||||||
|
func TestShareQuotaAndThreads(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
percent, cores, slots int
|
||||||
|
quota, threads int
|
||||||
|
}{
|
||||||
|
{"shanks: half of 24, three slots", 50, 24, 3, 1200, 4},
|
||||||
|
{"ace: three quarters of 24, two slots", 75, 24, 2, 1800, 9},
|
||||||
|
{"g14: half of 16, one slot", 50, 16, 1, 800, 8},
|
||||||
|
{"one processor at least", 10, 2, 4, 100, 1},
|
||||||
|
{"no slots counts as one", 100, 4, 0, 400, 4},
|
||||||
|
{"a share out of range is the default", 0, 8, 1, 0, 4},
|
||||||
|
} {
|
||||||
|
if c.quota != 0 {
|
||||||
|
if q := Quota(c.percent, c.cores); q != c.quota {
|
||||||
|
t.Errorf("%s: Quota = %d, want %d", c.name, q, c.quota)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if th := Threads(c.percent, c.cores, c.slots); th != c.threads {
|
||||||
|
t.Errorf("%s: Threads = %d, want %d", c.name, th, c.threads)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
l := FromWords(func(k string) string { return map[string]string{ShareWord: "75"}[k] }, 2)
|
||||||
|
if l.Percent != 75 || l.From != ShareWord || l.Slots != 2 || l.Threads != Threads(75, runtime.NumCPU(), 2) {
|
||||||
|
t.Errorf("from the word: %+v", l)
|
||||||
|
}
|
||||||
|
// Told nothing usable, a step has no thread limit: nothing in its environment.
|
||||||
|
for _, word := range []string{"", "250", "lots"} {
|
||||||
|
l := FromWords(func(string) string { return word }, 3)
|
||||||
|
if l.Threads != 0 || l.From != "none" || Env(l.Threads) != nil {
|
||||||
|
t.Errorf("share %q: %+v, env %v", word, l, Env(l.Threads))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := Env(3); !reflect.DeepEqual(got, []string{"GOMAXPROCS=3", "UV_THREADPOOL_SIZE=3"}) {
|
||||||
|
t.Errorf("Env = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What the kernel holds a process to is read from its unit's and its slice's cgroup and its stat file, not assumed.
|
||||||
|
func TestReadInForce(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
cg := filepath.Join(dir, "cg")
|
||||||
|
slice := filepath.Join(cg, "meshyield.slice")
|
||||||
|
unit := filepath.Join(slice, "mesh-build-go-runtime.service")
|
||||||
|
if err := os.MkdirAll(unit, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for at, files := range map[string]map[string]string{
|
||||||
|
slice: {"cpu.idle": "1\n", "cpu.weight": "0\n", "cpu.max": "1200000 100000\n", "io.weight": "default 1\n"},
|
||||||
|
unit: {"cpu.idle": "0\n", "cpu.weight": "100\n", "cpu.max": "max 100000\n", "io.weight": "default 100\n"},
|
||||||
|
} {
|
||||||
|
for name, v := range files {
|
||||||
|
if err := os.WriteFile(filepath.Join(at, name), []byte(v), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self := filepath.Join(dir, "cgroup")
|
||||||
|
stat := filepath.Join(dir, "stat")
|
||||||
|
_ = os.WriteFile(self, []byte("0::/meshyield.slice/mesh-build-go-runtime.service\n"), 0o644)
|
||||||
|
// A command with a space and a parenthesis in it, then nice 19 in field 19.
|
||||||
|
_ = os.WriteFile(stat, []byte("42 (mesh (build) go) S 1 42 42 0 -1 4194560 1 0 0 0 0 0 0 0 39 19 8 0 1 2 3\n"), 0o644)
|
||||||
|
cgroupRoot, procSelfCgroup, procSelfStat = cg, self, stat
|
||||||
|
t.Cleanup(func() {
|
||||||
|
cgroupRoot, procSelfCgroup, procSelfStat = "/sys/fs/cgroup", "/proc/self/cgroup", "/proc/self/stat"
|
||||||
|
})
|
||||||
|
|
||||||
|
got := ReadInForce()
|
||||||
|
want := InForce{Nice: 19, Cgroup: "/meshyield.slice/mesh-build-go-runtime.service",
|
||||||
|
Unit: Group{Path: "/meshyield.slice/mesh-build-go-runtime.service", CPUIdle: "0", CPUWeight: "100",
|
||||||
|
CPUMax: "max 100000", IOWeight: "default 100"},
|
||||||
|
Slice: Group{Path: "/meshyield.slice", CPUIdle: "1", CPUWeight: "0", CPUMax: "1200000 100000",
|
||||||
|
IOWeight: "default 1"},
|
||||||
|
Quota: 1200}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("ReadInForce = %+v\nwant %+v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -36,3 +36,15 @@ func TestNodeCapacity(t *testing.T) {
|
|||||||
t.Errorf("a missing file = %d, want 0", got)
|
t.Errorf("a missing file = %d, want 0", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ADR 0553 §4: a step's threads are the composer's share over the module's own capacity; without the word, none.
|
||||||
|
func TestNodeLimits(t *testing.T) {
|
||||||
|
t.Setenv("MESH_BUILD_CPU_SHARE", "")
|
||||||
|
if l := NodeLimits(3); l.Threads != 0 || l.Slots != 3 {
|
||||||
|
t.Errorf("without the word: %+v", l)
|
||||||
|
}
|
||||||
|
t.Setenv("MESH_BUILD_CPU_SHARE", "75")
|
||||||
|
if l := NodeLimits(3); l.Percent != 75 || l.Slots != 3 || l.Threads < 1 {
|
||||||
|
t.Errorf("from the word: %+v", l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
package buildstep
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildcpu"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NodeLimits is what one step of a build module is held to (novox/hq ADR 0553 §4): the node's processor share from the
|
||||||
|
// word the composer gives the module (buildcpu.ShareWord), divided by the module's own capacity, own, over the machine's
|
||||||
|
// processors. Without the word, no thread limit: Threads is 0.
|
||||||
|
func NodeLimits(own int) buildcpu.Limits {
|
||||||
|
return buildcpu.FromWords(os.Getenv, own)
|
||||||
|
}
|
||||||
@@ -172,7 +172,7 @@ func TestAStoreRowWithoutAProtocolKeepsTheCompiledOne(t *testing.T) {
|
|||||||
t.Cleanup(func() { UseSeats(was) })
|
t.Cleanup(func() { UseSeats(was) })
|
||||||
UseSeats([]Seat{{Name: "node-build-go", Scope: ScopeNode, Decision: "row"}})
|
UseSeats([]Seat{{Name: "node-build-go", Scope: ScopeNode, Decision: "row"}})
|
||||||
got, ok := SeatNamed("node-build-go")
|
got, ok := SeatNamed("node-build-go")
|
||||||
if !ok || len(got.Accepts) == 0 || !got.PulledByKind {
|
if !ok || len(got.Accepts) == 0 || !got.PulledByKind || !got.Yields {
|
||||||
t.Fatalf("the Go build seat lost what it accepts when loaded from the store: %+v", got)
|
t.Fatalf("the Go build seat lost what it accepts when loaded from the store: %+v", got)
|
||||||
}
|
}
|
||||||
if got.Decision != "row" {
|
if got.Decision != "row" {
|
||||||
|
|||||||
@@ -49,6 +49,11 @@ type Seat struct {
|
|||||||
// order Accepts lists them (novox/hq ADR 0525 §2, ADR 0536 §4): the build seats'. Compiled, never stored, like
|
// order Accepts lists them (novox/hq ADR 0525 §2, ADR 0536 §4): the build seats'. Compiled, never stored, like
|
||||||
// Receives.
|
// Receives.
|
||||||
PulledByKind bool
|
PulledByKind bool
|
||||||
|
// Yields says the seat's holder gives way to everything else on its node (novox/hq ADR 0553): its runtime runs at
|
||||||
|
// idle priority in the node's one yielding slice, under the node's one processor share (buildcpu.ShareSetting) that
|
||||||
|
// every yielding holder of the node shares. The build seats': a build must never slow the operator's desktop.
|
||||||
|
// Compiled, never stored, like Receives.
|
||||||
|
Yields bool
|
||||||
// Serves carries each verb in full — name, description, schema — because a role's tools are the
|
// Serves carries each verb in full — name, description, schema — because a role's tools are the
|
||||||
// mesh's to define and an agent's to call (novox/hq ADR 0132, design 33 §2).
|
// mesh's to define and an agent's to call (novox/hq ADR 0132, design 33 §2).
|
||||||
Serves []Verb
|
Serves []Verb
|
||||||
@@ -256,11 +261,11 @@ var defaultSeats = append([]Seat{
|
|||||||
// its holder taking nothing, kept on the module's own pause key (issue 1887's design limit 2). They replace
|
// its holder taking nothing, kept on the module's own pause key (issue 1887's design limit 2). They replace
|
||||||
// node-build-agent, retired with build-agent (ADR 0524 §5).
|
// node-build-agent, retired with build-agent (ADR 0524 §5).
|
||||||
{Name: "node-build-go", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("Go"),
|
{Name: "node-build-go", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("Go"),
|
||||||
Decision: "novox/hq ADR 0525, ADR 0536"},
|
Yields: true, Decision: "novox/hq ADR 0525, ADR 0536, ADR 0553"},
|
||||||
{Name: "node-build-node", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("Node"),
|
{Name: "node-build-node", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("Node"),
|
||||||
Decision: "novox/hq ADR 0525, ADR 0536"},
|
Yields: true, Decision: "novox/hq ADR 0525, ADR 0536, ADR 0553"},
|
||||||
{Name: "node-build-image", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("image"),
|
{Name: "node-build-image", Scope: ScopeNode, Accepts: buildAskKinds(), PulledByKind: true, Serves: buildSeatVerbs("image"),
|
||||||
Decision: "novox/hq ADR 0525, ADR 0536"},
|
Yields: true, Decision: "novox/hq ADR 0525, ADR 0536, ADR 0553"},
|
||||||
// **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and
|
// **The mesh's resolvers** (novox/hq ADR 0194, 0196, 0223): every node's internal domain, and
|
||||||
// every node and container asks them and nothing else. Replicated since ADR 0223: held on more
|
// every node and container asks them and nothing else. Replicated since ADR 0223: held on more
|
||||||
// than one machine, each answering the same names from the same roster, and every machine's
|
// than one machine, each answering the same names from the same roster, and every machine's
|
||||||
@@ -497,6 +502,7 @@ func UseSeats(s []Seat) {
|
|||||||
row.Paths = d.Paths
|
row.Paths = d.Paths
|
||||||
row.Replicated = d.Replicated
|
row.Replicated = d.Replicated
|
||||||
row.PulledByKind = d.PulledByKind
|
row.PulledByKind = d.PulledByKind
|
||||||
|
row.Yields = d.Yields // never stored (ADR 0553)
|
||||||
row.Serves = optionalAsCompiled(row.Serves, d.Serves)
|
row.Serves = optionalAsCompiled(row.Serves, d.Serves)
|
||||||
}
|
}
|
||||||
merged = append(merged, row)
|
merged = append(merged, row)
|
||||||
|
|||||||
@@ -324,6 +324,9 @@ func TestNodeBuildAgentIsRetiredForTheBuildSeats(t *testing.T) {
|
|||||||
if !ok || s.Scope != ScopeNode || !s.PulledByKind || strings.Join(s.Accepts, ",") != "delivery,check,recheck" {
|
if !ok || s.Scope != ScopeNode || !s.PulledByKind || strings.Join(s.Accepts, ",") != "delivery,check,recheck" {
|
||||||
t.Fatalf("%s = %+v", name, s)
|
t.Fatalf("%s = %+v", name, s)
|
||||||
}
|
}
|
||||||
|
if !s.Yields {
|
||||||
|
t.Errorf("%s does not yield (ADR 0553)", name)
|
||||||
|
}
|
||||||
var verbs []string
|
var verbs []string
|
||||||
for _, v := range s.Serves {
|
for _, v := range s.Serves {
|
||||||
verbs = append(verbs, v.Name)
|
verbs = append(verbs, v.Name)
|
||||||
@@ -332,4 +335,14 @@ func TestNodeBuildAgentIsRetiredForTheBuildSeats(t *testing.T) {
|
|||||||
t.Errorf("%s serves %v", name, verbs)
|
t.Errorf("%s serves %v", name, verbs)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// The build seats are the seats that yield, and no other (ADR 0553 §1).
|
||||||
|
var yielding []string
|
||||||
|
for _, s := range Seats() {
|
||||||
|
if s.Yields {
|
||||||
|
yielding = append(yielding, s.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Join(yielding, ",") != "node-build-go,node-build-node,node-build-image" {
|
||||||
|
t.Errorf("the seats that yield are %v", yielding)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -916,6 +916,56 @@ type Process struct {
|
|||||||
// Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its
|
// Health is how this resource is ready (novox/hq ADR 0240 rule 2, Phase B): one kind and its
|
||||||
// timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more.
|
// timing, judged by the node-engine beside liveness. Absent: judged alive or not, and nothing more.
|
||||||
Health *Health `json:"health,omitempty"`
|
Health *Health `json:"health,omitempty"`
|
||||||
|
|
||||||
|
// Yields says this process gives way to everything else on its machine (novox/hq ADR 0553, issue 1806): the
|
||||||
|
// runtime of a build seat's holder, whose builds must never slow the operator's desktop. Absent: the machine's
|
||||||
|
// defaults. For a process that stays up.
|
||||||
|
Yields *Yields `json:"yields,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// YieldIdle is the one priority a process yields at: the lowest there is.
|
||||||
|
const YieldIdle = "idle"
|
||||||
|
|
||||||
|
// YieldSlice is the one slice the node-engine places every yielding process of its node in (novox/hq ADR 0553 §2): a
|
||||||
|
// slice at the top of the hierarchy, beside the operator's user slice and the system's, so its idle weight and its
|
||||||
|
// quota bind against everything else on the machine. No dash in its name: a dash makes a slice a child of the slice
|
||||||
|
// named before it, where its weight would count only against its siblings.
|
||||||
|
const YieldSlice = "meshyield.slice"
|
||||||
|
|
||||||
|
// Yields is how a process gives way, written by the node-engine (novox/hq ADR 0553).
|
||||||
|
//
|
||||||
|
// **Priority** "idle": the process runs in YieldSlice, whose CPUWeight is idle (it runs only on cycles nothing else
|
||||||
|
// wants) and whose IOWeight is 1, with Nice=19 and the lowest best-effort IO level for every process it starts. The IO
|
||||||
|
// controls bind only where the disk's scheduler honours them (bfq, or io.cost for the weight); under `none`, the
|
||||||
|
// usual one for NVMe, they do nothing, and the CPU bound is what holds. **CPUShare** is the most of the machine's
|
||||||
|
// processors the slice may take, in percent, shared by every yielding process of the node together: the node-engine
|
||||||
|
// writes it as the slice's CPUQuota for the processors the machine has, so the bound is the kernel's. Every yielding
|
||||||
|
// process of one node carries the same share: the composer gives the node's one value (ADR 0553 §3). 0 is no cap.
|
||||||
|
type Yields struct {
|
||||||
|
Priority string `json:"priority,omitempty"`
|
||||||
|
CPUShare int `json:"cpu-share,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (y *Yields) problems(where string, staysUp bool) []string {
|
||||||
|
if y == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var problems []string
|
||||||
|
if !staysUp {
|
||||||
|
problems = append(problems, where+": yields is for a process that stays up")
|
||||||
|
}
|
||||||
|
switch y.Priority {
|
||||||
|
case "", YieldIdle:
|
||||||
|
default:
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: yields priority %q is not one this host knows: idle", where, y.Priority))
|
||||||
|
}
|
||||||
|
if y.CPUShare < 0 || y.CPUShare > 100 {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: yields cpu-share %d is not a percent of the machine, 0 to 100", where, y.CPUShare))
|
||||||
|
}
|
||||||
|
if y.Priority == "" && y.CPUShare == 0 {
|
||||||
|
problems = append(problems, where+": yields says neither a priority nor a cpu-share")
|
||||||
|
}
|
||||||
|
return problems
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Process) Identity() string { return d.ID }
|
func (d *Process) Identity() string { return d.ID }
|
||||||
@@ -948,6 +998,7 @@ func ProcessNameProblem(name string) string {
|
|||||||
|
|
||||||
func (d *Process) validate(where string, _ bool) []string {
|
func (d *Process) validate(where string, _ bool) []string {
|
||||||
problems := d.Health.problems(where, false, !d.RunOnce && d.Schedule == "")
|
problems := d.Health.problems(where, false, !d.RunOnce && d.Schedule == "")
|
||||||
|
problems = append(problems, d.Yields.problems(where, !d.RunOnce && d.Schedule == "")...)
|
||||||
if problem := ProcessNameProblem(d.Name); problem != "" {
|
if problem := ProcessNameProblem(d.Name); problem != "" {
|
||||||
problems = append(problems, where+": "+problem)
|
problems = append(problems, where+": "+problem)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -92,6 +92,20 @@ func TestTheModesAreExclusive(t *testing.T) {
|
|||||||
if problems := watching.validate("a process", false); len(problems) == 0 {
|
if problems := watching.validate("a process", false); len(problems) == 0 {
|
||||||
t.Fatal("a scheduled process was given something to restart on, and it is never running")
|
t.Fatal("a scheduled process was given something to restart on, and it is never running")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A process yields only while it stays up, at a priority the host knows, within the machine (issue 1806).
|
||||||
|
for name, bad := range map[string]func(*Process){
|
||||||
|
"a step that yields": func(p *Process) { p.RunOnce = true; p.Yields = &Yields{Priority: YieldIdle} },
|
||||||
|
"an unknown priority": func(p *Process) { p.Yields = &Yields{Priority: "low"} },
|
||||||
|
"a share over the whole": func(p *Process) { p.Yields = &Yields{CPUShare: 150} },
|
||||||
|
"a yield saying nothing": func(p *Process) { p.Yields = &Yields{} },
|
||||||
|
} {
|
||||||
|
p := aProcess()
|
||||||
|
bad(p)
|
||||||
|
if problems := p.validate("a process", false); len(problems) == 0 {
|
||||||
|
t.Errorf("%s was accepted", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A cadence that is not a cadence is refused near its author, rather than by a machine at the far
|
// A cadence that is not a cadence is refused near its author, rather than by a machine at the far
|
||||||
@@ -118,6 +132,11 @@ func TestEachModeOnItsOwnIsAccepted(t *testing.T) {
|
|||||||
if problems := every.validate("a process", false); len(problems) != 0 {
|
if problems := every.validate("a process", false); len(problems) != 0 {
|
||||||
t.Fatalf("a scheduled process was refused: %v", problems)
|
t.Fatalf("a scheduled process was refused: %v", problems)
|
||||||
}
|
}
|
||||||
|
yielding := aProcess()
|
||||||
|
yielding.Yields = &Yields{Priority: YieldIdle, CPUShare: 50}
|
||||||
|
if problems := yielding.validate("a process", false); len(problems) != 0 {
|
||||||
|
t.Fatalf("a daemon yielding at idle under half the machine was refused: %v", problems)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// **The one that cannot be escaped, only refused.**
|
// **The one that cannot be escaped, only refused.**
|
||||||
|
|||||||
+26
-1
@@ -43,7 +43,8 @@ func ExecRunnerFor(bound time.Duration) Runner {
|
|||||||
|
|
||||||
// ExecRunnerClean is ExecRunnerFor whose commands get exactly the env argument as their environment, and nothing of
|
// ExecRunnerClean is ExecRunnerFor whose commands get exactly the env argument as their environment, and nothing of
|
||||||
// this process's (issue 1497; novox/hq ADR 0466 §4: no registry password in a fetch's environment). The caller
|
// this process's (issue 1497; novox/hq ADR 0466 §4: no registry password in a fetch's environment). The caller
|
||||||
// gives PATH and HOME itself. A directory is the command's own business (`git -C`, `go -C`).
|
// gives PATH and HOME itself. A directory is the command's own business (`git -C`, `go -C`): it is started in the HOME
|
||||||
|
// it is given, or the machine's temporary directory, never in this process's (stableDir).
|
||||||
func ExecRunnerClean(bound time.Duration) Runner {
|
func ExecRunnerClean(bound time.Duration) Runner {
|
||||||
return func(ctx context.Context, env []string, name string, args ...string) Ran {
|
return func(ctx context.Context, env []string, name string, args ...string) Ran {
|
||||||
return execRun(ctx, bound, true, env, name, args...)
|
return execRun(ctx, bound, true, env, name, args...)
|
||||||
@@ -62,6 +63,7 @@ func execRun(ctx context.Context, bound time.Duration, clean bool, env []string,
|
|||||||
} else {
|
} else {
|
||||||
cmd.Env = append(os.Environ(), env...)
|
cmd.Env = append(os.Environ(), env...)
|
||||||
}
|
}
|
||||||
|
cmd.Dir = stableDir(clean, env)
|
||||||
var out, errb bytes.Buffer
|
var out, errb bytes.Buffer
|
||||||
cmd.Stdout, cmd.Stderr = &out, &errb
|
cmd.Stdout, cmd.Stderr = &out, &errb
|
||||||
err := cmd.Run()
|
err := cmd.Run()
|
||||||
@@ -83,6 +85,29 @@ func execRun(ctx context.Context, bound time.Duration, clean bool, env []string,
|
|||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stableDir is the directory a command starts in when it names none (novox/hq issue 734). **Never one that may be
|
||||||
|
// deleted under it**: a module runs in the directory its build was unpacked in, which a new build replaces, and a
|
||||||
|
// command started there inherits it — `go version` then fails "getwd: no such file or directory" before it reads
|
||||||
|
// anything. A clean command starts in the HOME it is given, or the temporary directory; any other keeps this
|
||||||
|
// process's directory while it exists, and starts in the temporary directory once it is gone. "" is this process's.
|
||||||
|
func stableDir(clean bool, env []string) string {
|
||||||
|
if !clean {
|
||||||
|
if _, err := os.Getwd(); err == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return os.TempDir()
|
||||||
|
}
|
||||||
|
for i := len(env) - 1; i >= 0; i-- {
|
||||||
|
if home, ok := strings.CutPrefix(env[i], "HOME="); ok {
|
||||||
|
if info, err := os.Stat(home); err == nil && info.IsDir() {
|
||||||
|
return home
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return os.TempDir()
|
||||||
|
}
|
||||||
|
|
||||||
func firstLine(s string) string {
|
func firstLine(s string) string {
|
||||||
for _, l := range strings.Split(s, "\n") {
|
for _, l := range strings.Split(s, "\n") {
|
||||||
if l = strings.TrimSpace(l); l != "" {
|
if l = strings.TrimSpace(l); l != "" {
|
||||||
|
|||||||
@@ -48,13 +48,29 @@ func TestExecRunnerForZeroIsCutByItsContext(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1497/AC4: ExecRunnerClean(0) runs env with only the given words: PATH and HOME, not the process's secret.
|
// 1497/AC4: ExecRunnerClean(0) runs env with only the given words: PATH and HOME, not the process's secret. And issue
|
||||||
|
// 734: run from a process whose own directory was deleted under it (a module whose build a new one replaced), a clean
|
||||||
|
// command starts in its HOME, and any other in the temporary directory, rather than failing to find where it is.
|
||||||
func TestExecRunnerCleanGivesTheCommandOnlyTheEnvArgument(t *testing.T) {
|
func TestExecRunnerCleanGivesTheCommandOnlyTheEnvArgument(t *testing.T) {
|
||||||
t.Setenv("MESH_TEST_SECRET", "x")
|
t.Setenv("MESH_TEST_SECRET", "x")
|
||||||
r := ExecRunnerClean(0)(context.Background(), []string{"PATH=" + os.Getenv("PATH"), "HOME=/h"}, "env")
|
r := ExecRunnerClean(0)(context.Background(), []string{"PATH=" + os.Getenv("PATH"), "HOME=/h"}, "env")
|
||||||
if r.Status != 0 || !strings.Contains(r.Stdout, "HOME=/h") || !strings.Contains(r.Stdout, "PATH=") || strings.Contains(r.Stdout, "MESH_TEST_SECRET") {
|
if r.Status != 0 || !strings.Contains(r.Stdout, "HOME=/h") || !strings.Contains(r.Stdout, "PATH=") || strings.Contains(r.Stdout, "MESH_TEST_SECRET") {
|
||||||
t.Errorf("ran = %+v", r)
|
t.Errorf("ran = %+v", r)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
home, gone := t.TempDir(), t.TempDir()
|
||||||
|
t.Chdir(gone)
|
||||||
|
if err := os.Remove(gone); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
r = ExecRunnerClean(0)(context.Background(), []string{"PATH=" + os.Getenv("PATH"), "HOME=" + home}, "pwd")
|
||||||
|
if r.Status != 0 || strings.TrimSpace(r.Stdout) != home {
|
||||||
|
t.Errorf("a clean command run from a deleted directory = %+v, want it started in its HOME %s", r, home)
|
||||||
|
}
|
||||||
|
r = ExecRunnerFor(0)(context.Background(), nil, "pwd")
|
||||||
|
if r.Status != 0 || strings.TrimSpace(r.Stdout) != os.TempDir() {
|
||||||
|
t.Errorf("a command run from a deleted directory = %+v, want it started in %s", r, os.TempDir())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1497/AC5: ExecRunnerFor(0) is unchanged: the env argument is added to the process's environment.
|
// 1497/AC5: ExecRunnerFor(0) is unchanged: the env argument is added to the process's environment.
|
||||||
|
|||||||
Reference in New Issue
Block a user