diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 3ac93d61..09d57370 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -64,6 +64,8 @@ is dialled except the broker. MESH_NPM_TOKEN the token for it, likewise MESH_NPM_SCOPE the scope it answers for (default: @novox) MESH_WORKSPACE where to clone and build (default: a temporary directory) + MESH_BUILD_CGROUP_PARENT the slice every container a build or check starts joins, which carries the + machine's limits on builds (novox/hq issue 1798; default: the runtime's own) It also builds one module and stops, which is how a mesh is raised — before there is a broker to take work from or a registry to publish into: @@ -137,6 +139,10 @@ func run() error { if err != nil { return err } + // A build and a check run below the machine's own work (novox/hq issue 1798): the agent itself at the + // lowest priority, which its git and go children inherit, and every container in the build slice. + _ = syscall.Setpriority(syscall.PRIO_PROCESS, 0, 19) + builder.CgroupParent = strings.TrimSpace(os.Getenv("MESH_BUILD_CGROUP_PARENT")) workspace := os.Getenv("MESH_WORKSPACE") if workspace == "" { workspace = os.TempDir() + "/mesh-builder" diff --git a/internal/builder/kill.go b/internal/builder/kill.go index 19bec371..a4fced20 100644 --- a/internal/builder/kill.go +++ b/internal/builder/kill.go @@ -42,6 +42,11 @@ func inItsOwnGroup(cmd *exec.Cmd) { cmd.WaitDelay = KillWait } +// CgroupParent is the slice every container a build or check starts joins (novox/hq issue 1798), so the node's own +// weights and cap bind it: the build-agent module declares the slice with the machine's CPU weight, IO weight and +// memory limits, and names it in MESH_BUILD_CGROUP_PARENT. Empty, a container joins the runtime's default. +var CgroupParent string + // Labelled is a Runner that marks every container a build starts with the build's id, so a kill // finds what outlived the docker client (novox/hq ADR 0219). Applied at the one place every command // passes rather than at each `docker run` the builder composes: a run added later is labelled too. @@ -56,8 +61,11 @@ func LabelledArgs(name string, args []string, id string) []string { if name != "docker" || len(args) == 0 || args[0] != "run" || id == "" { return args } - out := make([]string, 0, len(args)+2) + out := make([]string, 0, len(args)+4) out = append(out, "run", "--label", BuildLabel+"="+id) + if CgroupParent != "" { + out = append(out, "--cgroup-parent", CgroupParent) + } return append(out, args[1:]...) } diff --git a/internal/builder/kill_test.go b/internal/builder/kill_test.go index 52ba7c46..a963586d 100644 --- a/internal/builder/kill_test.go +++ b/internal/builder/kill_test.go @@ -143,3 +143,24 @@ func TestAVolumeAlreadyGoneIsNoFailure(t *testing.T) { t.Fatalf("%d %v", n, err) } } + +// A node's build slice limits everything a build or check starts (novox/hq issue 1798): with a parent +// set, every `docker run` joins it, so the node's CPU and IO weights and memory cap bind the container; +// with none set the arguments are as they always were. +func TestEveryContainerJoinsTheNodesBuildSlice(t *testing.T) { + was := CgroupParent + defer func() { CgroupParent = was }() + CgroupParent = "meshbuild.slice" + got := LabelledArgs("docker", []string{"run", "--rm", "img", "sh"}, "build-1") + want := []string{"run", "--label", "mesh.build=build-1", "--cgroup-parent", "meshbuild.slice", "--rm", "img", "sh"} + if !reflect.DeepEqual(got, want) { + t.Errorf("1798/AC1: docker run became %v", got) + } + if got := LabelledArgs("docker", []string{"build", "."}, "build-1"); !reflect.DeepEqual(got, []string{"build", "."}) { + t.Errorf("a docker build became %v", got) + } + CgroupParent = "" + if got := LabelledArgs("docker", []string{"run", "img"}, "build-1"); !reflect.DeepEqual(got, []string{"run", "--label", "mesh.build=build-1", "img"}) { + t.Errorf("1798/AC1: with no slice, docker run became %v", got) + } +} diff --git a/modules/build-agent/files/build-agent.env b/modules/build-agent/files/build-agent.env index 15ee1bbb..819098b4 100644 --- a/modules/build-agent/files/build-agent.env +++ b/modules/build-agent/files/build-agent.env @@ -6,3 +6,4 @@ MESH_NPM_TOKEN_FILE=/run/mesh/package-registry.secret MESH_GENESIS_PACKAGE_REGISTRY=/run/mesh/genesis-package-registry.json MESH_WORKSPACE=${dir:workspace} MESH_JUDGE_CACHE=${dir:judge-cache} +MESH_BUILD_CGROUP_PARENT=meshbuild.slice diff --git a/modules/build-agent/files/meshbuild.slice b/modules/build-agent/files/meshbuild.slice new file mode 100644 index 00000000..6258a1af --- /dev/null +++ b/modules/build-agent/files/meshbuild.slice @@ -0,0 +1,10 @@ +# Managed by the mesh (module build-agent). Replaced on every push; edit the module's settings instead. +# +# Everything a build or check starts on this machine runs in this slice (novox/hq issue 1798): the build +# agent starts every container with --cgroup-parent=meshbuild.slice. The weights only bind while the machine +# is busy, so an idle machine still builds at full speed; the memory limits are the part that always binds. +[Slice] +CPUWeight=${setting:cpu-weight} +IOWeight=${setting:io-weight} +MemoryHigh=${setting:memory-high} +MemoryMax=${setting:memory-max} diff --git a/modules/build-agent/manifest_test.go b/modules/build-agent/manifest_test.go index a0716251..06d0f7ff 100644 --- a/modules/build-agent/manifest_test.go +++ b/modules/build-agent/manifest_test.go @@ -187,3 +187,61 @@ func TestTheAgentReadsTheGatesPullKeysAndTodaysBoth(t *testing.T) { } } } + +// Builds and checks run below the machine's own work (novox/hq issue 1798): the module declares the slice +// every build container joins with its limits as settings that have defaults, and tells the agent its name. +func TestTheBuildSliceCarriesTheMachinesLimits(t *testing.T) { + raw, err := os.ReadFile("module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Config map[string]struct { + Default any `json:"default"` + } `json:"config"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + slice, err := os.ReadFile("files/meshbuild.slice") + if err != nil { + t.Fatal(err) + } + for key, property := range map[string]string{"cpu-weight": "CPUWeight", "io-weight": "IOWeight", "memory-high": "MemoryHigh", "memory-max": "MemoryMax"} { + if c, ok := m.Config[key]; !ok || c.Default == nil { + t.Errorf("1798/AC2: setting %s is missing or has no default", key) + } + if !strings.Contains(string(slice), property+"=${setting:"+key+"}") { + t.Errorf("1798/AC2: the slice does not set %s from %s", property, key) + } + } + var placed, running bool + for _, r := range read(t).Resources { + placed = placed || (r["type"] == "file" && r["path"] == "/etc/systemd/system/meshbuild.slice") + running = running || (r["type"] == "service" && r["unit"] == "meshbuild.slice" && r["state"] == "running") + } + if !placed || !running { + t.Errorf("1798/AC2: the slice must be placed (%v) and running (%v)", placed, running) + } + // 1798/AC3: a dash in a slice's name nests it in the slice named before the dash, and cgroup v2 weights only + // compare siblings, so the name has none and the slice sits at the top level beside user.slice. + for _, r := range read(t).Resources { + if r["type"] == "service" && r["unit"] == "meshbuild.slice" { + if strings.Contains(strings.TrimSuffix(r["unit"].(string), ".slice"), "-") { + t.Errorf("1798/AC3: slice name %v nests", r["unit"]) + } + // 1798/AC4: a changed limit reaches systemd only if the unit restarts when its file changes. + restart, _ := r["restart-on"].([]any) + if len(restart) != 1 || restart[0] != "build-slice" { + t.Errorf("1798/AC4: the slice unit does not restart on its file: %v", restart) + } + } + } + env, err := os.ReadFile("files/build-agent.env") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(env), "MESH_BUILD_CGROUP_PARENT=meshbuild.slice") { + t.Error("1798/AC1: the agent is not told the slice's name") + } +} diff --git a/modules/build-agent/module.json b/modules/build-agent/module.json index 5a550ef6..c25e711a 100644 --- a/modules/build-agent/module.json +++ b/modules/build-agent/module.json @@ -2,6 +2,34 @@ "module": "build-agent", "version": "1", "slug": "agent", + "config": { + "cpu-weight": { + "type": "number", + "description": "The CPU weight of everything a build or check starts on this machine, from 1 to 10000 against 100 for an ordinary service or a login session: under load the builds get their share of that, on an idle machine they take what is free (novox/hq issue 1798). The default 20 is about a sixth of the CPU against one competing workload; a desktop may take 10. Changing it restarts the build slice, which stops every build and check running on that machine at that moment; they fail and must be retried.", + "minimum": 1, + "maximum": 10000, + "default": 20 + }, + "io-weight": { + "type": "number", + "description": "The disk IO weight of everything a build or check starts, from 1 to 10000 against 100 for an ordinary service: the default 10 lets the machine's own work go first (novox/hq issue 1798). Changing it restarts the build slice, which stops every build and check running on that machine at that moment; they fail and must be retried.", + "minimum": 1, + "maximum": 10000, + "default": 10 + }, + "memory-high": { + "type": "string", + "description": "The memory above which builds and checks are slowed and made to give memory back, a percentage of the machine's memory or a size (systemd's MemoryHigh): the default 50% leaves half the machine to what it runs (novox/hq issue 1798). Changing it restarts the build slice, which stops every build and check running on that machine at that moment; they fail and must be retried.", + "pattern": "([1-9][0-9]?|100)%|[1-9][0-9]*[KMGT]", + "default": "50%" + }, + "memory-max": { + "type": "string", + "description": "The memory builds and checks together may never exceed, a percentage or a size (systemd's MemoryMax); a build past it is killed by the kernel and fails, so keep it above memory-high: the default 70% (novox/hq issue 1798). Changing it restarts the build slice, which stops every build and check running on that machine at that moment; they fail and must be retried.", + "pattern": "([1-9][0-9]?|100)%|[1-9][0-9]*[KMGT]", + "default": "70%" + } + }, "capabilities": [ "container-runtime" ], @@ -9,7 +37,12 @@ { "name": "node-build-agent", "scope": "node", - "serves": ["current", "kill", "pause", "resume"] + "serves": [ + "current", + "kill", + "pause", + "resume" + ] } ], "reads": [ @@ -48,7 +81,7 @@ "id": "judge-cache", "path": "${dir:judge-cache}", "class": "cache", - "why": "the Go cache every merge check's judge is built with and the group judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 §6)" + "why": "the Go cache every merge check's judge is built with and the group judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 \u00a76)" } ] }, @@ -69,6 +102,23 @@ "type": "directory", "mode": "0700" }, + { + "id": "build-slice", + "type": "file", + "path": "/etc/systemd/system/meshbuild.slice", + "mode": "0644", + "trusted": false, + "template": "files/meshbuild.slice" + }, + { + "id": "build-slice-unit", + "type": "service", + "unit": "meshbuild.slice", + "state": "running", + "restart-on": [ + "build-slice" + ] + }, { "id": "agent-env", "type": "file",