Limit what a build or check starts on a build node: a per-node build slice (issue 1798) #432

Merged
mesh-tools merged 4 commits from fix/1798-limit-build-resources into main 2026-10-21 23:20:57 +00:00
7 changed files with 157 additions and 3 deletions
+6
View File
@@ -64,6 +64,8 @@ is dialled except the broker.
MESH_NPM_TOKEN the token for it, likewise
MESH_NPM_SCOPE the scope it answers for (default: @novox)
MESH_WORKSPACE where to clone and build (default: a temporary directory)
MESH_BUILD_CGROUP_PARENT the slice every container a build or check starts joins, which carries the
machine's limits on builds (novox/hq issue 1798; default: the runtime's own)
It also builds one module and stops, which is how a mesh is raised — before there is a
broker to take work from or a registry to publish into:
@@ -137,6 +139,10 @@ func run() error {
if err != nil {
return err
}
// A build and a check run below the machine's own work (novox/hq issue 1798): the agent itself at the
// lowest priority, which its git and go children inherit, and every container in the build slice.
_ = syscall.Setpriority(syscall.PRIO_PROCESS, 0, 19)
builder.CgroupParent = strings.TrimSpace(os.Getenv("MESH_BUILD_CGROUP_PARENT"))
workspace := os.Getenv("MESH_WORKSPACE")
if workspace == "" {
workspace = os.TempDir() + "/mesh-builder"
+9 -1
View File
@@ -42,6 +42,11 @@ func inItsOwnGroup(cmd *exec.Cmd) {
cmd.WaitDelay = KillWait
}
// CgroupParent is the slice every container a build or check starts joins (novox/hq issue 1798), so the node's own
// weights and cap bind it: the build-agent module declares the slice with the machine's CPU weight, IO weight and
// memory limits, and names it in MESH_BUILD_CGROUP_PARENT. Empty, a container joins the runtime's default.
var CgroupParent string
// Labelled is a Runner that marks every container a build starts with the build's id, so a kill
// finds what outlived the docker client (novox/hq ADR 0219). Applied at the one place every command
// passes rather than at each `docker run` the builder composes: a run added later is labelled too.
@@ -56,8 +61,11 @@ func LabelledArgs(name string, args []string, id string) []string {
if name != "docker" || len(args) == 0 || args[0] != "run" || id == "" {
return args
}
out := make([]string, 0, len(args)+2)
out := make([]string, 0, len(args)+4)
out = append(out, "run", "--label", BuildLabel+"="+id)
if CgroupParent != "" {
out = append(out, "--cgroup-parent", CgroupParent)
}
return append(out, args[1:]...)
}
+21
View File
@@ -143,3 +143,24 @@ func TestAVolumeAlreadyGoneIsNoFailure(t *testing.T) {
t.Fatalf("%d %v", n, err)
}
}
// A node's build slice limits everything a build or check starts (novox/hq issue 1798): with a parent
// set, every `docker run` joins it, so the node's CPU and IO weights and memory cap bind the container;
// with none set the arguments are as they always were.
func TestEveryContainerJoinsTheNodesBuildSlice(t *testing.T) {
was := CgroupParent
defer func() { CgroupParent = was }()
CgroupParent = "meshbuild.slice"
got := LabelledArgs("docker", []string{"run", "--rm", "img", "sh"}, "build-1")
want := []string{"run", "--label", "mesh.build=build-1", "--cgroup-parent", "meshbuild.slice", "--rm", "img", "sh"}
if !reflect.DeepEqual(got, want) {
t.Errorf("1798/AC1: docker run became %v", got)
}
if got := LabelledArgs("docker", []string{"build", "."}, "build-1"); !reflect.DeepEqual(got, []string{"build", "."}) {
t.Errorf("a docker build became %v", got)
}
CgroupParent = ""
if got := LabelledArgs("docker", []string{"run", "img"}, "build-1"); !reflect.DeepEqual(got, []string{"run", "--label", "mesh.build=build-1", "img"}) {
t.Errorf("1798/AC1: with no slice, docker run became %v", got)
}
}
@@ -6,3 +6,4 @@ MESH_NPM_TOKEN_FILE=/run/mesh/package-registry.secret
MESH_GENESIS_PACKAGE_REGISTRY=/run/mesh/genesis-package-registry.json
MESH_WORKSPACE=${dir:workspace}
MESH_JUDGE_CACHE=${dir:judge-cache}
MESH_BUILD_CGROUP_PARENT=meshbuild.slice
+10
View File
@@ -0,0 +1,10 @@
# Managed by the mesh (module build-agent). Replaced on every push; edit the module's settings instead.
#
# Everything a build or check starts on this machine runs in this slice (novox/hq issue 1798): the build
# agent starts every container with --cgroup-parent=meshbuild.slice. The weights only bind while the machine
# is busy, so an idle machine still builds at full speed; the memory limits are the part that always binds.
[Slice]
CPUWeight=${setting:cpu-weight}
IOWeight=${setting:io-weight}
MemoryHigh=${setting:memory-high}
MemoryMax=${setting:memory-max}
+58
View File
@@ -187,3 +187,61 @@ func TestTheAgentReadsTheGatesPullKeysAndTodaysBoth(t *testing.T) {
}
}
}
// Builds and checks run below the machine's own work (novox/hq issue 1798): the module declares the slice
// every build container joins with its limits as settings that have defaults, and tells the agent its name.
func TestTheBuildSliceCarriesTheMachinesLimits(t *testing.T) {
raw, err := os.ReadFile("module.json")
if err != nil {
t.Fatal(err)
}
var m struct {
Config map[string]struct {
Default any `json:"default"`
} `json:"config"`
}
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
slice, err := os.ReadFile("files/meshbuild.slice")
if err != nil {
t.Fatal(err)
}
for key, property := range map[string]string{"cpu-weight": "CPUWeight", "io-weight": "IOWeight", "memory-high": "MemoryHigh", "memory-max": "MemoryMax"} {
if c, ok := m.Config[key]; !ok || c.Default == nil {
t.Errorf("1798/AC2: setting %s is missing or has no default", key)
}
if !strings.Contains(string(slice), property+"=${setting:"+key+"}") {
t.Errorf("1798/AC2: the slice does not set %s from %s", property, key)
}
}
var placed, running bool
for _, r := range read(t).Resources {
placed = placed || (r["type"] == "file" && r["path"] == "/etc/systemd/system/meshbuild.slice")
running = running || (r["type"] == "service" && r["unit"] == "meshbuild.slice" && r["state"] == "running")
}
if !placed || !running {
t.Errorf("1798/AC2: the slice must be placed (%v) and running (%v)", placed, running)
}
// 1798/AC3: a dash in a slice's name nests it in the slice named before the dash, and cgroup v2 weights only
// compare siblings, so the name has none and the slice sits at the top level beside user.slice.
for _, r := range read(t).Resources {
if r["type"] == "service" && r["unit"] == "meshbuild.slice" {
if strings.Contains(strings.TrimSuffix(r["unit"].(string), ".slice"), "-") {
t.Errorf("1798/AC3: slice name %v nests", r["unit"])
}
// 1798/AC4: a changed limit reaches systemd only if the unit restarts when its file changes.
restart, _ := r["restart-on"].([]any)
if len(restart) != 1 || restart[0] != "build-slice" {
t.Errorf("1798/AC4: the slice unit does not restart on its file: %v", restart)
}
}
}
env, err := os.ReadFile("files/build-agent.env")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(env), "MESH_BUILD_CGROUP_PARENT=meshbuild.slice") {
t.Error("1798/AC1: the agent is not told the slice's name")
}
}
+52 -2
View File
@@ -2,6 +2,34 @@
"module": "build-agent",
"version": "1",
"slug": "agent",
"config": {
"cpu-weight": {
"type": "number",
"description": "The CPU weight of everything a build or check starts on this machine, from 1 to 10000 against 100 for an ordinary service or a login session: under load the builds get their share of that, on an idle machine they take what is free (novox/hq issue 1798). The default 20 is about a sixth of the CPU against one competing workload; a desktop may take 10. Changing it restarts the build slice, which stops every build and check running on that machine at that moment; they fail and must be retried.",
"minimum": 1,
"maximum": 10000,
"default": 20
},
"io-weight": {
"type": "number",
"description": "The disk IO weight of everything a build or check starts, from 1 to 10000 against 100 for an ordinary service: the default 10 lets the machine's own work go first (novox/hq issue 1798). Changing it restarts the build slice, which stops every build and check running on that machine at that moment; they fail and must be retried.",
"minimum": 1,
"maximum": 10000,
"default": 10
},
"memory-high": {
"type": "string",
"description": "The memory above which builds and checks are slowed and made to give memory back, a percentage of the machine's memory or a size (systemd's MemoryHigh): the default 50% leaves half the machine to what it runs (novox/hq issue 1798). Changing it restarts the build slice, which stops every build and check running on that machine at that moment; they fail and must be retried.",
"pattern": "([1-9][0-9]?|100)%|[1-9][0-9]*[KMGT]",
"default": "50%"
},
"memory-max": {
"type": "string",
"description": "The memory builds and checks together may never exceed, a percentage or a size (systemd's MemoryMax); a build past it is killed by the kernel and fails, so keep it above memory-high: the default 70% (novox/hq issue 1798). Changing it restarts the build slice, which stops every build and check running on that machine at that moment; they fail and must be retried.",
"pattern": "([1-9][0-9]?|100)%|[1-9][0-9]*[KMGT]",
"default": "70%"
}
},
"capabilities": [
"container-runtime"
],
@@ -9,7 +37,12 @@
{
"name": "node-build-agent",
"scope": "node",
"serves": ["current", "kill", "pause", "resume"]
"serves": [
"current",
"kill",
"pause",
"resume"
]
}
],
"reads": [
@@ -48,7 +81,7 @@
"id": "judge-cache",
"path": "${dir:judge-cache}",
"class": "cache",
"why": "the Go cache every merge check's judge is built with and the group judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 §6)"
"why": "the Go cache every merge check's judge is built with and the group judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 \u00a76)"
}
]
},
@@ -69,6 +102,23 @@
"type": "directory",
"mode": "0700"
},
{
"id": "build-slice",
"type": "file",
"path": "/etc/systemd/system/meshbuild.slice",
"mode": "0644",
"trusted": false,
"template": "files/meshbuild.slice"
},
{
"id": "build-slice-unit",
"type": "service",
"unit": "meshbuild.slice",
"state": "running",
"restart-on": [
"build-slice"
]
},
{
"id": "agent-env",
"type": "file",