Run builds and checks in a per-node build slice with CPU, IO and memory limits (issue 1798)

cmd/mesh-builder/main.go

@@ -64,6 +64,8 @@ is dialled except the broker.
   MESH_NPM_TOKEN     the token for it, likewise
   MESH_NPM_SCOPE     the scope it answers for (default: @novox)
   MESH_WORKSPACE     where to clone and build (default: a temporary directory)
+  MESH_BUILD_CGROUP_PARENT  the slice every container a build or check starts joins, which carries the
+                     machine's limits on builds (novox/hq issue 1798; default: the runtime's own)
 
 It also builds one module and stops, which is how a mesh is raised — before there is a
 broker to take work from or a registry to publish into:
@@ -137,6 +139,10 @@ func run() error {
 	if err != nil {
 		return err
 	}
+	// A build and a check run below the machine's own work (novox/hq issue 1798): the agent itself at the
+	// lowest priority, which its git and go children inherit, and every container in the build slice.
+	_ = syscall.Setpriority(syscall.PRIO_PROCESS, 0, 19)
+	builder.CgroupParent = strings.TrimSpace(os.Getenv("MESH_BUILD_CGROUP_PARENT"))
 	workspace := os.Getenv("MESH_WORKSPACE")
 	if workspace == "" {
 		workspace = os.TempDir() + "/mesh-builder"

internal/builder/kill.go

@@ -42,6 +42,11 @@ func inItsOwnGroup(cmd *exec.Cmd) {
 	cmd.WaitDelay = KillWait
 }
 
+// CgroupParent is the slice every container a build or check starts joins (novox/hq issue 1798), so the node's own
+// weights and cap bind it: the build-agent module declares the slice with the machine's CPU weight, IO weight and
+// memory limits, and names it in MESH_BUILD_CGROUP_PARENT. Empty, a container joins the runtime's default.
+var CgroupParent string
+
 // Labelled is a Runner that marks every container a build starts with the build's id, so a kill
 // finds what outlived the docker client (novox/hq ADR 0219). Applied at the one place every command
 // passes rather than at each `docker run` the builder composes: a run added later is labelled too.
@@ -56,8 +61,11 @@ func LabelledArgs(name string, args []string, id string) []string {
 	if name != "docker" || len(args) == 0 || args[0] != "run" || id == "" {
 		return args
 	}
-	out := make([]string, 0, len(args)+2)
+	out := make([]string, 0, len(args)+4)
 	out = append(out, "run", "--label", BuildLabel+"="+id)
+	if CgroupParent != "" {
+		out = append(out, "--cgroup-parent", CgroupParent)
+	}
 	return append(out, args[1:]...)
 }
 

internal/builder/kill_test.go

@@ -143,3 +143,24 @@ func TestAVolumeAlreadyGoneIsNoFailure(t *testing.T) {
 		t.Fatalf("%d %v", n, err)
 	}
 }
+
+// A node's build slice limits everything a build or check starts (novox/hq issue 1798): with a parent
+// set, every `docker run` joins it, so the node's CPU and IO weights and memory cap bind the container;
+// with none set the arguments are as they always were.
+func TestEveryContainerJoinsTheNodesBuildSlice(t *testing.T) {
+	was := CgroupParent
+	defer func() { CgroupParent = was }()
+	CgroupParent = "mesh-build.slice"
+	got := LabelledArgs("docker", []string{"run", "--rm", "img", "sh"}, "build-1")
+	want := []string{"run", "--label", "mesh.build=build-1", "--cgroup-parent", "mesh-build.slice", "--rm", "img", "sh"}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("1798/AC1: docker run became %v", got)
+	}
+	if got := LabelledArgs("docker", []string{"build", "."}, "build-1"); !reflect.DeepEqual(got, []string{"build", "."}) {
+		t.Errorf("a docker build became %v", got)
+	}
+	CgroupParent = ""
+	if got := LabelledArgs("docker", []string{"run", "img"}, "build-1"); !reflect.DeepEqual(got, []string{"run", "--label", "mesh.build=build-1", "img"}) {
+		t.Errorf("1798/AC1: with no slice, docker run became %v", got)
+	}
+}

modules/build-agent/files/build-agent.env

@@ -6,3 +6,4 @@ MESH_NPM_TOKEN_FILE=/run/mesh/package-registry.secret
 MESH_GENESIS_PACKAGE_REGISTRY=/run/mesh/genesis-package-registry.json
 MESH_WORKSPACE=${dir:workspace}
 MESH_JUDGE_CACHE=${dir:judge-cache}
+MESH_BUILD_CGROUP_PARENT=mesh-build.slice

modules/build-agent/files/mesh-build.slice

@@ -0,0 +1,10 @@
+# Managed by the mesh (module build-agent). Replaced on every push; edit the module's settings instead.
+#
+# Everything a build or check starts on this machine runs in this slice (novox/hq issue 1798): the build
+# agent starts every container with --cgroup-parent=mesh-build.slice. The weights only bind while the machine
+# is busy, so an idle machine still builds at full speed; the memory limits are the part that always binds.
+[Slice]
+CPUWeight=${config:cpu-weight}
+IOWeight=${config:io-weight}
+MemoryHigh=${config:memory-high}
+MemoryMax=${config:memory-max}

modules/build-agent/manifest_test.go

@@ -187,3 +187,47 @@ func TestTheAgentReadsTheGatesPullKeysAndTodaysBoth(t *testing.T) {
 		}
 	}
 }
+
+// Builds and checks run below the machine's own work (novox/hq issue 1798): the module declares the slice
+// every build container joins with its limits as settings that have defaults, and tells the agent its name.
+func TestTheBuildSliceCarriesTheMachinesLimits(t *testing.T) {
+	raw, err := os.ReadFile("module.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var m struct {
+		Config map[string]struct {
+			Default any `json:"default"`
+		} `json:"config"`
+	}
+	if err := json.Unmarshal(raw, &m); err != nil {
+		t.Fatal(err)
+	}
+	slice, err := os.ReadFile("files/mesh-build.slice")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for key, property := range map[string]string{"cpu-weight": "CPUWeight", "io-weight": "IOWeight", "memory-high": "MemoryHigh", "memory-max": "MemoryMax"} {
+		if c, ok := m.Config[key]; !ok || c.Default == nil {
+			t.Errorf("1798/AC2: setting %s is missing or has no default", key)
+		}
+		if !strings.Contains(string(slice), property+"=${config:"+key+"}") {
+			t.Errorf("1798/AC2: the slice does not set %s from %s", property, key)
+		}
+	}
+	var placed, running bool
+	for _, r := range read(t).Resources {
+		placed = placed || (r["type"] == "file" && r["path"] == "/etc/systemd/system/mesh-build.slice")
+		running = running || (r["type"] == "service" && r["unit"] == "mesh-build.slice" && r["state"] == "running")
+	}
+	if !placed || !running {
+		t.Errorf("1798/AC2: the slice must be placed (%v) and running (%v)", placed, running)
+	}
+	env, err := os.ReadFile("files/build-agent.env")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(env), "MESH_BUILD_CGROUP_PARENT=mesh-build.slice") {
+		t.Error("1798/AC1: the agent is not told the slice's name")
+	}
+}

modules/build-agent/module.json

@@ -2,6 +2,34 @@
   "module": "build-agent",
   "version": "1",
   "slug": "agent",
+  "config": {
+    "cpu-weight": {
+      "type": "number",
+      "description": "The CPU weight of everything a build or check starts on this machine, from 1 to 10000 against 100 for an ordinary service or a login session: under load the builds get their share of that, on an idle machine they take what is free (novox/hq issue 1798). The default 20 is about a sixth of the CPU against one competing workload; a desktop may take 10.",
+      "minimum": 1,
+      "maximum": 10000,
+      "default": 20
+    },
+    "io-weight": {
+      "type": "number",
+      "description": "The disk IO weight of everything a build or check starts, from 1 to 10000 against 100 for an ordinary service: the default 10 lets the machine's own work go first (novox/hq issue 1798).",
+      "minimum": 1,
+      "maximum": 10000,
+      "default": 10
+    },
+    "memory-high": {
+      "type": "string",
+      "description": "The memory above which builds and checks are slowed and made to give memory back, a percentage of the machine's memory or a size (systemd's MemoryHigh): the default 50% leaves half the machine to what it runs (novox/hq issue 1798).",
+      "pattern": "([1-9][0-9]?|100)%|[1-9][0-9]*[KMGT]",
+      "default": "50%"
+    },
+    "memory-max": {
+      "type": "string",
+      "description": "The memory builds and checks together may never exceed, a percentage or a size (systemd's MemoryMax); a build past it is killed by the kernel and fails, so keep it above memory-high: the default 70% (novox/hq issue 1798).",
+      "pattern": "([1-9][0-9]?|100)%|[1-9][0-9]*[KMGT]",
+      "default": "70%"
+    }
+  },
   "capabilities": [
     "container-runtime"
   ],
@@ -9,7 +37,12 @@
     {
       "name": "node-build-agent",
       "scope": "node",
-      "serves": ["current", "kill", "pause", "resume"]
+      "serves": [
+        "current",
+        "kill",
+        "pause",
+        "resume"
+      ]
     }
   ],
   "reads": [
@@ -48,7 +81,7 @@
         "id": "judge-cache",
         "path": "${dir:judge-cache}",
         "class": "cache",
-        "why": "the Go cache every merge check's judge is built with and the group judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 §6)"
+        "why": "the Go cache every merge check's judge is built with and the group judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 \u00a76)"
       }
     ]
   },
@@ -69,6 +102,19 @@
       "type": "directory",
       "mode": "0700"
     },
+    {
+      "id": "build-slice",
+      "type": "file",
+      "path": "/etc/systemd/system/mesh-build.slice",
+      "mode": "0644",
+      "template": "files/mesh-build.slice"
+    },
+    {
+      "id": "build-slice-unit",
+      "type": "service",
+      "unit": "mesh-build.slice",
+      "state": "running"
+    },
     {
       "id": "agent-env",
       "type": "file",