Run builds and checks in a per-node build slice with CPU, IO and memory limits (issue 1798)

modules/build-agent/files/build-agent.env

@@ -6,3 +6,4 @@ MESH_NPM_TOKEN_FILE=/run/mesh/package-registry.secret
 MESH_GENESIS_PACKAGE_REGISTRY=/run/mesh/genesis-package-registry.json
 MESH_WORKSPACE=${dir:workspace}
 MESH_JUDGE_CACHE=${dir:judge-cache}
+MESH_BUILD_CGROUP_PARENT=mesh-build.slice

modules/build-agent/files/mesh-build.slice

@@ -0,0 +1,10 @@
+# Managed by the mesh (module build-agent). Replaced on every push; edit the module's settings instead.
+#
+# Everything a build or check starts on this machine runs in this slice (novox/hq issue 1798): the build
+# agent starts every container with --cgroup-parent=mesh-build.slice. The weights only bind while the machine
+# is busy, so an idle machine still builds at full speed; the memory limits are the part that always binds.
+[Slice]
+CPUWeight=${config:cpu-weight}
+IOWeight=${config:io-weight}
+MemoryHigh=${config:memory-high}
+MemoryMax=${config:memory-max}

modules/build-agent/manifest_test.go

@@ -187,3 +187,47 @@ func TestTheAgentReadsTheGatesPullKeysAndTodaysBoth(t *testing.T) {
 		}
 	}
 }
+
+// Builds and checks run below the machine's own work (novox/hq issue 1798): the module declares the slice
+// every build container joins with its limits as settings that have defaults, and tells the agent its name.
+func TestTheBuildSliceCarriesTheMachinesLimits(t *testing.T) {
+	raw, err := os.ReadFile("module.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var m struct {
+		Config map[string]struct {
+			Default any `json:"default"`
+		} `json:"config"`
+	}
+	if err := json.Unmarshal(raw, &m); err != nil {
+		t.Fatal(err)
+	}
+	slice, err := os.ReadFile("files/mesh-build.slice")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for key, property := range map[string]string{"cpu-weight": "CPUWeight", "io-weight": "IOWeight", "memory-high": "MemoryHigh", "memory-max": "MemoryMax"} {
+		if c, ok := m.Config[key]; !ok || c.Default == nil {
+			t.Errorf("1798/AC2: setting %s is missing or has no default", key)
+		}
+		if !strings.Contains(string(slice), property+"=${config:"+key+"}") {
+			t.Errorf("1798/AC2: the slice does not set %s from %s", property, key)
+		}
+	}
+	var placed, running bool
+	for _, r := range read(t).Resources {
+		placed = placed || (r["type"] == "file" && r["path"] == "/etc/systemd/system/mesh-build.slice")
+		running = running || (r["type"] == "service" && r["unit"] == "mesh-build.slice" && r["state"] == "running")
+	}
+	if !placed || !running {
+		t.Errorf("1798/AC2: the slice must be placed (%v) and running (%v)", placed, running)
+	}
+	env, err := os.ReadFile("files/build-agent.env")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(env), "MESH_BUILD_CGROUP_PARENT=mesh-build.slice") {
+		t.Error("1798/AC1: the agent is not told the slice's name")
+	}
+}

modules/build-agent/module.json

@@ -2,6 +2,34 @@
   "module": "build-agent",
   "version": "1",
   "slug": "agent",
+  "config": {
+    "cpu-weight": {
+      "type": "number",
+      "description": "The CPU weight of everything a build or check starts on this machine, from 1 to 10000 against 100 for an ordinary service or a login session: under load the builds get their share of that, on an idle machine they take what is free (novox/hq issue 1798). The default 20 is about a sixth of the CPU against one competing workload; a desktop may take 10.",
+      "minimum": 1,
+      "maximum": 10000,
+      "default": 20
+    },
+    "io-weight": {
+      "type": "number",
+      "description": "The disk IO weight of everything a build or check starts, from 1 to 10000 against 100 for an ordinary service: the default 10 lets the machine's own work go first (novox/hq issue 1798).",
+      "minimum": 1,
+      "maximum": 10000,
+      "default": 10
+    },
+    "memory-high": {
+      "type": "string",
+      "description": "The memory above which builds and checks are slowed and made to give memory back, a percentage of the machine's memory or a size (systemd's MemoryHigh): the default 50% leaves half the machine to what it runs (novox/hq issue 1798).",
+      "pattern": "([1-9][0-9]?|100)%|[1-9][0-9]*[KMGT]",
+      "default": "50%"
+    },
+    "memory-max": {
+      "type": "string",
+      "description": "The memory builds and checks together may never exceed, a percentage or a size (systemd's MemoryMax); a build past it is killed by the kernel and fails, so keep it above memory-high: the default 70% (novox/hq issue 1798).",
+      "pattern": "([1-9][0-9]?|100)%|[1-9][0-9]*[KMGT]",
+      "default": "70%"
+    }
+  },
   "capabilities": [
     "container-runtime"
   ],
@@ -9,7 +37,12 @@
     {
       "name": "node-build-agent",
       "scope": "node",
-      "serves": ["current", "kill", "pause", "resume"]
+      "serves": [
+        "current",
+        "kill",
+        "pause",
+        "resume"
+      ]
     }
   ],
   "reads": [
@@ -48,7 +81,7 @@
         "id": "judge-cache",
         "path": "${dir:judge-cache}",
         "class": "cache",
-        "why": "the Go cache every merge check's judge is built with and the group judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 §6)"
+        "why": "the Go cache every merge check's judge is built with and the group judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 \u00a76)"
       }
     ]
   },
@@ -69,6 +102,19 @@
       "type": "directory",
       "mode": "0700"
     },
+    {
+      "id": "build-slice",
+      "type": "file",
+      "path": "/etc/systemd/system/mesh-build.slice",
+      "mode": "0644",
+      "template": "files/mesh-build.slice"
+    },
+    {
+      "id": "build-slice-unit",
+      "type": "service",
+      "unit": "mesh-build.slice",
+      "state": "running"
+    },
     {
       "id": "agent-env",
       "type": "file",