novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Build modules: a per-node processor cap, and each step's threads within it (issue 1806)

Browse Source 
The operator's desktop slows while it builds (shanks at 92%, the Go build
runtime at 1650% of 24 processors). The build modules declare cpu-cap
(default shanks=50,g14=50,ace=75), which the composer turns into the runtime's
quota at idle priority; each step's go, node and buildah commands get
GOMAXPROCS and UV_THREADPOOL_SIZE for their part of it, and the status tools
show the limits and what the kernel holds the runtime to.
This commit is contained in:
jschoubben
2026-10-26 13:41:05 +01:00
parent 879600202a
commit 64b9ded2db
26 changed files with 195 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-build-node/cmd/mesh-build-node/cpu.go
+39
View File
@@ -0,0 +1,39 @@
package main
import (
"git.novox.be/novox/mesh-sdk/go/buildcpu"
"git.novox.be/novox/mesh-sdk/go/buildstep"
)
// The processors a step may take (novox/hq issue 1806). The kernel holds the module's runtime to idle priority and to
// its node's share of the processors (the setting cpu-cap, written into its unit by the node-engine); inside that
// bound a step starts no more threads than its part of it, so the steps running at once do not wait in the kernel's
// throttle.
// cpuLimits is what one step is held to now: the cap from the settings file, the machine's processors and the node's
// capacity, read at every call like the capacity, so a change takes effect at the next step.
func (m *Module) cpuLimits(say func(string, ...any)) buildcpu.Limits {
slots := 1
switch {
case m.Capacity != nil:
slots = m.Capacity()
case m.Settings != "":
slots = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, nil)()
}
if slots < 1 {
slots = 1
}
return buildstep.NodeLimits(m.Settings, m.Node, slots, say)
}
// threadEnv is the environment every command of a step is given for its threads: GOMAXPROCS for esbuild and every Go
// program, UV_THREADPOOL_SIZE for Node's worker pool.
func (m *Module) threadEnv() []string {
return buildcpu.Env(m.cpuLimits(nil).Threads)
}
// cpuStatus is what the status tool says of it: the limits a step is given, and what the kernel holds this runtime
// to, read from its cgroup rather than assumed.
func (m *Module) cpuStatus() map[string]any {
return map[string]any{"limits": m.cpuLimits(m.Say), "in-force": buildcpu.ReadInForce()}
}
modules/mesh-build-node/cmd/mesh-build-node/fetch_test.go
+6
View File
@@ -4,6 +4,7 @@ import (
"context"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
@@ -245,6 +246,11 @@ func TestNoCommandOfTheStepSeesATokenOfTheProcess(t *testing.T) {
if len(c.env) < 3 || !strings.HasPrefix(c.env[0], "PATH=") || !strings.HasPrefix(c.env[1], "HOME=") || c.env[2] != "LC_ALL=C" {
t.Errorf("%s was given %v, want PATH, HOME and the locale first", c.name, c.env)
}
// And its threads, its part of the node's processor cap (issue 1806).
if !slices.ContainsFunc(c.env, func(e string) bool { return strings.HasPrefix(e, "GOMAXPROCS=") }) ||
!slices.ContainsFunc(c.env, func(e string) bool { return strings.HasPrefix(e, "UV_THREADPOOL_SIZE=") }) {
t.Errorf("%s was given %v, without its threads", c.name, c.env)
}
}
// The runner main wires: what it hands a command is exactly what the step gave.
ran := forgekey.ExecRunnerClean(10*time.Second)(context.Background(), append(m.baseEnv(), "ONLY=this"), "env")
modules/mesh-build-node/cmd/mesh-build-node/isolate.go
+1 -1
View File
@@ -35,7 +35,7 @@ func (m *Module) isolation(root string, write bool) buildstep.Isolation {
ReadOnly: []string{m.toolchainDir()},
Caches: []string{m.npmCache()},
WriteCaches: write,
Env: []string{"PATH=" + pathOrDefault(), "LC_ALL=C"},
Env: append([]string{"PATH=" + pathOrDefault(), "LC_ALL=C"}, m.threadEnv()...),
}
}
modules/mesh-build-node/cmd/mesh-build-node/module.go
+3 -2
View File
@@ -202,7 +202,8 @@ func (m *Module) Status(ctx context.Context) map[string]any {
err := m.Check(ctx)
installed, _ := m.Probe(ctx)
r := m.recipe()
out := map[string]any{"node": m.Node, "recipe": r.Version(), "installed": installed, "refusing": err != nil}
out := map[string]any{"node": m.Node, "recipe": r.Version(), "installed": installed, "refusing": err != nil,
"cpu": m.cpuStatus()}
if m.Toolchain != nil {
out["toolchain-dir"] = m.Toolchain.Paths.Dir(r.Version())
out["toolchain-installed"] = m.Toolchain.Present(r.Version())
@@ -227,7 +228,7 @@ const defaultPath = "/usr/local/bin:/usr/bin:/bin"
// files and git configuration) and a fixed locale. A command is given these and its own words, never this process's
// environment, which holds the module's bus credentials (ADR 0466 §4).
func (m *Module) baseEnv() []string {
return []string{"PATH=" + pathOrDefault(), "HOME=" + m.Toolchain.homeOrEmpty(), "LC_ALL=C"}
return append([]string{"PATH=" + pathOrDefault(), "HOME=" + m.Toolchain.homeOrEmpty(), "LC_ALL=C"}, m.threadEnv()...)
}
func pathOrDefault() string {
modules/mesh-build-node/files/settings.json
+2 -1
View File
@@ -1,3 +1,4 @@
{
"capacity": "shanks=3,ace=2,g14=0,novox=0"
"capacity": "shanks=3,ace=2,g14=0,novox=0",
"cpu-cap": "shanks=50,g14=50,ace=75"
}
modules/mesh-build-node/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module meshbuildnode
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026123700-04902ff24a0e
github.com/distribution/distribution/v3 v3.1.2
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-build-node/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4 h1:05sVel/UOvz4/g0dxrDOeVUL7BGakVrxPBlqRq02ZhM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026123700-04902ff24a0e h1:9Kwe61NVrT4NcNr6PZoupzsGuZCQ5WV+luqrxDV9frs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026123700-04902ff24a0e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
modules/mesh-build-node/module.json
+7 -1
View File
@@ -25,6 +25,12 @@
"description": "How many asks of the Node build seat this module runs at once on each node, as node=asks pairs separated by commas, such as shanks=3,ace=2 (novox/hq ADR 0525 §5); a node set to 0 pulls nothing, a node not named runs one at a time. The control-node and the laptop build nothing.",
"pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*",
"default": "shanks=3,ace=2,g14=0,novox=0"
},
"cpu-cap": {
"type": "string",
"description": "The most of each node's processors this module's builds may take, as node=percent pairs separated by commas, such as shanks=50,ace=75 (novox/hq issue 1806); a node not named takes 50. The node-engine holds the module's runtime to it as a CPU quota, at idle priority, and each build step starts no more threads than its part of it.",
"pattern": "|[a-z0-9_-]+=([1-9]|[1-9][0-9]|100)%?(,[a-z0-9_-]+=([1-9]|[1-9][0-9]|100)%?)*",
"default": "shanks=50,g14=50,ace=75"
}
},
"tools": [
@@ -99,7 +105,7 @@
"path": "${dir:state}/settings.json",
"mode": "0600",
"owner": "mesh-build-node",
"takes-config": ["capacity"],
"takes-config": ["capacity", "cpu-cap"],
"merge": "json",
"template": "files/settings.json"
},