Refuse build steps under a free-space floor and release workspaces, so the build modules stop filling a build node's disk (issue 1839)

mesh-build-go, mesh-build-node and mesh-build-image refuse every fetch, test
and build, and say unhealthy, while their work or cache filesystem has less
than 20 GiB free (build-agent's floor), so the builder chooses another node
instead of filling this one. Each serves build_<x>_release, removing a
workspace once its caller's last step answered; a failed fetch removes its
own; a workspace never released goes two hours after its last use (ADR 0509)
in place of a day, never while a step holds it. Pins mesh-sdk's
fix/1839-build-floor (buildstep.Floor, buildstep.Workspaces).

modules/mesh-build-node/README.md

@@ -57,7 +57,11 @@ toolchain install: a long step answers at once with its id and ends on `<node>.<
   removed). The mesh's registry reaches `npm ci` through a private config file (`--userconfig`), so
   other `@novox/` packages install. A module with packages and no lock does not fail the fetch: its result lists it
   under `refused` with the reason, and a build of that bundle fails naming it (issue 1459). Every command runs through `forgekey.ExecRunnerClean(0)` with PATH, HOME, the locale and its own
-  words, nothing of this process's environment. Workspaces older than a day are removed at each fetch.
+  words, nothing of this process's environment. A failed fetch removes its own workspace; `build_node_release {workspace}` removes one at once (no guard, no
+  floor; its caller calls it once its last step answered), and one no caller released is removed two hours after its
+  last use (ADR 0509), never while a step holds it. Every step is refused, `refused: <node> has N GiB free under the
+  floor of 20 GiB (on <dir>)`, and the health is unhealthy with that text, while the work or the toolchain cache
+  filesystem has less than 20 GiB free (issue 1839).
   The arguments are mesh-sdk's `buildstep.FetchRequest` (ADR 0499 §4): with `dir` and `paths` (from the builder, never
   from a manifest) the workspace is narrowed to those paths, the manifest and the lock files before anything else. A
   member is built against the SDK version its own lock names (the lead's decision L10 of R4e, ADR 0500 §5-§6): the

modules/mesh-build-node/cmd/mesh-build-node/fetch.go

@@ -33,9 +33,6 @@ const ForgeAlias = "mesh-forge"
 // sdkPackage is the package the toolchain carries and a module's own dependencies never supply.
 const sdkPackage = "@novox/mesh-sdk"
 
-// WorkspaceAge is how long a workspace is kept.
-const WorkspaceAge = 24 * time.Hour
-
 var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
 
 var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
@@ -105,20 +102,36 @@ func (m *Module) fetchTool() StepTool {
 	}
 }
 
-func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
+// fetch makes a workspace and fills it. The workspace is held while the fetch runs, and removed when the fetch fails,
+// since nobody learns its id to release it (issue 1839).
+func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
 	// What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its
 	// named paths, the workspace is narrowed to them (buildstep.Narrow).
 	req, err := buildstep.ParseFetchRequest(args)
 	if err != nil {
 		return nil, err
 	}
-	repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
 	group, err := groupPackageArg(args["package"])
 	if err != nil {
 		return nil, err
 	}
 	m.CleanWorkspaces(m.now())
 	id := buildstep.NewID()
+	done := m.spaces().Hold(id)
+	defer func() {
+		done()
+		if err != nil {
+			if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
+				m.Say("the failed fetch's workspace %s: %v", id, rerr)
+			}
+		}
+	}()
+	return m.fetchInto(ctx, req, group, id)
+}
+
+func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, group *groupPackage, id string) (any, error) {
+	repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
+	var err error
 	root := filepath.Join(m.Work, id)
 	tree := filepath.Join(root, "tree")
 	if err := os.MkdirAll(tree, 0o700); err != nil {
@@ -354,19 +367,8 @@ func removeSDKCopies(nm string) error {
 	return nil
 }
 
-// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
+// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety
+// net for a caller that never released one (issue 1839, ADR 0509).
 func (m *Module) CleanWorkspaces(now time.Time) {
-	entries, err := os.ReadDir(m.Work)
-	if err != nil {
-		return
-	}
-	for _, e := range entries {
-		info, err := e.Info()
-		if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
-			continue
-		}
-		if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
-			m.Say("the workspace %s could not be removed: %v", e.Name(), err)
-		}
-	}
+	m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
 }

modules/mesh-build-node/cmd/mesh-build-node/main.go

@@ -54,6 +54,7 @@ func main() {
 		Bubblewrap:          ProbeBubblewrap,
 		RegistryFile:        filepath.Join(home, "registry"),
 	}
+	m.Floor = floorOf(m.Node)
 	// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
 	if m.Node != "" {
 		go m.Start(ctx)

modules/mesh-build-node/cmd/mesh-build-node/module.go

@@ -59,8 +59,14 @@ type Module struct {
 	// unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing.
 	Bubblewrap func(ctx context.Context) (string, error)
 
-	mu      sync.Mutex // guards Recipe
-	install sync.Mutex // one install at a time
+	// Floor refuses every step while the work or toolchain cache filesystem has less free than it (issue 1839); main
+	// sets floorOf(node), and a zero Floor checks nothing.
+	Floor buildstep.Floor
+
+	spacesOnce sync.Once
+	workspaces *buildstep.Workspaces
+	mu         sync.Mutex // guards Recipe
+	install    sync.Mutex // one install at a time
 }
 
 func (m *Module) recipe() Recipe {
@@ -122,6 +128,7 @@ func (m *Module) Start(ctx context.Context) {
 		if err := m.Prepare(ctx); err != nil && m.Say != nil {
 			m.Say("the toolchain: %v", err)
 		}
+		m.CleanWorkspaces(m.now())
 		select {
 		case <-ctx.Done():
 			return
@@ -138,6 +145,9 @@ func (m *Module) Health(ctx context.Context) health.Answer {
 	if err != nil {
 		return health.Unhealthy(err.Error())
 	}
+	if err := m.room(); err != nil {
+		return health.Unhealthy(err.Error())
+	}
 	return health.Healthy(fmt.Sprintf("node %s installed, the version the module builds with (recipe %s)", installed, m.recipe().Version()))
 }
 
@@ -150,6 +160,10 @@ func (m *Module) Long(tool string, run func(ctx context.Context, args map[string
 			if err := m.Prepare(ctx); err != nil {
 				return nil, err
 			}
+			if err := m.room(); err != nil {
+				return nil, err
+			}
+			defer m.holdArg(args)()
 			return run(ctx, args)
 		})
 		if err != nil {
@@ -166,6 +180,10 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any,
 		if err := m.Prepare(m.Ctx); err != nil {
 			return nil, err
 		}
+		if err := m.room(); err != nil {
+			return nil, err
+		}
+		defer m.holdArg(args)()
 		return run(m.Ctx, args)
 	}
 }

modules/mesh-build-node/cmd/mesh-build-node/module_test.go

@@ -117,7 +117,7 @@ func TestANodeThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T)
 		{Name: "short_probe", Run: run},
 	}, (&Module{}).StepTools()...)
 	for _, tool := range m.Tools(steps) {
-		if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool {
+		if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool {
 			continue
 		}
 		var reason string

modules/mesh-build-node/cmd/mesh-build-node/room.go

@@ -0,0 +1,67 @@
+package main
+
+// A step's room to work and its workspace's end (novox/hq issue 1839, ADR 0509): every step is refused while the work
+// or toolchain cache filesystem is under the floor, a workspace is held while a step runs on it, removed when its caller releases
+// it, and swept unheld past buildstep.UnheldBound.
+
+import (
+	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+)
+
+// ReleaseTool removes a workspace whose caller's steps have ended.
+const ReleaseTool = "build_node_release"
+
+// floorOf is the floor the module keeps on node: build-agent's.
+func floorOf(node string) buildstep.Floor {
+	return buildstep.Floor{Node: node, Bytes: buildstep.FreeSpaceFloor}
+}
+
+// room refuses while the work or toolchain cache filesystem is under the floor.
+func (m *Module) room() error {
+	paths := []string{m.Work}
+	if m.Toolchain != nil {
+		paths = append(paths, m.Toolchain.Paths.Cache)
+	}
+	return m.Floor.Check(paths...)
+}
+
+// spaces are the module's workspaces under Work.
+func (m *Module) spaces() *buildstep.Workspaces {
+	m.spacesOnce.Do(func() { m.workspaces = &buildstep.Workspaces{Dir: m.Work} })
+	return m.workspaces
+}
+
+// holdArg holds the workspace a step's arguments name while the step runs; nothing when they name none.
+func (m *Module) holdArg(args map[string]any) func() {
+	ws, _ := args["workspace"].(string)
+	if ws == "" {
+		return func() {}
+	}
+	return m.spaces().Hold(ws)
+}
+
+// releaseTool is build_node_release: no guard and no floor, since it only frees room.
+func (m *Module) releaseTool() stdio.Tool {
+	return stdio.Tool{Name: ReleaseTool,
+		Description: "Remove a workspace whose steps have ended (fetch, test, build): its caller says so once its last step " +
+			"answered. A workspace already gone is no error; one a running step holds is refused. A workspace never released " +
+			"is removed two hours after its last use.",
+		Input: map[string]any{"type": "object", "properties": map[string]any{
+			"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch answered"},
+		}, "required": []string{"workspace"}},
+		Run: func(args map[string]any) (any, error) {
+			ws, _ := args["workspace"].(string)
+			if err := m.spaces().Release(ws); err != nil {
+				return nil, err
+			}
+			return map[string]any{"released": ws}, nil
+		}}
+}
+
+// sweepSaid is CleanWorkspaces' saying of a workspace a caller never released: a fault of the caller (ADR 0509).
+func (m *Module) sweepSaid(removed []string) {
+	if len(removed) > 0 && m.Say != nil {
+		m.Say("removed %d workspaces no caller released within %s: %v", len(removed), buildstep.UnheldBound, removed)
+	}
+}

modules/mesh-build-node/cmd/mesh-build-node/room_test.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/forgekey"
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB"
+
+func underFloor(node string) buildstep.Floor {
+	f := floorOf(node)
+	f.Free = func(string) (uint64, error) { return 4 << 30, nil }
+	return f
+}
+
+// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB.
+func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) {
+	if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" {
+		t.Errorf("floorOf = %+v, want ace and 20 GiB", f)
+	}
+}
+
+// 1839/AC1: under the floor every step tool is refused, naming the node, what is free and the floor, no step runs, and
+// the module is unhealthy saying so, so the builder chooses another node.
+func TestUnderTheFloorEveryStepIsRefusedAndTheModuleIsUnhealthy(t *testing.T) {
+	states := newStates()
+	m := moduleWith(states, "v26.10.0")
+	m.Work, m.Floor = t.TempDir(), underFloor("ace")
+	ran := false
+	run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil }
+	steps := []StepTool{{Name: "long_probe", Long: true, Run: run}, {Name: "short_probe", Run: run}}
+	for _, tool := range m.Tools(steps) {
+		switch tool.Name {
+		case "long_probe":
+			if _, err := tool.Run(map[string]any{}); err != nil {
+				t.Fatal(err)
+			}
+			if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) {
+				t.Errorf("the long step ended %s: %q, want refused: %q", st.State, st.Reason, underTheFloor)
+			}
+		case "short_probe":
+			if _, err := tool.Run(map[string]any{}); err == nil || !strings.HasPrefix(err.Error(), underTheFloor) {
+				t.Errorf("the short step answered %v, want %q", err, underTheFloor)
+			}
+		}
+	}
+	if ran {
+		t.Errorf("a step ran under the floor")
+	}
+	if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) {
+		t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor)
+	}
+}
+
+func releaseTool(t *testing.T, m *Module) func(map[string]any) (any, error) {
+	t.Helper()
+	for _, tool := range m.Tools(m.StepTools()) {
+		if tool.Name == ReleaseTool {
+			return tool.Run
+		}
+	}
+	t.Fatalf("no tool %s", ReleaseTool)
+	return nil
+}
+
+// 1839/AC2: build_node_release removes the workspace at once, even under the floor; a second release is no error.
+func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) {
+	m, _, work := newMachine(t)
+	m.Floor = underFloor("ace")
+	ws := filepath.Join(work, "abc1")
+	os.MkdirAll(filepath.Join(ws, "tree"), 0o700)
+	for i := 0; i < 2; i++ {
+		if _, err := releaseTool(t, m)(map[string]any{"workspace": "abc1"}); err != nil {
+			t.Fatalf("release %d: %v", i, err)
+		}
+	}
+	if _, err := os.Stat(ws); !os.IsNotExist(err) {
+		t.Errorf("the workspace is still there after its release")
+	}
+}
+
+// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it.
+func TestAFailedFetchLeavesNoWorkspace(t *testing.T) {
+	m, f := fetchMachine(t, typeScriptModule(`{}`), nil)
+	f.answer["git fetch"] = forgekey.Ran{Status: 128, Stderr: "fatal: unreachable"}
+	if _, err := m.fetch(context.Background(), fetchArgs); err == nil {
+		t.Fatal("the fetch passed, want it failed")
+	}
+	if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
+		t.Errorf("the failed fetch left %d entries in the work directory", len(entries))
+	}
+}
+
+// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is,
+// also at the module's own round and not only at a fetch.
+func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) {
+	m := &Module{Work: t.TempDir()}
+	for _, name := range []string{"held1", "lost1"} {
+		os.MkdirAll(filepath.Join(m.Work, name, "tree"), 0o755)
+	}
+	done := m.spaces().Hold("held1")
+	defer done()
+	when := time.Now().Add(-3 * time.Hour)
+	for _, name := range []string{"held1", "lost1"} {
+		os.Chtimes(filepath.Join(m.Work, name), when, when)
+	}
+	m.CleanWorkspaces(time.Now())
+	if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil {
+		t.Errorf("the held workspace was swept")
+	}
+	if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) {
+		t.Errorf("the unheld workspace three hours old was kept")
+	}
+}

modules/mesh-build-node/cmd/mesh-build-node/tools.go

@@ -40,6 +40,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
 		tools = append(tools, t)
 	}
 	tools = append(tools, m.ReadTools()...)
+	tools = append(tools, m.releaseTool())
 	return append(tools,
 		stdio.Tool{Name: StatusTool,
 			Description: "What the Node build module is on this machine: the Node version its recipe builds with and the one installed, " +

modules/mesh-build-node/go.mod

@@ -3,7 +3,7 @@ module meshbuildnode
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467
 	github.com/distribution/distribution/v3 v3.1.2
 	github.com/nats-io/nats.go v1.54.0
 )

modules/mesh-build-node/go.sum

@@ -1,5 +1,7 @@
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
 github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=

modules/mesh-build-node/module.json

@@ -16,6 +16,7 @@
     "build_node_fetch",
     "build_node_test",
     "build_node_build",
+    "build_node_release",
     "build_node_step",
     "build_node_steps",
     "build_node_toolchains",