Merge pull request 'The build modules refuse every step and report unhealthy under a 20 GiB free-space floor, release workspaces, and sweep after 2 h (issues 1839 and 1894, ADR 0532)' (#458) from fix/1839-build-floor into main

Reviewed-on: #458

modules/mesh-build-go/README.md

@@ -37,8 +37,17 @@ Three read tools answer what the module stored in its own states, without the gu
 forge-account provision's ssh name, so the account's own ssh files and git rewrite are used) into a workspace of the
 work directory, then runs `go mod download` and `go mod verify` in every directory with a `go.mod` that does not vendor,
 with `GOTOOLCHAIN=local`, `GOFLAGS=-mod=mod` and the module's cache directory. Every command runs with an
-environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes. Workspaces older than a day
-are removed.
+environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes.
+
+## Room to work and the workspace's end
+
+Every step is refused, ending `refused` with `refused: <node> has N GiB free under the floor of 20 GiB (on <dir>)`, while
+the work or the cache directory's filesystem has less than 20 GiB free (build-agent's floor, issue 1839); the health is
+then unhealthy with the same text, so the builder chooses another node and the controller raises the condition. A
+workspace is held while a step runs on it. `build_go_release {workspace}` removes it at once (no guard, no floor; one
+already gone is no error, one a running step holds is refused); its caller calls it once its last step answered. A
+failed fetch removes its own workspace. A workspace no caller released is removed two hours after its last use (ADR
+0509), and the removal is said in the log as the caller's fault.
 
 A pin on a pull request's head (issue 1148): when `go mod download` fails on a `git.novox.be/novox/*` pseudo-version, the
 fetch looks for that commit among the forge's `refs/pull/*/head`, places the repository at it under the workspace's

modules/mesh-build-go/cmd/mesh-build-go/fetch.go

@@ -23,11 +23,6 @@ var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
 // remote of a fetch is ssh://<it>/<owner>/<repository>.git.
 const ForgeAlias = "mesh-forge"
 
-// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it
-// fetches and every round it checks its toolchain. A workspace lies in the module's work directory, never in its
-// home (ADR 0466 §1).
-const WorkspaceAge = 24 * time.Hour
-
 var (
 	repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
 	workspaceOK  = regexp.MustCompile(`^[a-z0-9]+$`)
@@ -82,20 +77,37 @@ func (m *Module) fetchTool() StepTool {
 	}
 }
 
-func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
+// fetch makes a workspace and fills it. A workspace lies in the module's work directory, never in its home (ADR 0466
+// §1); it is held while the fetch runs, and removed when the fetch fails, since nobody learns its id to release it
+// (issue 1839).
+func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
 	// What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its
 	// named paths, the workspace is narrowed to them (buildstep.Narrow).
 	req, err := buildstep.ParseFetchRequest(args)
 	if err != nil {
 		return nil, err
 	}
-	repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
 	siblings, err := siblingsArg(args["beside"])
 	if err != nil {
 		return nil, err
 	}
 	m.CleanWorkspaces(m.now())
 	id := buildstep.NewID()
+	done := m.spaces().Hold(id)
+	defer func() {
+		done()
+		if err != nil {
+			if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
+				m.Say("the failed fetch's workspace %s: %v", id, rerr)
+			}
+		}
+	}()
+	return m.fetchInto(ctx, req, id, siblings)
+}
+
+func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id string, siblings []siblingArg) (any, error) {
+	repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
+	var err error
 	root := filepath.Join(m.Work, id)
 	tree := filepath.Join(root, "tree")
 	if err := os.MkdirAll(tree, 0o700); err != nil {
@@ -236,21 +248,10 @@ func (m *Module) goEnvFor(dir string) []string {
 	return env
 }
 
-// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
+// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety
+// net for a caller that never released one (issue 1839, ADR 0509).
 func (m *Module) CleanWorkspaces(now time.Time) {
-	entries, err := os.ReadDir(m.Work)
-	if err != nil {
-		return
-	}
-	for _, e := range entries {
-		info, err := e.Info()
-		if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
-			continue
-		}
-		if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
-			m.Say("the workspace %s could not be removed: %v", e.Name(), err)
-		}
-	}
+	m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
 }
 
 // clock is the time now: Now when a test set it, else the wall clock.

modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go

@@ -265,10 +265,11 @@ func TestARepositoryThatIsNotOwnerSlashNameIsRefusedBeforeAnythingRuns(t *testin
 	}
 }
 
-// Workspaces older than a day are removed; younger ones, and anything that is not a workspace, stay.
-func TestWorkspacesOlderThanADayAreRemoved(t *testing.T) {
+// Unheld workspaces last used more than two hours ago are removed (issue 1839, ADR 0509); younger ones, and anything
+// that is not a workspace, stay.
+func TestUnheldWorkspacesPastTwoHoursAreRemoved(t *testing.T) {
 	m := &Module{Work: t.TempDir()}
-	for name, age := range map[string]time.Duration{"old1": 25 * time.Hour, "young": time.Hour, "Keep-This": 48 * time.Hour} {
+	for name, age := range map[string]time.Duration{"old1": 3 * time.Hour, "young": time.Hour, "Keep-This": 48 * time.Hour} {
 		dir := filepath.Join(m.Work, name)
 		os.MkdirAll(filepath.Join(dir, "tree"), 0o755)
 		when := time.Now().Add(-age)

modules/mesh-build-go/cmd/mesh-build-go/main.go

@@ -46,6 +46,7 @@ func main() {
 		Work:                WorkDir,
 		Cache:               CacheDir,
 	}
+	m.Floor = floorOf(m.Node)
 	if u, err := user.Current(); err == nil {
 		m.Home = u.HomeDir
 	}

modules/mesh-build-go/cmd/mesh-build-go/module.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"sync"
 	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/buildstep"
@@ -68,6 +69,12 @@ type Module struct {
 	// StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own
 	// program, StartBus in bus.go, when nil).
 	StartBus func(ctx context.Context, dir string) (url string, stop func(), err error)
+	// Floor refuses every step while the work or cache filesystem has less free than it (issue 1839); main sets
+	// floorOf(node), and a zero Floor checks nothing.
+	Floor buildstep.Floor
+
+	spacesOnce sync.Once
+	workspaces *buildstep.Workspaces
 }
 
 // guard is the SDK guard for the version the probe just read: the module names no Go version (ADR 0477 §2), so the
@@ -116,6 +123,9 @@ func (m *Module) Health(ctx context.Context) health.Answer {
 	if err := m.checkBubblewrap(ctx); err != nil {
 		return health.Unhealthy(err.Error())
 	}
+	if err := m.room(); err != nil {
+		return health.Unhealthy(err.Error())
+	}
 	return health.Healthy(fmt.Sprintf("%s installed, the version the module builds with (recipe %s)", installed, m.Recipe.Version()))
 }
 
@@ -128,6 +138,10 @@ func (m *Module) Long(tool string, run func(ctx context.Context, args map[string
 			if err := m.Check(ctx); err != nil {
 				return nil, err
 			}
+			if err := m.room(); err != nil {
+				return nil, err
+			}
+			defer m.holdArg(args)()
 			return run(ctx, args)
 		})
 		if err != nil {
@@ -144,6 +158,10 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any,
 		if err := m.Check(m.Ctx); err != nil {
 			return nil, err
 		}
+		if err := m.room(); err != nil {
+			return nil, err
+		}
+		defer m.holdArg(args)()
 		return run(m.Ctx, args)
 	}
 }

modules/mesh-build-go/cmd/mesh-build-go/module_test.go

@@ -115,7 +115,7 @@ func TestAGoThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) {
 		{Name: "short_probe", Run: run},
 	}, m.StepTools()...)
 	for _, tool := range m.Tools(steps) {
-		if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool {
+		if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool {
 			continue
 		}
 		var reason string

modules/mesh-build-go/cmd/mesh-build-go/room.go

@@ -0,0 +1,61 @@
+package main
+
+// A step's room to work and its workspace's end (novox/hq issue 1839, ADR 0509): every step is refused while the work
+// or cache filesystem is under the floor, a workspace is held while a step runs on it, removed when its caller releases
+// it, and swept unheld past buildstep.UnheldBound.
+
+import (
+	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+)
+
+// ReleaseTool removes a workspace whose caller's steps have ended.
+const ReleaseTool = "build_go_release"
+
+// floorOf is the floor the module keeps on node: build-agent's.
+func floorOf(node string) buildstep.Floor {
+	return buildstep.Floor{Node: node, Bytes: buildstep.FreeSpaceFloor}
+}
+
+// room refuses while the work or cache filesystem is under the floor.
+func (m *Module) room() error { return m.Floor.Check(m.Work, m.Cache) }
+
+// spaces are the module's workspaces under Work.
+func (m *Module) spaces() *buildstep.Workspaces {
+	m.spacesOnce.Do(func() { m.workspaces = &buildstep.Workspaces{Dir: m.Work} })
+	return m.workspaces
+}
+
+// holdArg holds the workspace a step's arguments name while the step runs; nothing when they name none.
+func (m *Module) holdArg(args map[string]any) func() {
+	ws, _ := args["workspace"].(string)
+	if ws == "" {
+		return func() {}
+	}
+	return m.spaces().Hold(ws)
+}
+
+// releaseTool is build_go_release: no guard and no floor, since it only frees room.
+func (m *Module) releaseTool() stdio.Tool {
+	return stdio.Tool{Name: ReleaseTool,
+		Description: "Remove a workspace whose steps have ended (fetch, test, build): its caller says so once its last step " +
+			"answered. A workspace already gone is no error; one a running step holds is refused. A workspace never released " +
+			"is removed two hours after its last use.",
+		Input: map[string]any{"type": "object", "properties": map[string]any{
+			"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch answered"},
+		}, "required": []string{"workspace"}},
+		Run: func(args map[string]any) (any, error) {
+			ws, _ := args["workspace"].(string)
+			if err := m.spaces().Release(ws); err != nil {
+				return nil, err
+			}
+			return map[string]any{"released": ws}, nil
+		}}
+}
+
+// sweepSaid is CleanWorkspaces' saying of a workspace a caller never released: a fault of the caller (ADR 0509).
+func (m *Module) sweepSaid(removed []string) {
+	if len(removed) > 0 && m.Say != nil {
+		m.Say("removed %d workspaces no caller released within %s: %v", len(removed), buildstep.UnheldBound, removed)
+	}
+}

modules/mesh-build-go/cmd/mesh-build-go/room_test.go

@@ -0,0 +1,126 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/forgekey"
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB"
+
+func underFloor(node string) buildstep.Floor {
+	f := floorOf(node)
+	f.Free = func(string) (uint64, error) { return 4 << 30, nil }
+	return f
+}
+
+// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB.
+func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) {
+	if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" {
+		t.Errorf("floorOf = %+v, want ace and 20 GiB", f)
+	}
+}
+
+// 1839/AC1: under the floor every step tool is refused, naming the node, what is free and the floor, no step runs, and
+// the module is unhealthy saying so, so the builder chooses another node.
+func TestUnderTheFloorEveryStepIsRefusedAndTheModuleIsUnhealthy(t *testing.T) {
+	states := newStates()
+	m := moduleWith(states, "go1.27.1")
+	m.Work, m.Cache, m.Floor = t.TempDir(), t.TempDir(), underFloor("ace")
+	ran := false
+	run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil }
+	steps := []StepTool{{Name: "long_probe", Long: true, Run: run}, {Name: "short_probe", Run: run}}
+	for _, tool := range m.Tools(steps) {
+		switch tool.Name {
+		case "long_probe":
+			if _, err := tool.Run(map[string]any{}); err != nil {
+				t.Fatal(err)
+			}
+			if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) {
+				t.Errorf("the long step ended %s: %q, want refused: %q", st.State, st.Reason, underTheFloor)
+			}
+		case "short_probe":
+			if _, err := tool.Run(map[string]any{}); err == nil || !strings.HasPrefix(err.Error(), underTheFloor) {
+				t.Errorf("the short step answered %v, want %q", err, underTheFloor)
+			}
+		}
+	}
+	if ran {
+		t.Errorf("a step ran under the floor")
+	}
+	if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) {
+		t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor)
+	}
+}
+
+func releaseTool(t *testing.T, m *Module) func(map[string]any) (any, error) {
+	t.Helper()
+	for _, tool := range m.Tools(m.StepTools()) {
+		if tool.Name == ReleaseTool {
+			return tool.Run
+		}
+	}
+	t.Fatalf("no tool %s", ReleaseTool)
+	return nil
+}
+
+// 1839/AC2: build_go_release removes the workspace at once, even under the floor; a second release is no error.
+func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) {
+	m, _ := fetchModule(t, &fakeStep{})
+	m.Floor = underFloor("ace")
+	ws := filepath.Join(m.Work, "abc1")
+	os.MkdirAll(filepath.Join(ws, "tree"), 0o700)
+	for i := 0; i < 2; i++ {
+		if _, err := releaseTool(t, m)(map[string]any{"workspace": "abc1"}); err != nil {
+			t.Fatalf("release %d: %v", i, err)
+		}
+	}
+	if _, err := os.Stat(ws); !os.IsNotExist(err) {
+		t.Errorf("the workspace is still there after its release")
+	}
+}
+
+// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it.
+func TestAFailedFetchLeavesNoWorkspace(t *testing.T) {
+	f := &fakeStep{gomods: "go.mod\n", goRule: func(string) forgekey.Ran { return forgekey.Ran{Status: 1, Stderr: "boom"} }}
+	m, states := fetchModule(t, f)
+	if _, err := fetchTool(t, m)(fetchArgs); err != nil {
+		t.Fatal(err)
+	}
+	if st := states.waitEnd(t); st.State != delivery.StepFailed {
+		t.Fatalf("the step ended %s, want failed", st.State)
+	}
+	if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
+		t.Errorf("the failed fetch left %d entries in the work directory", len(entries))
+	}
+}
+
+// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is.
+func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) {
+	m := &Module{Work: t.TempDir()}
+	for _, name := range []string{"held1", "lost1"} {
+		dir := filepath.Join(m.Work, name)
+		os.MkdirAll(filepath.Join(dir, "tree"), 0o755)
+	}
+	done := m.spaces().Hold("held1")
+	defer done()
+	when := time.Now().Add(-3 * time.Hour)
+	for _, name := range []string{"held1", "lost1"} {
+		os.Chtimes(filepath.Join(m.Work, name), when, when)
+	}
+	m.CleanWorkspaces(time.Now())
+	if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil {
+		t.Errorf("the held workspace was swept")
+	}
+	if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) {
+		t.Errorf("the unheld workspace three hours old was kept")
+	}
+}

modules/mesh-build-go/cmd/mesh-build-go/tools.go

@@ -40,6 +40,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
 		tools = append(tools, t)
 	}
 	tools = append(tools, m.ReadTools()...)
+	tools = append(tools, m.releaseTool())
 	return append(tools,
 		stdio.Tool{Name: StatusTool,
 			Description: "What the Go build module is on this machine: the Go version its recipe builds with and the one installed, " +

modules/mesh-build-go/go.mod

@@ -3,7 +3,7 @@ module meshbuildgo
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467
 	github.com/distribution/distribution/v3 v3.1.2
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0

modules/mesh-build-go/go.sum

@@ -1,5 +1,7 @@
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
 github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=

modules/mesh-build-go/module.json

@@ -17,6 +17,7 @@
     "build_go_test",
     "build_go_digest",
     "build_go_build",
+    "build_go_release",
     "build_go_step",
     "build_go_steps",
     "build_go_toolchains",

modules/mesh-build-image/README.md

@@ -37,6 +37,12 @@ The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can re
 - It refuses, naming why, an image whose build context is a whole repository (`context`) or whose `build.on` names
   another module's artifact (`unsupportedImage` in `build.go`): those forms belong to the images build-agent builds
   until it is removed at R4g.
+- `build_image_release {workspace}` removes a workspace at once (no guard, no floor; one already gone is no error, one a
+  running step holds is refused); its caller calls it once its last step answered. A failed fetch removes its own
+  workspace, and one no caller released is removed two hours after its last use (ADR 0509).
+- A fetch and a build are refused, `refused: <node> has N GiB free under the floor of 20 GiB (on <dir>)`, and the health
+  is unhealthy with that text, while the work directory's or the image store's filesystem has less than 20 GiB free
+  (issue 1839). A copy writes nothing to the disk and is not refused.
 - `build_image_step`, `build_image_steps`, `build_image_status` and `build_image_health` read.
 
 The registry's address is placed into `<home>/registry` as `${seat:mesh-artifact-store:reach}` (host:port): the provisioner's client fills a moved module's placement, and the artifact store has not moved, so a `${bound:artifact-store:...}` is refused at composition (issue 1703).

modules/mesh-build-image/cmd/mesh-build-image/fetch.go

@@ -26,11 +26,6 @@ var (
 // remote of a fetch is ssh://<it>/<owner>/<repository>.git.
 const ForgeAlias = "mesh-forge"
 
-// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it
-// fetches and every round it checks itself. A workspace lies in the module's work directory, never in its home
-// (ADR 0466 §1).
-const WorkspaceAge = 24 * time.Hour
-
 // FetchResult is what a passed fetch step says: the workspace it made and the commit in it, and, only when the call
 // gave a base, the merge preview of ADR 0484 clause 3 (the trunk commit, the merge commit of the head onto it, the
 // files in conflict, the paths the merge changes and the tree of each module directory). A head that does not
@@ -66,7 +61,9 @@ func (m *Module) fetchTool() StepTool {
 	}
 }
 
-func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
+// fetch makes a workspace and fills it. The workspace is held while the fetch runs, and removed when the fetch fails,
+// since nobody learns its id to release it (issue 1839).
+func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
 	repository, _ := args["repository"].(string)
 	commit, _ := args["commit"].(string)
 	ref, _ := args["ref"].(string)
@@ -82,6 +79,19 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
 	}
 	m.CleanWorkspaces(m.clock())
 	id := buildstep.NewID()
+	done := m.spaces().Hold(id)
+	defer func() {
+		done()
+		if err != nil {
+			if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
+				m.Say("the failed fetch's workspace %s: %v", id, rerr)
+			}
+		}
+	}()
+	return m.fetchInto(ctx, repository, commit, ref, base, id)
+}
+
+func (m *Module) fetchInto(ctx context.Context, repository, commit, ref, base, id string) (any, error) {
 	tree := filepath.Join(m.Work, id, "tree")
 	if err := os.MkdirAll(tree, 0o700); err != nil {
 		return nil, err
@@ -124,21 +134,10 @@ func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base
 	return err
 }
 
-// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
+// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety
+// net for a caller that never released one (issue 1839, ADR 0509).
 func (m *Module) CleanWorkspaces(now time.Time) {
-	entries, err := os.ReadDir(m.Work)
-	if err != nil {
-		return
-	}
-	for _, e := range entries {
-		info, err := e.Info()
-		if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
-			continue
-		}
-		if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
-			m.Say("the workspace %s could not be removed: %v", e.Name(), err)
-		}
-	}
+	m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
 }
 
 // clock is the time now: Now when a test set it, else the wall clock.

modules/mesh-build-image/cmd/mesh-build-image/integration_test.go

@@ -53,6 +53,11 @@ func realModule(t *testing.T) (*Module, *keptStates) {
 	m := &Module{Node: "ace", Account: u.Username, UID: u.Uid, Home: home, Work: t.TempDir(), Store: t.TempDir(),
 		Steps: states.putStep, Ctx: context.Background(), Say: t.Logf, Run: StepRunner(home),
 		FetchOptions: []buildstep.Option{buildstep.AllowFileRemote()}}
+	// buildah being installed is not enough: inside the check's sandbox rootless buildah cannot make its user
+	// namespace, so the module's own guard decides, as it does before every step (issue 1894).
+	if err := m.Check(context.Background()); err != nil {
+		t.Skipf("NOT RUN: this machine cannot build images where the check runs (%v); proven live on a build node", err)
+	}
 	return m, states
 }
 

modules/mesh-build-image/cmd/mesh-build-image/main.go

@@ -37,6 +37,7 @@ func main() {
 		Work:      WorkDir,
 		Store:     StoreDir,
 	}
+	m.Floor = floorOf(m.Node)
 	if u, err := user.Current(); err == nil {
 		m.Home, m.Account, m.UID = u.HomeDir, u.Username, u.Uid
 	}

modules/mesh-build-image/cmd/mesh-build-image/module.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"sync"
 	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/buildstep"
@@ -45,6 +46,12 @@ type Module struct {
 	RegistryPause time.Duration
 	// Now is the clock; time.Now when nil.
 	Now func() time.Time
+	// Floor refuses every fetch and build while the work or the image store's filesystem has less free than it (issue
+	// 1839); main sets floorOf(node), and a zero Floor checks nothing.
+	Floor buildstep.Floor
+
+	spacesOnce sync.Once
+	workspaces *buildstep.Workspaces
 }
 
 // Look is what the module found out about its machine.
@@ -104,6 +111,9 @@ func (m *Module) Health(ctx context.Context) health.Answer {
 	if err != nil {
 		return health.Unhealthy(err.Error())
 	}
+	if err := m.room(); err != nil {
+		return health.Unhealthy(err.Error())
+	}
 	return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made", l.Buildah, l.Account, l.UIDs, l.GIDs))
 }
 
@@ -126,6 +136,10 @@ func (m *Module) Long(tool string, run func(ctx context.Context, args map[string
 			if err := m.Check(ctx); err != nil {
 				return nil, err
 			}
+			if err := m.room(); err != nil {
+				return nil, err
+			}
+			defer m.holdArg(args)()
 			return run(ctx, args)
 		})
 		if err != nil {
@@ -142,6 +156,10 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any,
 		if err := m.Check(m.Ctx); err != nil {
 			return nil, err
 		}
+		if err := m.room(); err != nil {
+			return nil, err
+		}
+		defer m.holdArg(args)()
 		return run(m.Ctx, args)
 	}
 }

modules/mesh-build-image/cmd/mesh-build-image/room.go

@@ -0,0 +1,61 @@
+package main
+
+// A step's room to work and its workspace's end (novox/hq issue 1839, ADR 0509): every fetch and build is refused while
+// the work or image store's filesystem is under the floor, a workspace is held while a step runs on it, removed when its caller releases
+// it, and swept unheld past buildstep.UnheldBound.
+
+import (
+	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+)
+
+// ReleaseTool removes a workspace whose caller's steps have ended.
+const ReleaseTool = "build_image_release"
+
+// floorOf is the floor the module keeps on node: build-agent's.
+func floorOf(node string) buildstep.Floor {
+	return buildstep.Floor{Node: node, Bytes: buildstep.FreeSpaceFloor}
+}
+
+// room refuses while the work or the image store's filesystem is under the floor.
+func (m *Module) room() error { return m.Floor.Check(m.Work, m.Store) }
+
+// spaces are the module's workspaces under Work.
+func (m *Module) spaces() *buildstep.Workspaces {
+	m.spacesOnce.Do(func() { m.workspaces = &buildstep.Workspaces{Dir: m.Work} })
+	return m.workspaces
+}
+
+// holdArg holds the workspace a step's arguments name while the step runs; nothing when they name none.
+func (m *Module) holdArg(args map[string]any) func() {
+	ws, _ := args["workspace"].(string)
+	if ws == "" {
+		return func() {}
+	}
+	return m.spaces().Hold(ws)
+}
+
+// releaseTool is build_image_release: no guard and no floor, since it only frees room.
+func (m *Module) releaseTool() stdio.Tool {
+	return stdio.Tool{Name: ReleaseTool,
+		Description: "Remove a workspace whose steps have ended (fetch, build): its caller says so once its last step " +
+			"answered. A workspace already gone is no error; one a running step holds is refused. A workspace never released " +
+			"is removed two hours after its last use.",
+		Input: map[string]any{"type": "object", "properties": map[string]any{
+			"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch answered"},
+		}, "required": []string{"workspace"}},
+		Run: func(args map[string]any) (any, error) {
+			ws, _ := args["workspace"].(string)
+			if err := m.spaces().Release(ws); err != nil {
+				return nil, err
+			}
+			return map[string]any{"released": ws}, nil
+		}}
+}
+
+// sweepSaid is CleanWorkspaces' saying of a workspace a caller never released: a fault of the caller (ADR 0509).
+func (m *Module) sweepSaid(removed []string) {
+	if len(removed) > 0 && m.Say != nil {
+		m.Say("removed %d workspaces no caller released within %s: %v", len(removed), buildstep.UnheldBound, removed)
+	}
+}

modules/mesh-build-image/cmd/mesh-build-image/room_test.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/forgekey"
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB"
+
+func underFloor(node string) buildstep.Floor {
+	f := floorOf(node)
+	f.Free = func(string) (uint64, error) { return 4 << 30, nil }
+	return f
+}
+
+// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB.
+func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) {
+	if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" {
+		t.Errorf("floorOf = %+v, want ace and 20 GiB", f)
+	}
+}
+
+// 1839/AC1: under the floor a fetch and a build are refused, naming the node, what is free and the floor, nothing runs,
+// and the module is unhealthy saying so, so the builder chooses another node.
+func TestUnderTheFloorAFetchAndABuildAreRefusedAndTheModuleIsUnhealthy(t *testing.T) {
+	f := newFake(t)
+	states := newStates()
+	m := moduleWith(t, f, states)
+	m.Floor = underFloor("ace")
+	for name, args := range map[string]map[string]any{
+		FetchTool: {"repository": "novox/mesh-catalog", "commit": commitSHA},
+		BuildTool: {"workspace": "abc1", "module": "modules/m", "artifact": "image"},
+	} {
+		if _, err := tool(t, m, name)(args); err != nil {
+			t.Fatalf("%s: %v", name, err)
+		}
+		if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) {
+			t.Errorf("%s ended %s: %q, want refused: %q", name, st.State, st.Reason, underTheFloor)
+		}
+	}
+	for _, l := range f.lines() {
+		if strings.HasPrefix(l, "git ") || strings.Contains(l, " bud ") {
+			t.Errorf("a refused step ran %q", l)
+		}
+	}
+	if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) {
+		t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor)
+	}
+}
+
+// 1839/AC2: build_image_release removes the workspace at once, even under the floor; a second release is no error.
+func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) {
+	m := moduleWith(t, newFake(t), newStates())
+	m.Floor = underFloor("ace")
+	ws := filepath.Join(m.Work, "abc1")
+	os.MkdirAll(filepath.Join(ws, "tree"), 0o700)
+	for i := 0; i < 2; i++ {
+		if _, err := tool(t, m, ReleaseTool)(map[string]any{"workspace": "abc1"}); err != nil {
+			t.Fatalf("release %d: %v", i, err)
+		}
+	}
+	if _, err := os.Stat(ws); !os.IsNotExist(err) {
+		t.Errorf("the workspace is still there after its release")
+	}
+}
+
+// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it.
+func TestAFailedFetchLeavesNoWorkspace(t *testing.T) {
+	f := newFake(t)
+	states := newStates()
+	m := moduleWith(t, f, states)
+	m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
+		if name == "git" && strings.Contains(strings.Join(args, " "), " fetch ") {
+			return forgekey.Ran{Status: 128, Stderr: "fatal: unreachable"}
+		}
+		return f.run(ctx, env, name, args...)
+	}
+	if _, err := tool(t, m, FetchTool)(map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA}); err != nil {
+		t.Fatal(err)
+	}
+	if st := states.waitEnd(t); st.State != delivery.StepFailed {
+		t.Fatalf("the step ended %s (%s), want failed", st.State, st.Reason)
+	}
+	if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
+		t.Errorf("the failed fetch left %d entries in the work directory", len(entries))
+	}
+}
+
+// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is.
+func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) {
+	m := &Module{Work: t.TempDir()}
+	for _, name := range []string{"held1", "lost1"} {
+		os.MkdirAll(filepath.Join(m.Work, name, "tree"), 0o755)
+	}
+	done := m.spaces().Hold("held1")
+	defer done()
+	when := time.Now().Add(-3 * time.Hour)
+	for _, name := range []string{"held1", "lost1"} {
+		os.Chtimes(filepath.Join(m.Work, name), when, when)
+	}
+	m.CleanWorkspaces(time.Now())
+	if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil {
+		t.Errorf("the held workspace was swept")
+	}
+	if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) {
+		t.Errorf("the unheld workspace three hours old was kept")
+	}
+}

modules/mesh-build-image/cmd/mesh-build-image/tools.go

@@ -44,6 +44,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
 		tools = append(tools, t)
 	}
 	tools = append(tools, m.ReadTools()...)
+	tools = append(tools, m.releaseTool())
 	return append(tools,
 		stdio.Tool{Name: StatusTool,
 			Description: "What the image build module is on this machine: the buildah installed, the account's subordinate " +

modules/mesh-build-image/go.mod

@@ -3,7 +3,7 @@ module meshbuildimage
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0
 )

modules/mesh-build-image/go.sum

@@ -1,5 +1,7 @@
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=

modules/mesh-build-image/module.json

@@ -15,6 +15,7 @@
     "build_image_fetch",
     "build_image_build",
     "build_image_copy",
+    "build_image_release",
     "build_image_step",
     "build_image_steps",
     "build_image_status",

modules/mesh-build-node/README.md

@@ -57,7 +57,11 @@ toolchain install: a long step answers at once with its id and ends on `<node>.<
   removed). The mesh's registry reaches `npm ci` through a private config file (`--userconfig`), so
   other `@novox/` packages install. A module with packages and no lock does not fail the fetch: its result lists it
   under `refused` with the reason, and a build of that bundle fails naming it (issue 1459). Every command runs through `forgekey.ExecRunnerClean(0)` with PATH, HOME, the locale and its own
-  words, nothing of this process's environment. Workspaces older than a day are removed at each fetch.
+  words, nothing of this process's environment. A failed fetch removes its own workspace; `build_node_release {workspace}` removes one at once (no guard, no
+  floor; its caller calls it once its last step answered), and one no caller released is removed two hours after its
+  last use (ADR 0509), never while a step holds it. Every step is refused, `refused: <node> has N GiB free under the
+  floor of 20 GiB (on <dir>)`, and the health is unhealthy with that text, while the work or the toolchain cache
+  filesystem has less than 20 GiB free (issue 1839).
   The arguments are mesh-sdk's `buildstep.FetchRequest` (ADR 0499 §4): with `dir` and `paths` (from the builder, never
   from a manifest) the workspace is narrowed to those paths, the manifest and the lock files before anything else. A
   member is built against the SDK version its own lock names (the lead's decision L10 of R4e, ADR 0500 §5-§6): the

modules/mesh-build-node/cmd/mesh-build-node/fetch.go

@@ -33,9 +33,6 @@ const ForgeAlias = "mesh-forge"
 // sdkPackage is the package the toolchain carries and a module's own dependencies never supply.
 const sdkPackage = "@novox/mesh-sdk"
 
-// WorkspaceAge is how long a workspace is kept.
-const WorkspaceAge = 24 * time.Hour
-
 var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
 
 var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
@@ -105,20 +102,36 @@ func (m *Module) fetchTool() StepTool {
 	}
 }
 
-func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
+// fetch makes a workspace and fills it. The workspace is held while the fetch runs, and removed when the fetch fails,
+// since nobody learns its id to release it (issue 1839).
+func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
 	// What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its
 	// named paths, the workspace is narrowed to them (buildstep.Narrow).
 	req, err := buildstep.ParseFetchRequest(args)
 	if err != nil {
 		return nil, err
 	}
-	repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
 	group, err := groupPackageArg(args["package"])
 	if err != nil {
 		return nil, err
 	}
 	m.CleanWorkspaces(m.now())
 	id := buildstep.NewID()
+	done := m.spaces().Hold(id)
+	defer func() {
+		done()
+		if err != nil {
+			if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
+				m.Say("the failed fetch's workspace %s: %v", id, rerr)
+			}
+		}
+	}()
+	return m.fetchInto(ctx, req, group, id)
+}
+
+func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, group *groupPackage, id string) (any, error) {
+	repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
+	var err error
 	root := filepath.Join(m.Work, id)
 	tree := filepath.Join(root, "tree")
 	if err := os.MkdirAll(tree, 0o700); err != nil {
@@ -354,19 +367,8 @@ func removeSDKCopies(nm string) error {
 	return nil
 }
 
-// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
+// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety
+// net for a caller that never released one (issue 1839, ADR 0509).
 func (m *Module) CleanWorkspaces(now time.Time) {
-	entries, err := os.ReadDir(m.Work)
-	if err != nil {
-		return
-	}
-	for _, e := range entries {
-		info, err := e.Info()
-		if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
-			continue
-		}
-		if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
-			m.Say("the workspace %s could not be removed: %v", e.Name(), err)
-		}
-	}
+	m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
 }

modules/mesh-build-node/cmd/mesh-build-node/main.go

@@ -54,6 +54,7 @@ func main() {
 		Bubblewrap:          ProbeBubblewrap,
 		RegistryFile:        filepath.Join(home, "registry"),
 	}
+	m.Floor = floorOf(m.Node)
 	// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
 	if m.Node != "" {
 		go m.Start(ctx)

modules/mesh-build-node/cmd/mesh-build-node/module.go

@@ -59,8 +59,14 @@ type Module struct {
 	// unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing.
 	Bubblewrap func(ctx context.Context) (string, error)
 
-	mu      sync.Mutex // guards Recipe
-	install sync.Mutex // one install at a time
+	// Floor refuses every step while the work or toolchain cache filesystem has less free than it (issue 1839); main
+	// sets floorOf(node), and a zero Floor checks nothing.
+	Floor buildstep.Floor
+
+	spacesOnce sync.Once
+	workspaces *buildstep.Workspaces
+	mu         sync.Mutex // guards Recipe
+	install    sync.Mutex // one install at a time
 }
 
 func (m *Module) recipe() Recipe {
@@ -122,6 +128,7 @@ func (m *Module) Start(ctx context.Context) {
 		if err := m.Prepare(ctx); err != nil && m.Say != nil {
 			m.Say("the toolchain: %v", err)
 		}
+		m.CleanWorkspaces(m.now())
 		select {
 		case <-ctx.Done():
 			return
@@ -138,6 +145,9 @@ func (m *Module) Health(ctx context.Context) health.Answer {
 	if err != nil {
 		return health.Unhealthy(err.Error())
 	}
+	if err := m.room(); err != nil {
+		return health.Unhealthy(err.Error())
+	}
 	return health.Healthy(fmt.Sprintf("node %s installed, the version the module builds with (recipe %s)", installed, m.recipe().Version()))
 }
 
@@ -150,6 +160,10 @@ func (m *Module) Long(tool string, run func(ctx context.Context, args map[string
 			if err := m.Prepare(ctx); err != nil {
 				return nil, err
 			}
+			if err := m.room(); err != nil {
+				return nil, err
+			}
+			defer m.holdArg(args)()
 			return run(ctx, args)
 		})
 		if err != nil {
@@ -166,6 +180,10 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any,
 		if err := m.Prepare(m.Ctx); err != nil {
 			return nil, err
 		}
+		if err := m.room(); err != nil {
+			return nil, err
+		}
+		defer m.holdArg(args)()
 		return run(m.Ctx, args)
 	}
 }

modules/mesh-build-node/cmd/mesh-build-node/module_test.go

@@ -117,7 +117,7 @@ func TestANodeThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T)
 		{Name: "short_probe", Run: run},
 	}, (&Module{}).StepTools()...)
 	for _, tool := range m.Tools(steps) {
-		if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool {
+		if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool {
 			continue
 		}
 		var reason string

modules/mesh-build-node/cmd/mesh-build-node/room.go

@@ -0,0 +1,67 @@
+package main
+
+// A step's room to work and its workspace's end (novox/hq issue 1839, ADR 0509): every step is refused while the work
+// or toolchain cache filesystem is under the floor, a workspace is held while a step runs on it, removed when its caller releases
+// it, and swept unheld past buildstep.UnheldBound.
+
+import (
+	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+)
+
+// ReleaseTool removes a workspace whose caller's steps have ended.
+const ReleaseTool = "build_node_release"
+
+// floorOf is the floor the module keeps on node: build-agent's.
+func floorOf(node string) buildstep.Floor {
+	return buildstep.Floor{Node: node, Bytes: buildstep.FreeSpaceFloor}
+}
+
+// room refuses while the work or toolchain cache filesystem is under the floor.
+func (m *Module) room() error {
+	paths := []string{m.Work}
+	if m.Toolchain != nil {
+		paths = append(paths, m.Toolchain.Paths.Cache)
+	}
+	return m.Floor.Check(paths...)
+}
+
+// spaces are the module's workspaces under Work.
+func (m *Module) spaces() *buildstep.Workspaces {
+	m.spacesOnce.Do(func() { m.workspaces = &buildstep.Workspaces{Dir: m.Work} })
+	return m.workspaces
+}
+
+// holdArg holds the workspace a step's arguments name while the step runs; nothing when they name none.
+func (m *Module) holdArg(args map[string]any) func() {
+	ws, _ := args["workspace"].(string)
+	if ws == "" {
+		return func() {}
+	}
+	return m.spaces().Hold(ws)
+}
+
+// releaseTool is build_node_release: no guard and no floor, since it only frees room.
+func (m *Module) releaseTool() stdio.Tool {
+	return stdio.Tool{Name: ReleaseTool,
+		Description: "Remove a workspace whose steps have ended (fetch, test, build): its caller says so once its last step " +
+			"answered. A workspace already gone is no error; one a running step holds is refused. A workspace never released " +
+			"is removed two hours after its last use.",
+		Input: map[string]any{"type": "object", "properties": map[string]any{
+			"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch answered"},
+		}, "required": []string{"workspace"}},
+		Run: func(args map[string]any) (any, error) {
+			ws, _ := args["workspace"].(string)
+			if err := m.spaces().Release(ws); err != nil {
+				return nil, err
+			}
+			return map[string]any{"released": ws}, nil
+		}}
+}
+
+// sweepSaid is CleanWorkspaces' saying of a workspace a caller never released: a fault of the caller (ADR 0509).
+func (m *Module) sweepSaid(removed []string) {
+	if len(removed) > 0 && m.Say != nil {
+		m.Say("removed %d workspaces no caller released within %s: %v", len(removed), buildstep.UnheldBound, removed)
+	}
+}

modules/mesh-build-node/cmd/mesh-build-node/room_test.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/forgekey"
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB"
+
+func underFloor(node string) buildstep.Floor {
+	f := floorOf(node)
+	f.Free = func(string) (uint64, error) { return 4 << 30, nil }
+	return f
+}
+
+// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB.
+func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) {
+	if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" {
+		t.Errorf("floorOf = %+v, want ace and 20 GiB", f)
+	}
+}
+
+// 1839/AC1: under the floor every step tool is refused, naming the node, what is free and the floor, no step runs, and
+// the module is unhealthy saying so, so the builder chooses another node.
+func TestUnderTheFloorEveryStepIsRefusedAndTheModuleIsUnhealthy(t *testing.T) {
+	states := newStates()
+	m := moduleWith(states, "v26.10.0")
+	m.Work, m.Floor = t.TempDir(), underFloor("ace")
+	ran := false
+	run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil }
+	steps := []StepTool{{Name: "long_probe", Long: true, Run: run}, {Name: "short_probe", Run: run}}
+	for _, tool := range m.Tools(steps) {
+		switch tool.Name {
+		case "long_probe":
+			if _, err := tool.Run(map[string]any{}); err != nil {
+				t.Fatal(err)
+			}
+			if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) {
+				t.Errorf("the long step ended %s: %q, want refused: %q", st.State, st.Reason, underTheFloor)
+			}
+		case "short_probe":
+			if _, err := tool.Run(map[string]any{}); err == nil || !strings.HasPrefix(err.Error(), underTheFloor) {
+				t.Errorf("the short step answered %v, want %q", err, underTheFloor)
+			}
+		}
+	}
+	if ran {
+		t.Errorf("a step ran under the floor")
+	}
+	if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) {
+		t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor)
+	}
+}
+
+func releaseTool(t *testing.T, m *Module) func(map[string]any) (any, error) {
+	t.Helper()
+	for _, tool := range m.Tools(m.StepTools()) {
+		if tool.Name == ReleaseTool {
+			return tool.Run
+		}
+	}
+	t.Fatalf("no tool %s", ReleaseTool)
+	return nil
+}
+
+// 1839/AC2: build_node_release removes the workspace at once, even under the floor; a second release is no error.
+func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) {
+	m, _, work := newMachine(t)
+	m.Floor = underFloor("ace")
+	ws := filepath.Join(work, "abc1")
+	os.MkdirAll(filepath.Join(ws, "tree"), 0o700)
+	for i := 0; i < 2; i++ {
+		if _, err := releaseTool(t, m)(map[string]any{"workspace": "abc1"}); err != nil {
+			t.Fatalf("release %d: %v", i, err)
+		}
+	}
+	if _, err := os.Stat(ws); !os.IsNotExist(err) {
+		t.Errorf("the workspace is still there after its release")
+	}
+}
+
+// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it.
+func TestAFailedFetchLeavesNoWorkspace(t *testing.T) {
+	m, f := fetchMachine(t, typeScriptModule(`{}`), nil)
+	f.answer["git fetch"] = forgekey.Ran{Status: 128, Stderr: "fatal: unreachable"}
+	if _, err := m.fetch(context.Background(), fetchArgs); err == nil {
+		t.Fatal("the fetch passed, want it failed")
+	}
+	if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
+		t.Errorf("the failed fetch left %d entries in the work directory", len(entries))
+	}
+}
+
+// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is,
+// also at the module's own round and not only at a fetch.
+func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) {
+	m := &Module{Work: t.TempDir()}
+	for _, name := range []string{"held1", "lost1"} {
+		os.MkdirAll(filepath.Join(m.Work, name, "tree"), 0o755)
+	}
+	done := m.spaces().Hold("held1")
+	defer done()
+	when := time.Now().Add(-3 * time.Hour)
+	for _, name := range []string{"held1", "lost1"} {
+		os.Chtimes(filepath.Join(m.Work, name), when, when)
+	}
+	m.CleanWorkspaces(time.Now())
+	if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil {
+		t.Errorf("the held workspace was swept")
+	}
+	if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) {
+		t.Errorf("the unheld workspace three hours old was kept")
+	}
+}

modules/mesh-build-node/cmd/mesh-build-node/tools.go

@@ -40,6 +40,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
 		tools = append(tools, t)
 	}
 	tools = append(tools, m.ReadTools()...)
+	tools = append(tools, m.releaseTool())
 	return append(tools,
 		stdio.Tool{Name: StatusTool,
 			Description: "What the Node build module is on this machine: the Node version its recipe builds with and the one installed, " +

modules/mesh-build-node/go.mod

@@ -3,7 +3,7 @@ module meshbuildnode
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467
 	github.com/distribution/distribution/v3 v3.1.2
 	github.com/nats-io/nats.go v1.54.0
 )

modules/mesh-build-node/go.sum

@@ -1,5 +1,7 @@
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
 github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=

modules/mesh-build-node/module.json

@@ -16,6 +16,7 @@
     "build_node_fetch",
     "build_node_test",
     "build_node_build",
+    "build_node_release",
     "build_node_step",
     "build_node_steps",
     "build_node_toolchains",