Refuse build steps under a free-space floor and release workspaces, so the build modules stop filling a build node's disk (issue 1839)

mesh-build-go, mesh-build-node and mesh-build-image refuse every fetch, test
and build, and say unhealthy, while their work or cache filesystem has less
than 20 GiB free (build-agent's floor), so the builder chooses another node
instead of filling this one. Each serves build_<x>_release, removing a
workspace once its caller's last step answered; a failed fetch removes its
own; a workspace never released goes two hours after its last use (ADR 0509)
in place of a day, never while a step holds it. Pins mesh-sdk's
fix/1839-build-floor (buildstep.Floor, buildstep.Workspaces).

modules/mesh-build-image/cmd/mesh-build-image/fetch.go

@@ -26,11 +26,6 @@ var (
 // remote of a fetch is ssh://<it>/<owner>/<repository>.git.
 const ForgeAlias = "mesh-forge"
 
-// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it
-// fetches and every round it checks itself. A workspace lies in the module's work directory, never in its home
-// (ADR 0466 §1).
-const WorkspaceAge = 24 * time.Hour
-
 // FetchResult is what a passed fetch step says: the workspace it made and the commit in it, and, only when the call
 // gave a base, the merge preview of ADR 0484 clause 3 (the trunk commit, the merge commit of the head onto it, the
 // files in conflict, the paths the merge changes and the tree of each module directory). A head that does not
@@ -66,7 +61,9 @@ func (m *Module) fetchTool() StepTool {
 	}
 }
 
-func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
+// fetch makes a workspace and fills it. The workspace is held while the fetch runs, and removed when the fetch fails,
+// since nobody learns its id to release it (issue 1839).
+func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) {
 	repository, _ := args["repository"].(string)
 	commit, _ := args["commit"].(string)
 	ref, _ := args["ref"].(string)
@@ -82,6 +79,19 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
 	}
 	m.CleanWorkspaces(m.clock())
 	id := buildstep.NewID()
+	done := m.spaces().Hold(id)
+	defer func() {
+		done()
+		if err != nil {
+			if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil {
+				m.Say("the failed fetch's workspace %s: %v", id, rerr)
+			}
+		}
+	}()
+	return m.fetchInto(ctx, repository, commit, ref, base, id)
+}
+
+func (m *Module) fetchInto(ctx context.Context, repository, commit, ref, base, id string) (any, error) {
 	tree := filepath.Join(m.Work, id, "tree")
 	if err := os.MkdirAll(tree, 0o700); err != nil {
 		return nil, err
@@ -124,21 +134,10 @@ func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base
 	return err
 }
 
-// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
+// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety
+// net for a caller that never released one (issue 1839, ADR 0509).
 func (m *Module) CleanWorkspaces(now time.Time) {
-	entries, err := os.ReadDir(m.Work)
-	if err != nil {
-		return
-	}
-	for _, e := range entries {
-		info, err := e.Info()
-		if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
-			continue
-		}
-		if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
-			m.Say("the workspace %s could not be removed: %v", e.Name(), err)
-		}
-	}
+	m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
 }
 
 // clock is the time now: Now when a test set it, else the wall clock.

modules/mesh-build-image/cmd/mesh-build-image/main.go

@@ -37,6 +37,7 @@ func main() {
 		Work:      WorkDir,
 		Store:     StoreDir,
 	}
+	m.Floor = floorOf(m.Node)
 	if u, err := user.Current(); err == nil {
 		m.Home, m.Account, m.UID = u.HomeDir, u.Username, u.Uid
 	}

modules/mesh-build-image/cmd/mesh-build-image/module.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"sync"
 	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/buildstep"
@@ -45,6 +46,12 @@ type Module struct {
 	RegistryPause time.Duration
 	// Now is the clock; time.Now when nil.
 	Now func() time.Time
+	// Floor refuses every fetch and build while the work or the image store's filesystem has less free than it (issue
+	// 1839); main sets floorOf(node), and a zero Floor checks nothing.
+	Floor buildstep.Floor
+
+	spacesOnce sync.Once
+	workspaces *buildstep.Workspaces
 }
 
 // Look is what the module found out about its machine.
@@ -104,6 +111,9 @@ func (m *Module) Health(ctx context.Context) health.Answer {
 	if err != nil {
 		return health.Unhealthy(err.Error())
 	}
+	if err := m.room(); err != nil {
+		return health.Unhealthy(err.Error())
+	}
 	return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made", l.Buildah, l.Account, l.UIDs, l.GIDs))
 }
 
@@ -126,6 +136,10 @@ func (m *Module) Long(tool string, run func(ctx context.Context, args map[string
 			if err := m.Check(ctx); err != nil {
 				return nil, err
 			}
+			if err := m.room(); err != nil {
+				return nil, err
+			}
+			defer m.holdArg(args)()
 			return run(ctx, args)
 		})
 		if err != nil {
@@ -142,6 +156,10 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any,
 		if err := m.Check(m.Ctx); err != nil {
 			return nil, err
 		}
+		if err := m.room(); err != nil {
+			return nil, err
+		}
+		defer m.holdArg(args)()
 		return run(m.Ctx, args)
 	}
 }

modules/mesh-build-image/cmd/mesh-build-image/room.go

@@ -0,0 +1,61 @@
+package main
+
+// A step's room to work and its workspace's end (novox/hq issue 1839, ADR 0509): every fetch and build is refused while
+// the work or image store's filesystem is under the floor, a workspace is held while a step runs on it, removed when its caller releases
+// it, and swept unheld past buildstep.UnheldBound.
+
+import (
+	stdio "git.novox.be/novox/mesh-sdk/go"
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+)
+
+// ReleaseTool removes a workspace whose caller's steps have ended.
+const ReleaseTool = "build_image_release"
+
+// floorOf is the floor the module keeps on node: build-agent's.
+func floorOf(node string) buildstep.Floor {
+	return buildstep.Floor{Node: node, Bytes: buildstep.FreeSpaceFloor}
+}
+
+// room refuses while the work or the image store's filesystem is under the floor.
+func (m *Module) room() error { return m.Floor.Check(m.Work, m.Store) }
+
+// spaces are the module's workspaces under Work.
+func (m *Module) spaces() *buildstep.Workspaces {
+	m.spacesOnce.Do(func() { m.workspaces = &buildstep.Workspaces{Dir: m.Work} })
+	return m.workspaces
+}
+
+// holdArg holds the workspace a step's arguments name while the step runs; nothing when they name none.
+func (m *Module) holdArg(args map[string]any) func() {
+	ws, _ := args["workspace"].(string)
+	if ws == "" {
+		return func() {}
+	}
+	return m.spaces().Hold(ws)
+}
+
+// releaseTool is build_image_release: no guard and no floor, since it only frees room.
+func (m *Module) releaseTool() stdio.Tool {
+	return stdio.Tool{Name: ReleaseTool,
+		Description: "Remove a workspace whose steps have ended (fetch, build): its caller says so once its last step " +
+			"answered. A workspace already gone is no error; one a running step holds is refused. A workspace never released " +
+			"is removed two hours after its last use.",
+		Input: map[string]any{"type": "object", "properties": map[string]any{
+			"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch answered"},
+		}, "required": []string{"workspace"}},
+		Run: func(args map[string]any) (any, error) {
+			ws, _ := args["workspace"].(string)
+			if err := m.spaces().Release(ws); err != nil {
+				return nil, err
+			}
+			return map[string]any{"released": ws}, nil
+		}}
+}
+
+// sweepSaid is CleanWorkspaces' saying of a workspace a caller never released: a fault of the caller (ADR 0509).
+func (m *Module) sweepSaid(removed []string) {
+	if len(removed) > 0 && m.Say != nil {
+		m.Say("removed %d workspaces no caller released within %s: %v", len(removed), buildstep.UnheldBound, removed)
+	}
+}

modules/mesh-build-image/cmd/mesh-build-image/room_test.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/forgekey"
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB"
+
+func underFloor(node string) buildstep.Floor {
+	f := floorOf(node)
+	f.Free = func(string) (uint64, error) { return 4 << 30, nil }
+	return f
+}
+
+// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB.
+func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) {
+	if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" {
+		t.Errorf("floorOf = %+v, want ace and 20 GiB", f)
+	}
+}
+
+// 1839/AC1: under the floor a fetch and a build are refused, naming the node, what is free and the floor, nothing runs,
+// and the module is unhealthy saying so, so the builder chooses another node.
+func TestUnderTheFloorAFetchAndABuildAreRefusedAndTheModuleIsUnhealthy(t *testing.T) {
+	f := newFake(t)
+	states := newStates()
+	m := moduleWith(t, f, states)
+	m.Floor = underFloor("ace")
+	for name, args := range map[string]map[string]any{
+		FetchTool: {"repository": "novox/mesh-catalog", "commit": commitSHA},
+		BuildTool: {"workspace": "abc1", "module": "modules/m", "artifact": "image"},
+	} {
+		if _, err := tool(t, m, name)(args); err != nil {
+			t.Fatalf("%s: %v", name, err)
+		}
+		if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) {
+			t.Errorf("%s ended %s: %q, want refused: %q", name, st.State, st.Reason, underTheFloor)
+		}
+	}
+	for _, l := range f.lines() {
+		if strings.HasPrefix(l, "git ") || strings.Contains(l, " bud ") {
+			t.Errorf("a refused step ran %q", l)
+		}
+	}
+	if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) {
+		t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor)
+	}
+}
+
+// 1839/AC2: build_image_release removes the workspace at once, even under the floor; a second release is no error.
+func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) {
+	m := moduleWith(t, newFake(t), newStates())
+	m.Floor = underFloor("ace")
+	ws := filepath.Join(m.Work, "abc1")
+	os.MkdirAll(filepath.Join(ws, "tree"), 0o700)
+	for i := 0; i < 2; i++ {
+		if _, err := tool(t, m, ReleaseTool)(map[string]any{"workspace": "abc1"}); err != nil {
+			t.Fatalf("release %d: %v", i, err)
+		}
+	}
+	if _, err := os.Stat(ws); !os.IsNotExist(err) {
+		t.Errorf("the workspace is still there after its release")
+	}
+}
+
+// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it.
+func TestAFailedFetchLeavesNoWorkspace(t *testing.T) {
+	f := newFake(t)
+	states := newStates()
+	m := moduleWith(t, f, states)
+	m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
+		if name == "git" && strings.Contains(strings.Join(args, " "), " fetch ") {
+			return forgekey.Ran{Status: 128, Stderr: "fatal: unreachable"}
+		}
+		return f.run(ctx, env, name, args...)
+	}
+	if _, err := tool(t, m, FetchTool)(map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA}); err != nil {
+		t.Fatal(err)
+	}
+	if st := states.waitEnd(t); st.State != delivery.StepFailed {
+		t.Fatalf("the step ended %s (%s), want failed", st.State, st.Reason)
+	}
+	if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
+		t.Errorf("the failed fetch left %d entries in the work directory", len(entries))
+	}
+}
+
+// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is.
+func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) {
+	m := &Module{Work: t.TempDir()}
+	for _, name := range []string{"held1", "lost1"} {
+		os.MkdirAll(filepath.Join(m.Work, name, "tree"), 0o755)
+	}
+	done := m.spaces().Hold("held1")
+	defer done()
+	when := time.Now().Add(-3 * time.Hour)
+	for _, name := range []string{"held1", "lost1"} {
+		os.Chtimes(filepath.Join(m.Work, name), when, when)
+	}
+	m.CleanWorkspaces(time.Now())
+	if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil {
+		t.Errorf("the held workspace was swept")
+	}
+	if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) {
+		t.Errorf("the unheld workspace three hours old was kept")
+	}
+}

modules/mesh-build-image/cmd/mesh-build-image/tools.go

@@ -44,6 +44,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
 		tools = append(tools, t)
 	}
 	tools = append(tools, m.ReadTools()...)
+	tools = append(tools, m.releaseTool())
 	return append(tools,
 		stdio.Tool{Name: StatusTool,
 			Description: "What the image build module is on this machine: the buildah installed, the account's subordinate " +