Refuse build steps under a free-space floor and release workspaces, so the build modules stop filling a build node's disk (issue 1839)

mesh-build-go, mesh-build-node and mesh-build-image refuse every fetch, test
and build, and say unhealthy, while their work or cache filesystem has less
than 20 GiB free (build-agent's floor), so the builder chooses another node
instead of filling this one. Each serves build_<x>_release, removing a
workspace once its caller's last step answered; a failed fetch removes its
own; a workspace never released goes two hours after its last use (ADR 0509)
in place of a day, never while a step holds it. Pins mesh-sdk's
fix/1839-build-floor (buildstep.Floor, buildstep.Workspaces).

modules/mesh-build-node/cmd/mesh-build-node/room_test.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/buildstep"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+	"git.novox.be/novox/mesh-sdk/go/forgekey"
+	"git.novox.be/novox/mesh-sdk/go/health"
+)
+
+const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB"
+
+func underFloor(node string) buildstep.Floor {
+	f := floorOf(node)
+	f.Free = func(string) (uint64, error) { return 4 << 30, nil }
+	return f
+}
+
+// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB.
+func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) {
+	if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" {
+		t.Errorf("floorOf = %+v, want ace and 20 GiB", f)
+	}
+}
+
+// 1839/AC1: under the floor every step tool is refused, naming the node, what is free and the floor, no step runs, and
+// the module is unhealthy saying so, so the builder chooses another node.
+func TestUnderTheFloorEveryStepIsRefusedAndTheModuleIsUnhealthy(t *testing.T) {
+	states := newStates()
+	m := moduleWith(states, "v26.10.0")
+	m.Work, m.Floor = t.TempDir(), underFloor("ace")
+	ran := false
+	run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil }
+	steps := []StepTool{{Name: "long_probe", Long: true, Run: run}, {Name: "short_probe", Run: run}}
+	for _, tool := range m.Tools(steps) {
+		switch tool.Name {
+		case "long_probe":
+			if _, err := tool.Run(map[string]any{}); err != nil {
+				t.Fatal(err)
+			}
+			if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) {
+				t.Errorf("the long step ended %s: %q, want refused: %q", st.State, st.Reason, underTheFloor)
+			}
+		case "short_probe":
+			if _, err := tool.Run(map[string]any{}); err == nil || !strings.HasPrefix(err.Error(), underTheFloor) {
+				t.Errorf("the short step answered %v, want %q", err, underTheFloor)
+			}
+		}
+	}
+	if ran {
+		t.Errorf("a step ran under the floor")
+	}
+	if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) {
+		t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor)
+	}
+}
+
+func releaseTool(t *testing.T, m *Module) func(map[string]any) (any, error) {
+	t.Helper()
+	for _, tool := range m.Tools(m.StepTools()) {
+		if tool.Name == ReleaseTool {
+			return tool.Run
+		}
+	}
+	t.Fatalf("no tool %s", ReleaseTool)
+	return nil
+}
+
+// 1839/AC2: build_node_release removes the workspace at once, even under the floor; a second release is no error.
+func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) {
+	m, _, work := newMachine(t)
+	m.Floor = underFloor("ace")
+	ws := filepath.Join(work, "abc1")
+	os.MkdirAll(filepath.Join(ws, "tree"), 0o700)
+	for i := 0; i < 2; i++ {
+		if _, err := releaseTool(t, m)(map[string]any{"workspace": "abc1"}); err != nil {
+			t.Fatalf("release %d: %v", i, err)
+		}
+	}
+	if _, err := os.Stat(ws); !os.IsNotExist(err) {
+		t.Errorf("the workspace is still there after its release")
+	}
+}
+
+// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it.
+func TestAFailedFetchLeavesNoWorkspace(t *testing.T) {
+	m, f := fetchMachine(t, typeScriptModule(`{}`), nil)
+	f.answer["git fetch"] = forgekey.Ran{Status: 128, Stderr: "fatal: unreachable"}
+	if _, err := m.fetch(context.Background(), fetchArgs); err == nil {
+		t.Fatal("the fetch passed, want it failed")
+	}
+	if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
+		t.Errorf("the failed fetch left %d entries in the work directory", len(entries))
+	}
+}
+
+// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is,
+// also at the module's own round and not only at a fetch.
+func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) {
+	m := &Module{Work: t.TempDir()}
+	for _, name := range []string{"held1", "lost1"} {
+		os.MkdirAll(filepath.Join(m.Work, name, "tree"), 0o755)
+	}
+	done := m.spaces().Hold("held1")
+	defer done()
+	when := time.Now().Add(-3 * time.Hour)
+	for _, name := range []string{"held1", "lost1"} {
+		os.Chtimes(filepath.Join(m.Work, name), when, when)
+	}
+	m.CleanWorkspaces(time.Now())
+	if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil {
+		t.Errorf("the held workspace was swept")
+	}
+	if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) {
+		t.Errorf("the unheld workspace three hours old was kept")
+	}
+}