Files
mesh-catalog/modules/mesh-build-node/cmd/mesh-build-node/room_test.go
T
jschoubben 3ca48ff230
mesh/delivery-group group fix/1839-build-floor checking: 1 of 2 member(s) ready
mesh/delivery superseded: a newer head of the same pull request
mesh/check failed: novox: the check ran past its bound of 30m0s, stuck at: test the unit level of novox/mesh-catalog#458 on novox | 3ca48ff23005 on 89…
Refuse build steps under a free-space floor and release workspaces, so the build modules stop filling a build node's disk (issue 1839)
mesh-build-go, mesh-build-node and mesh-build-image refuse every fetch, test
and build, and say unhealthy, while their work or cache filesystem has less
than 20 GiB free (build-agent's floor), so the builder chooses another node
instead of filling this one. Each serves build_<x>_release, removing a
workspace once its caller's last step answered; a failed fetch removes its
own; a workspace never released goes two hours after its last use (ADR 0509)
in place of a day, never while a step holds it. Pins mesh-sdk's
fix/1839-build-floor (buildstep.Floor, buildstep.Workspaces).
2026-10-23 12:17:58 +02:00

124 lines
4.3 KiB
Go

package main
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/forgekey"
"git.novox.be/novox/mesh-sdk/go/health"
)
const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB"
func underFloor(node string) buildstep.Floor {
f := floorOf(node)
f.Free = func(string) (uint64, error) { return 4 << 30, nil }
return f
}
// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB.
func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) {
if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" {
t.Errorf("floorOf = %+v, want ace and 20 GiB", f)
}
}
// 1839/AC1: under the floor every step tool is refused, naming the node, what is free and the floor, no step runs, and
// the module is unhealthy saying so, so the builder chooses another node.
func TestUnderTheFloorEveryStepIsRefusedAndTheModuleIsUnhealthy(t *testing.T) {
states := newStates()
m := moduleWith(states, "v26.10.0")
m.Work, m.Floor = t.TempDir(), underFloor("ace")
ran := false
run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil }
steps := []StepTool{{Name: "long_probe", Long: true, Run: run}, {Name: "short_probe", Run: run}}
for _, tool := range m.Tools(steps) {
switch tool.Name {
case "long_probe":
if _, err := tool.Run(map[string]any{}); err != nil {
t.Fatal(err)
}
if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) {
t.Errorf("the long step ended %s: %q, want refused: %q", st.State, st.Reason, underTheFloor)
}
case "short_probe":
if _, err := tool.Run(map[string]any{}); err == nil || !strings.HasPrefix(err.Error(), underTheFloor) {
t.Errorf("the short step answered %v, want %q", err, underTheFloor)
}
}
}
if ran {
t.Errorf("a step ran under the floor")
}
if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) {
t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor)
}
}
func releaseTool(t *testing.T, m *Module) func(map[string]any) (any, error) {
t.Helper()
for _, tool := range m.Tools(m.StepTools()) {
if tool.Name == ReleaseTool {
return tool.Run
}
}
t.Fatalf("no tool %s", ReleaseTool)
return nil
}
// 1839/AC2: build_node_release removes the workspace at once, even under the floor; a second release is no error.
func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) {
m, _, work := newMachine(t)
m.Floor = underFloor("ace")
ws := filepath.Join(work, "abc1")
os.MkdirAll(filepath.Join(ws, "tree"), 0o700)
for i := 0; i < 2; i++ {
if _, err := releaseTool(t, m)(map[string]any{"workspace": "abc1"}); err != nil {
t.Fatalf("release %d: %v", i, err)
}
}
if _, err := os.Stat(ws); !os.IsNotExist(err) {
t.Errorf("the workspace is still there after its release")
}
}
// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it.
func TestAFailedFetchLeavesNoWorkspace(t *testing.T) {
m, f := fetchMachine(t, typeScriptModule(`{}`), nil)
f.answer["git fetch"] = forgekey.Ran{Status: 128, Stderr: "fatal: unreachable"}
if _, err := m.fetch(context.Background(), fetchArgs); err == nil {
t.Fatal("the fetch passed, want it failed")
}
if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
t.Errorf("the failed fetch left %d entries in the work directory", len(entries))
}
}
// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is,
// also at the module's own round and not only at a fetch.
func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) {
m := &Module{Work: t.TempDir()}
for _, name := range []string{"held1", "lost1"} {
os.MkdirAll(filepath.Join(m.Work, name, "tree"), 0o755)
}
done := m.spaces().Hold("held1")
defer done()
when := time.Now().Add(-3 * time.Hour)
for _, name := range []string{"held1", "lost1"} {
os.Chtimes(filepath.Join(m.Work, name), when, when)
}
m.CleanWorkspaces(time.Now())
if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil {
t.Errorf("the held workspace was swept")
}
if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) {
t.Errorf("the unheld workspace three hours old was kept")
}
}