mesh/delivery-group group fix/1839-build-floor checking: 1 of 2 member(s) ready
mesh/delivery superseded: a newer head of the same pull request
mesh/check failed: novox: the check ran past its bound of 30m0s, stuck at: test the unit level of novox/mesh-catalog#458 on novox | 3ca48ff23005 on 89…
mesh-build-go, mesh-build-node and mesh-build-image refuse every fetch, test and build, and say unhealthy, while their work or cache filesystem has less than 20 GiB free (build-agent's floor), so the builder chooses another node instead of filling this one. Each serves build_<x>_release, removing a workspace once its caller's last step answered; a failed fetch removes its own; a workspace never released goes two hours after its last use (ADR 0509) in place of a day, never while a step holds it. Pins mesh-sdk's fix/1839-build-floor (buildstep.Floor, buildstep.Workspaces).
118 lines
4.3 KiB
Go
118 lines
4.3 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
|
"git.novox.be/novox/mesh-sdk/go/health"
|
|
)
|
|
|
|
const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB"
|
|
|
|
func underFloor(node string) buildstep.Floor {
|
|
f := floorOf(node)
|
|
f.Free = func(string) (uint64, error) { return 4 << 30, nil }
|
|
return f
|
|
}
|
|
|
|
// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB.
|
|
func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) {
|
|
if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" {
|
|
t.Errorf("floorOf = %+v, want ace and 20 GiB", f)
|
|
}
|
|
}
|
|
|
|
// 1839/AC1: under the floor a fetch and a build are refused, naming the node, what is free and the floor, nothing runs,
|
|
// and the module is unhealthy saying so, so the builder chooses another node.
|
|
func TestUnderTheFloorAFetchAndABuildAreRefusedAndTheModuleIsUnhealthy(t *testing.T) {
|
|
f := newFake(t)
|
|
states := newStates()
|
|
m := moduleWith(t, f, states)
|
|
m.Floor = underFloor("ace")
|
|
for name, args := range map[string]map[string]any{
|
|
FetchTool: {"repository": "novox/mesh-catalog", "commit": commitSHA},
|
|
BuildTool: {"workspace": "abc1", "module": "modules/m", "artifact": "image"},
|
|
} {
|
|
if _, err := tool(t, m, name)(args); err != nil {
|
|
t.Fatalf("%s: %v", name, err)
|
|
}
|
|
if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) {
|
|
t.Errorf("%s ended %s: %q, want refused: %q", name, st.State, st.Reason, underTheFloor)
|
|
}
|
|
}
|
|
for _, l := range f.lines() {
|
|
if strings.HasPrefix(l, "git ") || strings.Contains(l, " bud ") {
|
|
t.Errorf("a refused step ran %q", l)
|
|
}
|
|
}
|
|
if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) {
|
|
t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor)
|
|
}
|
|
}
|
|
|
|
// 1839/AC2: build_image_release removes the workspace at once, even under the floor; a second release is no error.
|
|
func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) {
|
|
m := moduleWith(t, newFake(t), newStates())
|
|
m.Floor = underFloor("ace")
|
|
ws := filepath.Join(m.Work, "abc1")
|
|
os.MkdirAll(filepath.Join(ws, "tree"), 0o700)
|
|
for i := 0; i < 2; i++ {
|
|
if _, err := tool(t, m, ReleaseTool)(map[string]any{"workspace": "abc1"}); err != nil {
|
|
t.Fatalf("release %d: %v", i, err)
|
|
}
|
|
}
|
|
if _, err := os.Stat(ws); !os.IsNotExist(err) {
|
|
t.Errorf("the workspace is still there after its release")
|
|
}
|
|
}
|
|
|
|
// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it.
|
|
func TestAFailedFetchLeavesNoWorkspace(t *testing.T) {
|
|
f := newFake(t)
|
|
states := newStates()
|
|
m := moduleWith(t, f, states)
|
|
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
|
if name == "git" && strings.Contains(strings.Join(args, " "), " fetch ") {
|
|
return forgekey.Ran{Status: 128, Stderr: "fatal: unreachable"}
|
|
}
|
|
return f.run(ctx, env, name, args...)
|
|
}
|
|
if _, err := tool(t, m, FetchTool)(map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if st := states.waitEnd(t); st.State != delivery.StepFailed {
|
|
t.Fatalf("the step ended %s (%s), want failed", st.State, st.Reason)
|
|
}
|
|
if entries, _ := os.ReadDir(m.Work); len(entries) != 0 {
|
|
t.Errorf("the failed fetch left %d entries in the work directory", len(entries))
|
|
}
|
|
}
|
|
|
|
// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is.
|
|
func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) {
|
|
m := &Module{Work: t.TempDir()}
|
|
for _, name := range []string{"held1", "lost1"} {
|
|
os.MkdirAll(filepath.Join(m.Work, name, "tree"), 0o755)
|
|
}
|
|
done := m.spaces().Hold("held1")
|
|
defer done()
|
|
when := time.Now().Add(-3 * time.Hour)
|
|
for _, name := range []string{"held1", "lost1"} {
|
|
os.Chtimes(filepath.Join(m.Work, name), when, when)
|
|
}
|
|
m.CleanWorkspaces(time.Now())
|
|
if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil {
|
|
t.Errorf("the held workspace was swept")
|
|
}
|
|
if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) {
|
|
t.Errorf("the unheld workspace three hours old was kept")
|
|
}
|
|
}
|