diff --git a/modules/mesh-build-go/README.md b/modules/mesh-build-go/README.md index bdd3f286..69d8cfa2 100644 --- a/modules/mesh-build-go/README.md +++ b/modules/mesh-build-go/README.md @@ -37,8 +37,17 @@ Three read tools answer what the module stored in its own states, without the gu forge-account provision's ssh name, so the account's own ssh files and git rewrite are used) into a workspace of the work directory, then runs `go mod download` and `go mod verify` in every directory with a `go.mod` that does not vendor, with `GOTOOLCHAIN=local`, `GOFLAGS=-mod=mod` and the module's cache directory. Every command runs with an -environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes. Workspaces older than a day -are removed. +environment built from scratch (`StepRunner`): `PATH`, `HOME` and what the step passes. + +## Room to work and the workspace's end + +Every step is refused, ending `refused` with `refused: has N GiB free under the floor of 20 GiB (on )`, while +the work or the cache directory's filesystem has less than 20 GiB free (build-agent's floor, issue 1839); the health is +then unhealthy with the same text, so the builder chooses another node and the controller raises the condition. A +workspace is held while a step runs on it. `build_go_release {workspace}` removes it at once (no guard, no floor; one +already gone is no error, one a running step holds is refused); its caller calls it once its last step answered. A +failed fetch removes its own workspace. A workspace no caller released is removed two hours after its last use (ADR +0509), and the removal is said in the log as the caller's fault. A pin on a pull request's head (issue 1148): when `go mod download` fails on a `git.novox.be/novox/*` pseudo-version, the fetch looks for that commit among the forge's `refs/pull/*/head`, places the repository at it under the workspace's diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index a262ddb6..435def4d 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -23,11 +23,6 @@ var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`) // remote of a fetch is ssh:////.git. const ForgeAlias = "mesh-forge" -// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it -// fetches and every round it checks its toolchain. A workspace lies in the module's work directory, never in its -// home (ADR 0466 §1). -const WorkspaceAge = 24 * time.Hour - var ( repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) workspaceOK = regexp.MustCompile(`^[a-z0-9]+$`) @@ -82,20 +77,37 @@ func (m *Module) fetchTool() StepTool { } } -func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) { +// fetch makes a workspace and fills it. A workspace lies in the module's work directory, never in its home (ADR 0466 +// §1); it is held while the fetch runs, and removed when the fetch fails, since nobody learns its id to release it +// (issue 1839). +func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) { // What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its // named paths, the workspace is narrowed to them (buildstep.Narrow). req, err := buildstep.ParseFetchRequest(args) if err != nil { return nil, err } - repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base siblings, err := siblingsArg(args["beside"]) if err != nil { return nil, err } m.CleanWorkspaces(m.now()) id := buildstep.NewID() + done := m.spaces().Hold(id) + defer func() { + done() + if err != nil { + if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil { + m.Say("the failed fetch's workspace %s: %v", id, rerr) + } + } + }() + return m.fetchInto(ctx, req, id, siblings) +} + +func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, id string, siblings []siblingArg) (any, error) { + repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base + var err error root := filepath.Join(m.Work, id) tree := filepath.Join(root, "tree") if err := os.MkdirAll(tree, 0o700); err != nil { @@ -236,21 +248,10 @@ func (m *Module) goEnvFor(dir string) []string { return env } -// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory. +// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety +// net for a caller that never released one (issue 1839, ADR 0509). func (m *Module) CleanWorkspaces(now time.Time) { - entries, err := os.ReadDir(m.Work) - if err != nil { - return - } - for _, e := range entries { - info, err := e.Info() - if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge { - continue - } - if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil { - m.Say("the workspace %s could not be removed: %v", e.Name(), err) - } - } + m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound)) } // clock is the time now: Now when a test set it, else the wall clock. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go index 56657e06..d9d092c5 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go @@ -265,10 +265,11 @@ func TestARepositoryThatIsNotOwnerSlashNameIsRefusedBeforeAnythingRuns(t *testin } } -// Workspaces older than a day are removed; younger ones, and anything that is not a workspace, stay. -func TestWorkspacesOlderThanADayAreRemoved(t *testing.T) { +// Unheld workspaces last used more than two hours ago are removed (issue 1839, ADR 0509); younger ones, and anything +// that is not a workspace, stay. +func TestUnheldWorkspacesPastTwoHoursAreRemoved(t *testing.T) { m := &Module{Work: t.TempDir()} - for name, age := range map[string]time.Duration{"old1": 25 * time.Hour, "young": time.Hour, "Keep-This": 48 * time.Hour} { + for name, age := range map[string]time.Duration{"old1": 3 * time.Hour, "young": time.Hour, "Keep-This": 48 * time.Hour} { dir := filepath.Join(m.Work, name) os.MkdirAll(filepath.Join(dir, "tree"), 0o755) when := time.Now().Add(-age) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index de95f35a..85296b47 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -46,6 +46,7 @@ func main() { Work: WorkDir, Cache: CacheDir, } + m.Floor = floorOf(m.Node) if u, err := user.Current(); err == nil { m.Home = u.HomeDir } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index ee3cc99c..63bd0b16 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "sync" "time" "git.novox.be/novox/mesh-sdk/go/buildstep" @@ -65,6 +66,12 @@ type Module struct { // StartBus starts the throwaway bus of a run in dir and answers its address and what stops it (the module's own // program, StartBus in bus.go, when nil). StartBus func(ctx context.Context, dir string) (url string, stop func(), err error) + // Floor refuses every step while the work or cache filesystem has less free than it (issue 1839); main sets + // floorOf(node), and a zero Floor checks nothing. + Floor buildstep.Floor + + spacesOnce sync.Once + workspaces *buildstep.Workspaces } // guard is the SDK guard for the version the probe just read: the module names no Go version (ADR 0477 §2), so the @@ -113,6 +120,9 @@ func (m *Module) Health(ctx context.Context) health.Answer { if err := m.checkBubblewrap(ctx); err != nil { return health.Unhealthy(err.Error()) } + if err := m.room(); err != nil { + return health.Unhealthy(err.Error()) + } return health.Healthy(fmt.Sprintf("%s installed, the version the module builds with (recipe %s)", installed, m.Recipe.Version())) } @@ -125,6 +135,10 @@ func (m *Module) Long(tool string, run func(ctx context.Context, args map[string if err := m.Check(ctx); err != nil { return nil, err } + if err := m.room(); err != nil { + return nil, err + } + defer m.holdArg(args)() return run(ctx, args) }) if err != nil { @@ -141,6 +155,10 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any, if err := m.Check(m.Ctx); err != nil { return nil, err } + if err := m.room(); err != nil { + return nil, err + } + defer m.holdArg(args)() return run(m.Ctx, args) } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module_test.go b/modules/mesh-build-go/cmd/mesh-build-go/module_test.go index 02a84396..b9c79588 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module_test.go @@ -115,7 +115,7 @@ func TestAGoThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) { {Name: "short_probe", Run: run}, }, m.StepTools()...) for _, tool := range m.Tools(steps) { - if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool { + if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool { continue } var reason string diff --git a/modules/mesh-build-go/cmd/mesh-build-go/room.go b/modules/mesh-build-go/cmd/mesh-build-go/room.go new file mode 100644 index 00000000..21996a48 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/room.go @@ -0,0 +1,61 @@ +package main + +// A step's room to work and its workspace's end (novox/hq issue 1839, ADR 0509): every step is refused while the work +// or cache filesystem is under the floor, a workspace is held while a step runs on it, removed when its caller releases +// it, and swept unheld past buildstep.UnheldBound. + +import ( + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buildstep" +) + +// ReleaseTool removes a workspace whose caller's steps have ended. +const ReleaseTool = "build_go_release" + +// floorOf is the floor the module keeps on node: build-agent's. +func floorOf(node string) buildstep.Floor { + return buildstep.Floor{Node: node, Bytes: buildstep.FreeSpaceFloor} +} + +// room refuses while the work or cache filesystem is under the floor. +func (m *Module) room() error { return m.Floor.Check(m.Work, m.Cache) } + +// spaces are the module's workspaces under Work. +func (m *Module) spaces() *buildstep.Workspaces { + m.spacesOnce.Do(func() { m.workspaces = &buildstep.Workspaces{Dir: m.Work} }) + return m.workspaces +} + +// holdArg holds the workspace a step's arguments name while the step runs; nothing when they name none. +func (m *Module) holdArg(args map[string]any) func() { + ws, _ := args["workspace"].(string) + if ws == "" { + return func() {} + } + return m.spaces().Hold(ws) +} + +// releaseTool is build_go_release: no guard and no floor, since it only frees room. +func (m *Module) releaseTool() stdio.Tool { + return stdio.Tool{Name: ReleaseTool, + Description: "Remove a workspace whose steps have ended (fetch, test, build): its caller says so once its last step " + + "answered. A workspace already gone is no error; one a running step holds is refused. A workspace never released " + + "is removed two hours after its last use.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "workspace": map[string]any{"type": "string", "description": "the workspace id a fetch answered"}, + }, "required": []string{"workspace"}}, + Run: func(args map[string]any) (any, error) { + ws, _ := args["workspace"].(string) + if err := m.spaces().Release(ws); err != nil { + return nil, err + } + return map[string]any{"released": ws}, nil + }} +} + +// sweepSaid is CleanWorkspaces' saying of a workspace a caller never released: a fault of the caller (ADR 0509). +func (m *Module) sweepSaid(removed []string) { + if len(removed) > 0 && m.Say != nil { + m.Say("removed %d workspaces no caller released within %s: %v", len(removed), buildstep.UnheldBound, removed) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/room_test.go b/modules/mesh-build-go/cmd/mesh-build-go/room_test.go new file mode 100644 index 00000000..da7069e0 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/room_test.go @@ -0,0 +1,126 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" + "git.novox.be/novox/mesh-sdk/go/health" +) + +const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB" + +func underFloor(node string) buildstep.Floor { + f := floorOf(node) + f.Free = func(string) (uint64, error) { return 4 << 30, nil } + return f +} + +// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB. +func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) { + if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" { + t.Errorf("floorOf = %+v, want ace and 20 GiB", f) + } +} + +// 1839/AC1: under the floor every step tool is refused, naming the node, what is free and the floor, no step runs, and +// the module is unhealthy saying so, so the builder chooses another node. +func TestUnderTheFloorEveryStepIsRefusedAndTheModuleIsUnhealthy(t *testing.T) { + states := newStates() + m := moduleWith(states, "go1.27.1") + m.Work, m.Cache, m.Floor = t.TempDir(), t.TempDir(), underFloor("ace") + ran := false + run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil } + steps := []StepTool{{Name: "long_probe", Long: true, Run: run}, {Name: "short_probe", Run: run}} + for _, tool := range m.Tools(steps) { + switch tool.Name { + case "long_probe": + if _, err := tool.Run(map[string]any{}); err != nil { + t.Fatal(err) + } + if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) { + t.Errorf("the long step ended %s: %q, want refused: %q", st.State, st.Reason, underTheFloor) + } + case "short_probe": + if _, err := tool.Run(map[string]any{}); err == nil || !strings.HasPrefix(err.Error(), underTheFloor) { + t.Errorf("the short step answered %v, want %q", err, underTheFloor) + } + } + } + if ran { + t.Errorf("a step ran under the floor") + } + if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) { + t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor) + } +} + +func releaseTool(t *testing.T, m *Module) func(map[string]any) (any, error) { + t.Helper() + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == ReleaseTool { + return tool.Run + } + } + t.Fatalf("no tool %s", ReleaseTool) + return nil +} + +// 1839/AC2: build_go_release removes the workspace at once, even under the floor; a second release is no error. +func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) { + m, _ := fetchModule(t, &fakeStep{}) + m.Floor = underFloor("ace") + ws := filepath.Join(m.Work, "abc1") + os.MkdirAll(filepath.Join(ws, "tree"), 0o700) + for i := 0; i < 2; i++ { + if _, err := releaseTool(t, m)(map[string]any{"workspace": "abc1"}); err != nil { + t.Fatalf("release %d: %v", i, err) + } + } + if _, err := os.Stat(ws); !os.IsNotExist(err) { + t.Errorf("the workspace is still there after its release") + } +} + +// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it. +func TestAFailedFetchLeavesNoWorkspace(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n", goRule: func(string) forgekey.Ran { return forgekey.Ran{Status: 1, Stderr: "boom"} }} + m, states := fetchModule(t, f) + if _, err := fetchTool(t, m)(fetchArgs); err != nil { + t.Fatal(err) + } + if st := states.waitEnd(t); st.State != delivery.StepFailed { + t.Fatalf("the step ended %s, want failed", st.State) + } + if entries, _ := os.ReadDir(m.Work); len(entries) != 0 { + t.Errorf("the failed fetch left %d entries in the work directory", len(entries)) + } +} + +// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is. +func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) { + m := &Module{Work: t.TempDir()} + for _, name := range []string{"held1", "lost1"} { + dir := filepath.Join(m.Work, name) + os.MkdirAll(filepath.Join(dir, "tree"), 0o755) + } + done := m.spaces().Hold("held1") + defer done() + when := time.Now().Add(-3 * time.Hour) + for _, name := range []string{"held1", "lost1"} { + os.Chtimes(filepath.Join(m.Work, name), when, when) + } + m.CleanWorkspaces(time.Now()) + if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil { + t.Errorf("the held workspace was swept") + } + if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) { + t.Errorf("the unheld workspace three hours old was kept") + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/tools.go b/modules/mesh-build-go/cmd/mesh-build-go/tools.go index 06ff4d66..5aac21bd 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/tools.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/tools.go @@ -40,6 +40,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool { tools = append(tools, t) } tools = append(tools, m.ReadTools()...) + tools = append(tools, m.releaseTool()) return append(tools, stdio.Tool{Name: StatusTool, Description: "What the Go build module is on this machine: the Go version its recipe builds with and the one installed, " + diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index e80b240e..999b31b9 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -3,7 +3,7 @@ module meshbuildgo go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-build-go/go.sum b/modules/mesh-build-go/go.sum index 83c780bf..ccb75980 100644 --- a/modules/mesh-build-go/go.sum +++ b/modules/mesh-build-go/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 5d801100..b78bc91a 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -17,6 +17,7 @@ "build_go_test", "build_go_digest", "build_go_build", + "build_go_release", "build_go_step", "build_go_steps", "build_go_toolchains", diff --git a/modules/mesh-build-image/README.md b/modules/mesh-build-image/README.md index e25dd475..a6c3b62e 100644 --- a/modules/mesh-build-image/README.md +++ b/modules/mesh-build-image/README.md @@ -37,6 +37,12 @@ The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can re - It refuses, naming why, an image whose build context is a whole repository (`context`) or whose `build.on` names another module's artifact (`unsupportedImage` in `build.go`): those forms belong to the images build-agent builds until it is removed at R4g. +- `build_image_release {workspace}` removes a workspace at once (no guard, no floor; one already gone is no error, one a + running step holds is refused); its caller calls it once its last step answered. A failed fetch removes its own + workspace, and one no caller released is removed two hours after its last use (ADR 0509). +- A fetch and a build are refused, `refused: has N GiB free under the floor of 20 GiB (on )`, and the health + is unhealthy with that text, while the work directory's or the image store's filesystem has less than 20 GiB free + (issue 1839). A copy writes nothing to the disk and is not refused. - `build_image_step`, `build_image_steps`, `build_image_status` and `build_image_health` read. The registry's address is placed into `/registry` as `${seat:mesh-artifact-store:reach}` (host:port): the provisioner's client fills a moved module's placement, and the artifact store has not moved, so a `${bound:artifact-store:...}` is refused at composition (issue 1703). diff --git a/modules/mesh-build-image/cmd/mesh-build-image/fetch.go b/modules/mesh-build-image/cmd/mesh-build-image/fetch.go index 928173b8..e07bbfda 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/fetch.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/fetch.go @@ -26,11 +26,6 @@ var ( // remote of a fetch is ssh:////.git. const ForgeAlias = "mesh-forge" -// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it -// fetches and every round it checks itself. A workspace lies in the module's work directory, never in its home -// (ADR 0466 §1). -const WorkspaceAge = 24 * time.Hour - // FetchResult is what a passed fetch step says: the workspace it made and the commit in it, and, only when the call // gave a base, the merge preview of ADR 0484 clause 3 (the trunk commit, the merge commit of the head onto it, the // files in conflict, the paths the merge changes and the tree of each module directory). A head that does not @@ -66,7 +61,9 @@ func (m *Module) fetchTool() StepTool { } } -func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) { +// fetch makes a workspace and fills it. The workspace is held while the fetch runs, and removed when the fetch fails, +// since nobody learns its id to release it (issue 1839). +func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) { repository, _ := args["repository"].(string) commit, _ := args["commit"].(string) ref, _ := args["ref"].(string) @@ -82,6 +79,19 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) { } m.CleanWorkspaces(m.clock()) id := buildstep.NewID() + done := m.spaces().Hold(id) + defer func() { + done() + if err != nil { + if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil { + m.Say("the failed fetch's workspace %s: %v", id, rerr) + } + } + }() + return m.fetchInto(ctx, repository, commit, ref, base, id) +} + +func (m *Module) fetchInto(ctx context.Context, repository, commit, ref, base, id string) (any, error) { tree := filepath.Join(m.Work, id, "tree") if err := os.MkdirAll(tree, 0o700); err != nil { return nil, err @@ -124,21 +134,10 @@ func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base return err } -// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory. +// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety +// net for a caller that never released one (issue 1839, ADR 0509). func (m *Module) CleanWorkspaces(now time.Time) { - entries, err := os.ReadDir(m.Work) - if err != nil { - return - } - for _, e := range entries { - info, err := e.Info() - if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge { - continue - } - if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil { - m.Say("the workspace %s could not be removed: %v", e.Name(), err) - } - } + m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound)) } // clock is the time now: Now when a test set it, else the wall clock. diff --git a/modules/mesh-build-image/cmd/mesh-build-image/main.go b/modules/mesh-build-image/cmd/mesh-build-image/main.go index 24a3d193..169d9b10 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/main.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/main.go @@ -37,6 +37,7 @@ func main() { Work: WorkDir, Store: StoreDir, } + m.Floor = floorOf(m.Node) if u, err := user.Current(); err == nil { m.Home, m.Account, m.UID = u.HomeDir, u.Username, u.Uid } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/module.go b/modules/mesh-build-image/cmd/mesh-build-image/module.go index c6d869c6..1f3f206f 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/module.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/module.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "sync" "time" "git.novox.be/novox/mesh-sdk/go/buildstep" @@ -45,6 +46,12 @@ type Module struct { RegistryPause time.Duration // Now is the clock; time.Now when nil. Now func() time.Time + // Floor refuses every fetch and build while the work or the image store's filesystem has less free than it (issue + // 1839); main sets floorOf(node), and a zero Floor checks nothing. + Floor buildstep.Floor + + spacesOnce sync.Once + workspaces *buildstep.Workspaces } // Look is what the module found out about its machine. @@ -104,6 +111,9 @@ func (m *Module) Health(ctx context.Context) health.Answer { if err != nil { return health.Unhealthy(err.Error()) } + if err := m.room(); err != nil { + return health.Unhealthy(err.Error()) + } return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made", l.Buildah, l.Account, l.UIDs, l.GIDs)) } @@ -126,6 +136,10 @@ func (m *Module) Long(tool string, run func(ctx context.Context, args map[string if err := m.Check(ctx); err != nil { return nil, err } + if err := m.room(); err != nil { + return nil, err + } + defer m.holdArg(args)() return run(ctx, args) }) if err != nil { @@ -142,6 +156,10 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any, if err := m.Check(m.Ctx); err != nil { return nil, err } + if err := m.room(); err != nil { + return nil, err + } + defer m.holdArg(args)() return run(m.Ctx, args) } } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/room.go b/modules/mesh-build-image/cmd/mesh-build-image/room.go new file mode 100644 index 00000000..2d021306 --- /dev/null +++ b/modules/mesh-build-image/cmd/mesh-build-image/room.go @@ -0,0 +1,61 @@ +package main + +// A step's room to work and its workspace's end (novox/hq issue 1839, ADR 0509): every fetch and build is refused while +// the work or image store's filesystem is under the floor, a workspace is held while a step runs on it, removed when its caller releases +// it, and swept unheld past buildstep.UnheldBound. + +import ( + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buildstep" +) + +// ReleaseTool removes a workspace whose caller's steps have ended. +const ReleaseTool = "build_image_release" + +// floorOf is the floor the module keeps on node: build-agent's. +func floorOf(node string) buildstep.Floor { + return buildstep.Floor{Node: node, Bytes: buildstep.FreeSpaceFloor} +} + +// room refuses while the work or the image store's filesystem is under the floor. +func (m *Module) room() error { return m.Floor.Check(m.Work, m.Store) } + +// spaces are the module's workspaces under Work. +func (m *Module) spaces() *buildstep.Workspaces { + m.spacesOnce.Do(func() { m.workspaces = &buildstep.Workspaces{Dir: m.Work} }) + return m.workspaces +} + +// holdArg holds the workspace a step's arguments name while the step runs; nothing when they name none. +func (m *Module) holdArg(args map[string]any) func() { + ws, _ := args["workspace"].(string) + if ws == "" { + return func() {} + } + return m.spaces().Hold(ws) +} + +// releaseTool is build_image_release: no guard and no floor, since it only frees room. +func (m *Module) releaseTool() stdio.Tool { + return stdio.Tool{Name: ReleaseTool, + Description: "Remove a workspace whose steps have ended (fetch, build): its caller says so once its last step " + + "answered. A workspace already gone is no error; one a running step holds is refused. A workspace never released " + + "is removed two hours after its last use.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "workspace": map[string]any{"type": "string", "description": "the workspace id a fetch answered"}, + }, "required": []string{"workspace"}}, + Run: func(args map[string]any) (any, error) { + ws, _ := args["workspace"].(string) + if err := m.spaces().Release(ws); err != nil { + return nil, err + } + return map[string]any{"released": ws}, nil + }} +} + +// sweepSaid is CleanWorkspaces' saying of a workspace a caller never released: a fault of the caller (ADR 0509). +func (m *Module) sweepSaid(removed []string) { + if len(removed) > 0 && m.Say != nil { + m.Say("removed %d workspaces no caller released within %s: %v", len(removed), buildstep.UnheldBound, removed) + } +} diff --git a/modules/mesh-build-image/cmd/mesh-build-image/room_test.go b/modules/mesh-build-image/cmd/mesh-build-image/room_test.go new file mode 100644 index 00000000..95f1f58e --- /dev/null +++ b/modules/mesh-build-image/cmd/mesh-build-image/room_test.go @@ -0,0 +1,117 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" + "git.novox.be/novox/mesh-sdk/go/health" +) + +const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB" + +func underFloor(node string) buildstep.Floor { + f := floorOf(node) + f.Free = func(string) (uint64, error) { return 4 << 30, nil } + return f +} + +// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB. +func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) { + if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" { + t.Errorf("floorOf = %+v, want ace and 20 GiB", f) + } +} + +// 1839/AC1: under the floor a fetch and a build are refused, naming the node, what is free and the floor, nothing runs, +// and the module is unhealthy saying so, so the builder chooses another node. +func TestUnderTheFloorAFetchAndABuildAreRefusedAndTheModuleIsUnhealthy(t *testing.T) { + f := newFake(t) + states := newStates() + m := moduleWith(t, f, states) + m.Floor = underFloor("ace") + for name, args := range map[string]map[string]any{ + FetchTool: {"repository": "novox/mesh-catalog", "commit": commitSHA}, + BuildTool: {"workspace": "abc1", "module": "modules/m", "artifact": "image"}, + } { + if _, err := tool(t, m, name)(args); err != nil { + t.Fatalf("%s: %v", name, err) + } + if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) { + t.Errorf("%s ended %s: %q, want refused: %q", name, st.State, st.Reason, underTheFloor) + } + } + for _, l := range f.lines() { + if strings.HasPrefix(l, "git ") || strings.Contains(l, " bud ") { + t.Errorf("a refused step ran %q", l) + } + } + if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) { + t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor) + } +} + +// 1839/AC2: build_image_release removes the workspace at once, even under the floor; a second release is no error. +func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) { + m := moduleWith(t, newFake(t), newStates()) + m.Floor = underFloor("ace") + ws := filepath.Join(m.Work, "abc1") + os.MkdirAll(filepath.Join(ws, "tree"), 0o700) + for i := 0; i < 2; i++ { + if _, err := tool(t, m, ReleaseTool)(map[string]any{"workspace": "abc1"}); err != nil { + t.Fatalf("release %d: %v", i, err) + } + } + if _, err := os.Stat(ws); !os.IsNotExist(err) { + t.Errorf("the workspace is still there after its release") + } +} + +// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it. +func TestAFailedFetchLeavesNoWorkspace(t *testing.T) { + f := newFake(t) + states := newStates() + m := moduleWith(t, f, states) + m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + if name == "git" && strings.Contains(strings.Join(args, " "), " fetch ") { + return forgekey.Ran{Status: 128, Stderr: "fatal: unreachable"} + } + return f.run(ctx, env, name, args...) + } + if _, err := tool(t, m, FetchTool)(map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA}); err != nil { + t.Fatal(err) + } + if st := states.waitEnd(t); st.State != delivery.StepFailed { + t.Fatalf("the step ended %s (%s), want failed", st.State, st.Reason) + } + if entries, _ := os.ReadDir(m.Work); len(entries) != 0 { + t.Errorf("the failed fetch left %d entries in the work directory", len(entries)) + } +} + +// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is. +func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) { + m := &Module{Work: t.TempDir()} + for _, name := range []string{"held1", "lost1"} { + os.MkdirAll(filepath.Join(m.Work, name, "tree"), 0o755) + } + done := m.spaces().Hold("held1") + defer done() + when := time.Now().Add(-3 * time.Hour) + for _, name := range []string{"held1", "lost1"} { + os.Chtimes(filepath.Join(m.Work, name), when, when) + } + m.CleanWorkspaces(time.Now()) + if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil { + t.Errorf("the held workspace was swept") + } + if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) { + t.Errorf("the unheld workspace three hours old was kept") + } +} diff --git a/modules/mesh-build-image/cmd/mesh-build-image/tools.go b/modules/mesh-build-image/cmd/mesh-build-image/tools.go index f0ebf98a..b660e859 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/tools.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/tools.go @@ -44,6 +44,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool { tools = append(tools, t) } tools = append(tools, m.ReadTools()...) + tools = append(tools, m.releaseTool()) return append(tools, stdio.Tool{Name: StatusTool, Description: "What the image build module is on this machine: the buildah installed, the account's subordinate " + diff --git a/modules/mesh-build-image/go.mod b/modules/mesh-build-image/go.mod index 2e2014fe..a1456ae4 100644 --- a/modules/mesh-build-image/go.mod +++ b/modules/mesh-build-image/go.mod @@ -3,7 +3,7 @@ module meshbuildimage go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-image/go.sum b/modules/mesh-build-image/go.sum index db1ea18c..4ae4817b 100644 --- a/modules/mesh-build-image/go.sum +++ b/modules/mesh-build-image/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-build-image/module.json b/modules/mesh-build-image/module.json index 2cee8afa..1f1f86de 100644 --- a/modules/mesh-build-image/module.json +++ b/modules/mesh-build-image/module.json @@ -15,6 +15,7 @@ "build_image_fetch", "build_image_build", "build_image_copy", + "build_image_release", "build_image_step", "build_image_steps", "build_image_status", diff --git a/modules/mesh-build-node/README.md b/modules/mesh-build-node/README.md index f2cfce7e..2fac0b52 100644 --- a/modules/mesh-build-node/README.md +++ b/modules/mesh-build-node/README.md @@ -57,7 +57,11 @@ toolchain install: a long step answers at once with its id and ends on `.< removed). The mesh's registry reaches `npm ci` through a private config file (`--userconfig`), so other `@novox/` packages install. A module with packages and no lock does not fail the fetch: its result lists it under `refused` with the reason, and a build of that bundle fails naming it (issue 1459). Every command runs through `forgekey.ExecRunnerClean(0)` with PATH, HOME, the locale and its own - words, nothing of this process's environment. Workspaces older than a day are removed at each fetch. + words, nothing of this process's environment. A failed fetch removes its own workspace; `build_node_release {workspace}` removes one at once (no guard, no + floor; its caller calls it once its last step answered), and one no caller released is removed two hours after its + last use (ADR 0509), never while a step holds it. Every step is refused, `refused: has N GiB free under the + floor of 20 GiB (on )`, and the health is unhealthy with that text, while the work or the toolchain cache + filesystem has less than 20 GiB free (issue 1839). The arguments are mesh-sdk's `buildstep.FetchRequest` (ADR 0499 §4): with `dir` and `paths` (from the builder, never from a manifest) the workspace is narrowed to those paths, the manifest and the lock files before anything else. A member is built against the SDK version its own lock names (the lead's decision L10 of R4e, ADR 0500 §5-§6): the diff --git a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go index e0ca6bf3..35b1eeab 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/fetch.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/fetch.go @@ -33,9 +33,6 @@ const ForgeAlias = "mesh-forge" // sdkPackage is the package the toolchain carries and a module's own dependencies never supply. const sdkPackage = "@novox/mesh-sdk" -// WorkspaceAge is how long a workspace is kept. -const WorkspaceAge = 24 * time.Hour - var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`) var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) @@ -105,20 +102,36 @@ func (m *Module) fetchTool() StepTool { } } -func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) { +// fetch makes a workspace and fills it. The workspace is held while the fetch runs, and removed when the fetch fails, +// since nobody learns its id to release it (issue 1839). +func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err error) { // What the builder and the checker ask, in mesh-sdk's one form (ADR 0499 §4): with the module's directory and its // named paths, the workspace is narrowed to them (buildstep.Narrow). req, err := buildstep.ParseFetchRequest(args) if err != nil { return nil, err } - repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base group, err := groupPackageArg(args["package"]) if err != nil { return nil, err } m.CleanWorkspaces(m.now()) id := buildstep.NewID() + done := m.spaces().Hold(id) + defer func() { + done() + if err != nil { + if rerr := m.spaces().Release(id); rerr != nil && m.Say != nil { + m.Say("the failed fetch's workspace %s: %v", id, rerr) + } + } + }() + return m.fetchInto(ctx, req, group, id) +} + +func (m *Module) fetchInto(ctx context.Context, req buildstep.FetchRequest, group *groupPackage, id string) (any, error) { + repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base + var err error root := filepath.Join(m.Work, id) tree := filepath.Join(root, "tree") if err := os.MkdirAll(tree, 0o700); err != nil { @@ -354,19 +367,8 @@ func removeSDKCopies(nm string) error { return nil } -// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory. +// CleanWorkspaces removes the workspaces no step holds whose last use is older than buildstep.UnheldBound: the safety +// net for a caller that never released one (issue 1839, ADR 0509). func (m *Module) CleanWorkspaces(now time.Time) { - entries, err := os.ReadDir(m.Work) - if err != nil { - return - } - for _, e := range entries { - info, err := e.Info() - if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge { - continue - } - if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil { - m.Say("the workspace %s could not be removed: %v", e.Name(), err) - } - } + m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound)) } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/main.go b/modules/mesh-build-node/cmd/mesh-build-node/main.go index f06354a2..b6b1f30d 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/main.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/main.go @@ -54,6 +54,7 @@ func main() { Bubblewrap: ProbeBubblewrap, RegistryFile: filepath.Join(home, "registry"), } + m.Floor = floorOf(m.Node) // Outside a launch by the runtime — a build, a check — there is no node and no state to write. if m.Node != "" { go m.Start(ctx) diff --git a/modules/mesh-build-node/cmd/mesh-build-node/module.go b/modules/mesh-build-node/cmd/mesh-build-node/module.go index 94c4f87a..65660894 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/module.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/module.go @@ -59,8 +59,14 @@ type Module struct { // unhealthy, while it is missing or older than BubblewrapLeast. Nil probes nothing. Bubblewrap func(ctx context.Context) (string, error) - mu sync.Mutex // guards Recipe - install sync.Mutex // one install at a time + // Floor refuses every step while the work or toolchain cache filesystem has less free than it (issue 1839); main + // sets floorOf(node), and a zero Floor checks nothing. + Floor buildstep.Floor + + spacesOnce sync.Once + workspaces *buildstep.Workspaces + mu sync.Mutex // guards Recipe + install sync.Mutex // one install at a time } func (m *Module) recipe() Recipe { @@ -122,6 +128,7 @@ func (m *Module) Start(ctx context.Context) { if err := m.Prepare(ctx); err != nil && m.Say != nil { m.Say("the toolchain: %v", err) } + m.CleanWorkspaces(m.now()) select { case <-ctx.Done(): return @@ -138,6 +145,9 @@ func (m *Module) Health(ctx context.Context) health.Answer { if err != nil { return health.Unhealthy(err.Error()) } + if err := m.room(); err != nil { + return health.Unhealthy(err.Error()) + } return health.Healthy(fmt.Sprintf("node %s installed, the version the module builds with (recipe %s)", installed, m.recipe().Version())) } @@ -150,6 +160,10 @@ func (m *Module) Long(tool string, run func(ctx context.Context, args map[string if err := m.Prepare(ctx); err != nil { return nil, err } + if err := m.room(); err != nil { + return nil, err + } + defer m.holdArg(args)() return run(ctx, args) }) if err != nil { @@ -166,6 +180,10 @@ func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any, if err := m.Prepare(m.Ctx); err != nil { return nil, err } + if err := m.room(); err != nil { + return nil, err + } + defer m.holdArg(args)() return run(m.Ctx, args) } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/module_test.go b/modules/mesh-build-node/cmd/mesh-build-node/module_test.go index c144060d..0e5be38b 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/module_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/module_test.go @@ -117,7 +117,7 @@ func TestANodeThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) {Name: "short_probe", Run: run}, }, (&Module{}).StepTools()...) for _, tool := range m.Tools(steps) { - if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool { + if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool { continue } var reason string diff --git a/modules/mesh-build-node/cmd/mesh-build-node/room.go b/modules/mesh-build-node/cmd/mesh-build-node/room.go new file mode 100644 index 00000000..cc138f25 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/room.go @@ -0,0 +1,67 @@ +package main + +// A step's room to work and its workspace's end (novox/hq issue 1839, ADR 0509): every step is refused while the work +// or toolchain cache filesystem is under the floor, a workspace is held while a step runs on it, removed when its caller releases +// it, and swept unheld past buildstep.UnheldBound. + +import ( + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buildstep" +) + +// ReleaseTool removes a workspace whose caller's steps have ended. +const ReleaseTool = "build_node_release" + +// floorOf is the floor the module keeps on node: build-agent's. +func floorOf(node string) buildstep.Floor { + return buildstep.Floor{Node: node, Bytes: buildstep.FreeSpaceFloor} +} + +// room refuses while the work or toolchain cache filesystem is under the floor. +func (m *Module) room() error { + paths := []string{m.Work} + if m.Toolchain != nil { + paths = append(paths, m.Toolchain.Paths.Cache) + } + return m.Floor.Check(paths...) +} + +// spaces are the module's workspaces under Work. +func (m *Module) spaces() *buildstep.Workspaces { + m.spacesOnce.Do(func() { m.workspaces = &buildstep.Workspaces{Dir: m.Work} }) + return m.workspaces +} + +// holdArg holds the workspace a step's arguments name while the step runs; nothing when they name none. +func (m *Module) holdArg(args map[string]any) func() { + ws, _ := args["workspace"].(string) + if ws == "" { + return func() {} + } + return m.spaces().Hold(ws) +} + +// releaseTool is build_node_release: no guard and no floor, since it only frees room. +func (m *Module) releaseTool() stdio.Tool { + return stdio.Tool{Name: ReleaseTool, + Description: "Remove a workspace whose steps have ended (fetch, test, build): its caller says so once its last step " + + "answered. A workspace already gone is no error; one a running step holds is refused. A workspace never released " + + "is removed two hours after its last use.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "workspace": map[string]any{"type": "string", "description": "the workspace id a fetch answered"}, + }, "required": []string{"workspace"}}, + Run: func(args map[string]any) (any, error) { + ws, _ := args["workspace"].(string) + if err := m.spaces().Release(ws); err != nil { + return nil, err + } + return map[string]any{"released": ws}, nil + }} +} + +// sweepSaid is CleanWorkspaces' saying of a workspace a caller never released: a fault of the caller (ADR 0509). +func (m *Module) sweepSaid(removed []string) { + if len(removed) > 0 && m.Say != nil { + m.Say("removed %d workspaces no caller released within %s: %v", len(removed), buildstep.UnheldBound, removed) + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/room_test.go b/modules/mesh-build-node/cmd/mesh-build-node/room_test.go new file mode 100644 index 00000000..941aa149 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/room_test.go @@ -0,0 +1,123 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/forgekey" + "git.novox.be/novox/mesh-sdk/go/health" +) + +const underTheFloor = "refused: ace has 4 GiB free under the floor of 20 GiB" + +func underFloor(node string) buildstep.Floor { + f := floorOf(node) + f.Free = func(string) (uint64, error) { return 4 << 30, nil } + return f +} + +// 1839/AC4: the module as main makes it keeps build-agent's floor of 20 GiB. +func TestTheModuleKeepsTheFloorOf20GiB(t *testing.T) { + if f := floorOf("ace"); f.Bytes != 20<<30 || f.Node != "ace" { + t.Errorf("floorOf = %+v, want ace and 20 GiB", f) + } +} + +// 1839/AC1: under the floor every step tool is refused, naming the node, what is free and the floor, no step runs, and +// the module is unhealthy saying so, so the builder chooses another node. +func TestUnderTheFloorEveryStepIsRefusedAndTheModuleIsUnhealthy(t *testing.T) { + states := newStates() + m := moduleWith(states, "v26.10.0") + m.Work, m.Floor = t.TempDir(), underFloor("ace") + ran := false + run := func(context.Context, map[string]any) (any, error) { ran = true; return "done", nil } + steps := []StepTool{{Name: "long_probe", Long: true, Run: run}, {Name: "short_probe", Run: run}} + for _, tool := range m.Tools(steps) { + switch tool.Name { + case "long_probe": + if _, err := tool.Run(map[string]any{}); err != nil { + t.Fatal(err) + } + if st := states.waitEnd(t); st.State != delivery.StepRefused || !strings.HasPrefix(st.Reason, underTheFloor) { + t.Errorf("the long step ended %s: %q, want refused: %q", st.State, st.Reason, underTheFloor) + } + case "short_probe": + if _, err := tool.Run(map[string]any{}); err == nil || !strings.HasPrefix(err.Error(), underTheFloor) { + t.Errorf("the short step answered %v, want %q", err, underTheFloor) + } + } + } + if ran { + t.Errorf("a step ran under the floor") + } + if a := m.Health(context.Background()); a.State != health.StateUnhealthy || !strings.HasPrefix(a.Why, underTheFloor) { + t.Errorf("health = %+v, want unhealthy: %q", a, underTheFloor) + } +} + +func releaseTool(t *testing.T, m *Module) func(map[string]any) (any, error) { + t.Helper() + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == ReleaseTool { + return tool.Run + } + } + t.Fatalf("no tool %s", ReleaseTool) + return nil +} + +// 1839/AC2: build_node_release removes the workspace at once, even under the floor; a second release is no error. +func TestReleaseRemovesTheWorkspaceAtOnce(t *testing.T) { + m, _, work := newMachine(t) + m.Floor = underFloor("ace") + ws := filepath.Join(work, "abc1") + os.MkdirAll(filepath.Join(ws, "tree"), 0o700) + for i := 0; i < 2; i++ { + if _, err := releaseTool(t, m)(map[string]any{"workspace": "abc1"}); err != nil { + t.Fatalf("release %d: %v", i, err) + } + } + if _, err := os.Stat(ws); !os.IsNotExist(err) { + t.Errorf("the workspace is still there after its release") + } +} + +// 1839/AC2: a fetch that fails leaves no workspace behind: nobody was told its id, so nobody could release it. +func TestAFailedFetchLeavesNoWorkspace(t *testing.T) { + m, f := fetchMachine(t, typeScriptModule(`{}`), nil) + f.answer["git fetch"] = forgekey.Ran{Status: 128, Stderr: "fatal: unreachable"} + if _, err := m.fetch(context.Background(), fetchArgs); err == nil { + t.Fatal("the fetch passed, want it failed") + } + if entries, _ := os.ReadDir(m.Work); len(entries) != 0 { + t.Errorf("the failed fetch left %d entries in the work directory", len(entries)) + } +} + +// 1839/AC3: a workspace a running step holds is never swept, however old; an unheld one past the two-hour bound is, +// also at the module's own round and not only at a fetch. +func TestTheSweepKeepsAHeldWorkspaceAndRemovesAnUnheldOnePastTwoHours(t *testing.T) { + m := &Module{Work: t.TempDir()} + for _, name := range []string{"held1", "lost1"} { + os.MkdirAll(filepath.Join(m.Work, name, "tree"), 0o755) + } + done := m.spaces().Hold("held1") + defer done() + when := time.Now().Add(-3 * time.Hour) + for _, name := range []string{"held1", "lost1"} { + os.Chtimes(filepath.Join(m.Work, name), when, when) + } + m.CleanWorkspaces(time.Now()) + if _, err := os.Stat(filepath.Join(m.Work, "held1")); err != nil { + t.Errorf("the held workspace was swept") + } + if _, err := os.Stat(filepath.Join(m.Work, "lost1")); !os.IsNotExist(err) { + t.Errorf("the unheld workspace three hours old was kept") + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/tools.go b/modules/mesh-build-node/cmd/mesh-build-node/tools.go index 55b4e721..e28f7192 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/tools.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/tools.go @@ -40,6 +40,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool { tools = append(tools, t) } tools = append(tools, m.ReadTools()...) + tools = append(tools, m.releaseTool()) return append(tools, stdio.Tool{Name: StatusTool, Description: "What the Node build module is on this machine: the Node version its recipe builds with and the one installed, " + diff --git a/modules/mesh-build-node/go.mod b/modules/mesh-build-node/go.mod index bbdce43b..42810236 100644 --- a/modules/mesh-build-node/go.mod +++ b/modules/mesh-build-node/go.mod @@ -3,7 +3,7 @@ module meshbuildnode go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-node/go.sum b/modules/mesh-build-node/go.sum index 7186927c..ff19eb14 100644 --- a/modules/mesh-build-node/go.sum +++ b/modules/mesh-build-node/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467 h1:i85+fFT0wZiTZ9CRVfMzWg1+Zr7oOLl3UH7hEaKcOIc= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023062828-6147a2045467/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index c73aa9a1..df1691b5 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -16,6 +16,7 @@ "build_node_fetch", "build_node_test", "build_node_build", + "build_node_release", "build_node_step", "build_node_steps", "build_node_toolchains",