Refuse build steps under a free-space floor and release workspaces, so the build modules stop filling a build node's disk (issue 1839)
mesh/delivery-group group fix/1839-build-floor checking: 1 of 2 member(s) ready
mesh/delivery superseded: a newer head of the same pull request
mesh/check failed: novox: the check ran past its bound of 30m0s, stuck at: test the unit level of novox/mesh-catalog#458 on novox | 3ca48ff23005 on 89…
mesh/delivery-group group fix/1839-build-floor checking: 1 of 2 member(s) ready
mesh/delivery superseded: a newer head of the same pull request
mesh/check failed: novox: the check ran past its bound of 30m0s, stuck at: test the unit level of novox/mesh-catalog#458 on novox | 3ca48ff23005 on 89…
mesh-build-go, mesh-build-node and mesh-build-image refuse every fetch, test and build, and say unhealthy, while their work or cache filesystem has less than 20 GiB free (build-agent's floor), so the builder chooses another node instead of filling this one. Each serves build_<x>_release, removing a workspace once its caller's last step answered; a failed fetch removes its own; a workspace never released goes two hours after its last use (ADR 0509) in place of a day, never while a step holds it. Pins mesh-sdk's fix/1839-build-floor (buildstep.Floor, buildstep.Workspaces).
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
// A step's room to work and its workspace's end (novox/hq issue 1839, ADR 0509): every step is refused while the work
|
||||
// or toolchain cache filesystem is under the floor, a workspace is held while a step runs on it, removed when its caller releases
|
||||
// it, and swept unheld past buildstep.UnheldBound.
|
||||
|
||||
import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
// ReleaseTool removes a workspace whose caller's steps have ended.
|
||||
const ReleaseTool = "build_node_release"
|
||||
|
||||
// floorOf is the floor the module keeps on node: build-agent's.
|
||||
func floorOf(node string) buildstep.Floor {
|
||||
return buildstep.Floor{Node: node, Bytes: buildstep.FreeSpaceFloor}
|
||||
}
|
||||
|
||||
// room refuses while the work or toolchain cache filesystem is under the floor.
|
||||
func (m *Module) room() error {
|
||||
paths := []string{m.Work}
|
||||
if m.Toolchain != nil {
|
||||
paths = append(paths, m.Toolchain.Paths.Cache)
|
||||
}
|
||||
return m.Floor.Check(paths...)
|
||||
}
|
||||
|
||||
// spaces are the module's workspaces under Work.
|
||||
func (m *Module) spaces() *buildstep.Workspaces {
|
||||
m.spacesOnce.Do(func() { m.workspaces = &buildstep.Workspaces{Dir: m.Work} })
|
||||
return m.workspaces
|
||||
}
|
||||
|
||||
// holdArg holds the workspace a step's arguments name while the step runs; nothing when they name none.
|
||||
func (m *Module) holdArg(args map[string]any) func() {
|
||||
ws, _ := args["workspace"].(string)
|
||||
if ws == "" {
|
||||
return func() {}
|
||||
}
|
||||
return m.spaces().Hold(ws)
|
||||
}
|
||||
|
||||
// releaseTool is build_node_release: no guard and no floor, since it only frees room.
|
||||
func (m *Module) releaseTool() stdio.Tool {
|
||||
return stdio.Tool{Name: ReleaseTool,
|
||||
Description: "Remove a workspace whose steps have ended (fetch, test, build): its caller says so once its last step " +
|
||||
"answered. A workspace already gone is no error; one a running step holds is refused. A workspace never released " +
|
||||
"is removed two hours after its last use.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id a fetch answered"},
|
||||
}, "required": []string{"workspace"}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ws, _ := args["workspace"].(string)
|
||||
if err := m.spaces().Release(ws); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"released": ws}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// sweepSaid is CleanWorkspaces' saying of a workspace a caller never released: a fault of the caller (ADR 0509).
|
||||
func (m *Module) sweepSaid(removed []string) {
|
||||
if len(removed) > 0 && m.Say != nil {
|
||||
m.Say("removed %d workspaces no caller released within %s: %v", len(removed), buildstep.UnheldBound, removed)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user