Free a build node's disk after every build and refuse work under a floor of free space (issue 1834)

A full / made every check on shanks end 'store never became ready' with the reason erased by --rm.
Builds now remove their local image tag, narrowed source and context clone; a build or check
refuses under 20 GiB free on / and the workspace, saying so; the throwaway store is no longer
started with --rm and a store that does not come up reports its exit code and last 20 log lines.
This commit is contained in:
2026-10-22 13:14:00 +02:00
parent 4463991e06
commit 995f8237ce
6 changed files with 322 additions and 7 deletions
+44 -1
View File
@@ -219,6 +219,11 @@ func build(ctx context.Context, run Runner, publish Publisher,
if err := os.MkdirAll(workspace, 0o755); err != nil {
return Result{}, err
}
// **A node with no room takes no build** (novox/hq issue 1834).
if err := RoomToWork("/", workspace); err != nil {
say("clone", "refused: %v", err)
return Result{}, err
}
// The credential is a file git reads, never an argument: a URL carrying a password in argv
// would be readable by anything that can list processes for as long as a clone runs.
credentials, forget, err := storeCredential(workspace, forge)
@@ -738,11 +743,34 @@ func logging(log Log) func(step, format string, args ...any) {
}
}
// keepsImageLocally says the publisher leaves the image in the machine's runtime as the artifact itself.
func keepsImageLocally(publish Publisher) bool {
switch publish.(type) {
case Local, *Local:
return true
}
return false
}
// releaseImage removes the local image tag a build made, with a context of its own: the build's may be cancelled. A
// tag that was never made (the build failed before it) is no fault; one that cannot be removed is said.
func releaseImage(ctx context.Context, run Runner, tag string, say func(step, format string, args ...any)) {
removing, done := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
defer done()
if _, err := run(removing, "", "docker", "rmi", tag); err != nil {
if !strings.Contains(err.Error(), "No such image") {
say("image", "the local image %s could not be removed, and goes with the week's sweep: %v", tag, err)
}
return
}
say("image", "the local image %s is removed: the registry holds it", tag)
}
// contextFrom clones an image artifact's own build context, when it names one apart from this
// module's own repository — a fresh tree, the same way the module's own is, keyed by artifact
// name so two artifacts of one module naming different contexts do not collide.
func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentials string,
from catalogue.ArtifactContext, seats map[string]string, say func(step, format string, args ...any)) (string, error) {
from catalogue.ArtifactContext, seats map[string]string, say func(step, format string, args ...any)) (_ string, err error) {
url, err := contextURL(from, seats)
if err != nil {
return "", err
@@ -752,6 +780,12 @@ func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentia
if err := os.RemoveAll(dir); err != nil {
return "", err
}
// A clone that failed half-way is no context to leave behind (novox/hq issue 1834).
defer func() {
if err != nil {
os.RemoveAll(dir)
}
}()
if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", url, dir)...); err != nil {
return "", fmt.Errorf("cannot clone %s: %w", url, err)
}
@@ -1143,6 +1177,8 @@ func one(ctx context.Context, run Runner, publish Publisher,
src.contexts[a.Name] = t
}
buildDir = cloned
// Let go of when the image is built, whatever became of it (novox/hq issue 1834).
defer os.RemoveAll(cloned)
if t, _ := trunkOf(ctx, run, cloned, "HEAD"); t == "" || !atTrunkHead(ctx, run, cloned, "HEAD", t) {
src.contextOffHead()
}
@@ -1180,6 +1216,7 @@ func one(ctx context.Context, run Runner, publish Publisher,
src.ownHas(paths...)
}
buildDir = narrowed
defer os.RemoveAll(narrowed)
} else if a.Context != nil {
src.readWhole(*a.Context)
}
@@ -1205,6 +1242,12 @@ func one(ctx context.Context, run Runner, publish Publisher,
}
invocation = append(invocation, ".")
say("image", "docker build -f %s", recipePath)
// **The local tag goes with the build** (novox/hq issue 1834): an image published to a registry is held there, and
// a tag per module and commit left on the node was the 96 GB of a full disk. Only a node publishing nowhere (a
// mesh being raised) keeps it, since the local image is then the artifact.
if !keepsImageLocally(publish) {
defer releaseImage(ctx, run, local, say)
}
if _, err := run(ctx, buildDir, "docker", invocation...); err != nil {
return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err)
}