Free a build node's disk after every build and refuse work under a floor of free space (issue 1834)
A full / made every check on shanks end 'store never became ready' with the reason erased by --rm. Builds now remove their local image tag, narrowed source and context clone; a build or check refuses under 20 GiB free on / and the workspace, saying so; the throwaway store is no longer started with --rm and a store that does not come up reports its exit code and last 20 log lines.
This commit is contained in:
@@ -219,6 +219,11 @@ func build(ctx context.Context, run Runner, publish Publisher,
|
||||
if err := os.MkdirAll(workspace, 0o755); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
// **A node with no room takes no build** (novox/hq issue 1834).
|
||||
if err := RoomToWork("/", workspace); err != nil {
|
||||
say("clone", "refused: %v", err)
|
||||
return Result{}, err
|
||||
}
|
||||
// The credential is a file git reads, never an argument: a URL carrying a password in argv
|
||||
// would be readable by anything that can list processes for as long as a clone runs.
|
||||
credentials, forget, err := storeCredential(workspace, forge)
|
||||
@@ -738,11 +743,34 @@ func logging(log Log) func(step, format string, args ...any) {
|
||||
}
|
||||
}
|
||||
|
||||
// keepsImageLocally says the publisher leaves the image in the machine's runtime as the artifact itself.
|
||||
func keepsImageLocally(publish Publisher) bool {
|
||||
switch publish.(type) {
|
||||
case Local, *Local:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// releaseImage removes the local image tag a build made, with a context of its own: the build's may be cancelled. A
|
||||
// tag that was never made (the build failed before it) is no fault; one that cannot be removed is said.
|
||||
func releaseImage(ctx context.Context, run Runner, tag string, say func(step, format string, args ...any)) {
|
||||
removing, done := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
|
||||
defer done()
|
||||
if _, err := run(removing, "", "docker", "rmi", tag); err != nil {
|
||||
if !strings.Contains(err.Error(), "No such image") {
|
||||
say("image", "the local image %s could not be removed, and goes with the week's sweep: %v", tag, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
say("image", "the local image %s is removed: the registry holds it", tag)
|
||||
}
|
||||
|
||||
// contextFrom clones an image artifact's own build context, when it names one apart from this
|
||||
// module's own repository — a fresh tree, the same way the module's own is, keyed by artifact
|
||||
// name so two artifacts of one module naming different contexts do not collide.
|
||||
func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentials string,
|
||||
from catalogue.ArtifactContext, seats map[string]string, say func(step, format string, args ...any)) (string, error) {
|
||||
from catalogue.ArtifactContext, seats map[string]string, say func(step, format string, args ...any)) (_ string, err error) {
|
||||
url, err := contextURL(from, seats)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -752,6 +780,12 @@ func contextFrom(ctx context.Context, run Runner, workspace, artifact, credentia
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// A clone that failed half-way is no context to leave behind (novox/hq issue 1834).
|
||||
defer func() {
|
||||
if err != nil {
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}()
|
||||
if _, err := run(ctx, workspace, "git", cloneWith(credentials, "clone", "--quiet", url, dir)...); err != nil {
|
||||
return "", fmt.Errorf("cannot clone %s: %w", url, err)
|
||||
}
|
||||
@@ -1143,6 +1177,8 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
src.contexts[a.Name] = t
|
||||
}
|
||||
buildDir = cloned
|
||||
// Let go of when the image is built, whatever became of it (novox/hq issue 1834).
|
||||
defer os.RemoveAll(cloned)
|
||||
if t, _ := trunkOf(ctx, run, cloned, "HEAD"); t == "" || !atTrunkHead(ctx, run, cloned, "HEAD", t) {
|
||||
src.contextOffHead()
|
||||
}
|
||||
@@ -1180,6 +1216,7 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
src.ownHas(paths...)
|
||||
}
|
||||
buildDir = narrowed
|
||||
defer os.RemoveAll(narrowed)
|
||||
} else if a.Context != nil {
|
||||
src.readWhole(*a.Context)
|
||||
}
|
||||
@@ -1205,6 +1242,12 @@ func one(ctx context.Context, run Runner, publish Publisher,
|
||||
}
|
||||
invocation = append(invocation, ".")
|
||||
say("image", "docker build -f %s", recipePath)
|
||||
// **The local tag goes with the build** (novox/hq issue 1834): an image published to a registry is held there, and
|
||||
// a tag per module and commit left on the node was the 96 GB of a full disk. Only a node publishing nowhere (a
|
||||
// mesh being raised) keeps it, since the local image is then the artifact.
|
||||
if !keepsImageLocally(publish) {
|
||||
defer releaseImage(ctx, run, local, say)
|
||||
}
|
||||
if _, err := run(ctx, buildDir, "docker", invocation...); err != nil {
|
||||
return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user