Free a build node's disk after every build and refuse work under a floor of free space (issue 1834)

A full / made every check on shanks end 'store never became ready' with the reason erased by --rm.
Builds now remove their local image tag, narrowed source and context clone; a build or check
refuses under 20 GiB free on / and the workspace, saying so; the throwaway store is no longer
started with --rm and a store that does not come up reports its exit code and last 20 log lines.

internal/builder/check.go

@@ -392,6 +392,12 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry
 	ctx, stop := context.WithTimeout(ctx, CheckTimeout)
 	defer stop()
 
+	// **A node with no room takes no check** (novox/hq issue 1834): said as the check's verdict, never a throwaway store
+	// that dies at once on a full disk.
+	if err := RoomToWork("/", workspace); err != nil {
+		say("check", "refused: %v", err)
+		return CheckVerdict{}, err
+	}
 	root := filepath.Join(workspace, "check")
 	if err := os.RemoveAll(root); err != nil {
 		return CheckVerdict{}, err
@@ -1631,7 +1637,9 @@ func BusImage(version string) string {
 
 // throwaway starts a container publishing one port on loopback, and answers where it is reached.
 func throwaway(ctx context.Context, run, plain Runner, name, image string, port int, opts, args []string) (string, error) {
-	invocation := []string{"run", "-d", "--rm", "--name", name, "-p", fmt.Sprintf("127.0.0.1::%d", port)}
+	// Never with --rm (novox/hq issue 1834): a store that dies at once took its log with it. The check removes what it
+	// raised, with its volumes, after it has read why one stopped (RemoveContainersOf).
+	invocation := []string{"run", "-d", "--name", name, "-p", fmt.Sprintf("127.0.0.1::%d", port)}
 	invocation = append(invocation, opts...)
 	invocation = append(invocation, image)
 	invocation = append(invocation, args...)
@@ -1652,17 +1660,45 @@ func throwaway(ctx context.Context, run, plain Runner, name, image string, port
 
 // waitFor runs a readiness command in a container until it answers, for a minute.
 func waitFor(ctx context.Context, run Runner, name string, ready []string) error {
-	for i := 0; i < 60; i++ {
+	return waitReady(ctx, run, name, ready, 60)
+}
+
+// waitReady is waitFor for a number of tries, a second apart. A container that never answers is not left a mystery
+// (novox/hq issue 1834): its state and exit code and the last 20 lines of its log are in the error, read before the
+// check removes it.
+func waitReady(ctx context.Context, run Runner, name string, ready []string, attempts int) error {
+	for i := 0; i < attempts; i++ {
 		if _, err := run(ctx, "", "docker", append([]string{"exec", name}, ready...)...); err == nil {
 			return nil
 		}
+		if i == attempts-1 {
+			break
+		}
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
 		case <-time.After(time.Second):
 		}
 	}
-	return fmt.Errorf("%s never became ready", name)
+	return fmt.Errorf("%s never became ready%s", name, whyItStopped(ctx, run, name))
+}
+
+// whyItStopped is ": <state>, exit code N; its last log lines: …" for a container, or "" when docker says nothing.
+func whyItStopped(ctx context.Context, run Runner, name string) string {
+	reading, done := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+	defer done()
+	var said []string
+	if out, err := run(reading, "", "docker", "inspect", "--format",
+		"{{.State.Status}}, exit code {{.State.ExitCode}}", name); err == nil && strings.TrimSpace(out) != "" {
+		said = append(said, strings.TrimSpace(out))
+	}
+	if out, err := run(reading, "", "docker", "logs", "--tail", "20", name); err == nil && strings.TrimSpace(out) != "" {
+		said = append(said, "its last log lines: "+strings.TrimSpace(out))
+	}
+	if len(said) == 0 {
+		return ""
+	}
+	return ": " + strings.Join(said, "; ")
 }
 
 // dialable waits for an address to take a connection, for a minute.