Remove a build's throwaway containers with their anonymous volumes, so a build node's disk does not fill (issue 1772)

internal/builder/kill.go

@@ -3,6 +3,7 @@ package builder
 import (
 	"context"
 	"os/exec"
+	"regexp"
 	"strings"
 	"syscall"
 	"time"
@@ -71,12 +72,45 @@ func RemoveContainersOf(ctx context.Context, run Runner, id string) (int, error)
 	if len(ids) == 0 {
 		return 0, nil
 	}
-	if _, err := run(ctx, "", "docker", append([]string{"rm", "-f"}, ids...)...); err != nil {
+	if err := removeWithVolumes(ctx, run, ids); err != nil {
 		return 0, err
 	}
 	return len(ids), nil
 }
 
+// MountedVolumes is the template `docker inspect --format` reads the names of a container's volumes with.
+const MountedVolumes = `{{range .Mounts}}{{if eq .Type "volume"}}{{println .Name}}{{end}}{{end}}`
+
+// anonymousVolume is the name docker gives a volume it makes for an image's VOLUME line: 64 hex digits.
+var anonymousVolume = regexp.MustCompile(`^[0-9a-f]{64}$`)
+
+// removeWithVolumes removes containers together with their anonymous volumes (novox/hq issue 1772). A throwaway
+// store declares its data directory a VOLUME, docker makes an anonymous volume for it, and `docker rm -f`
+// without -v leaves that volume for ever: 1,629 of them, 107 GB, on one build node. So the volumes the
+// containers mount are read first, the containers are removed with -v, and an anonymous volume still there
+// afterwards is removed by name. Only a volume one of these containers mounted, and only one with an anonymous
+// volume's name, is touched: a named volume — an operator's own data — never is. A volume already gone is
+// no failure.
+func removeWithVolumes(ctx context.Context, run Runner, ids []string) error {
+	var volumes []string
+	if out, err := run(ctx, "", "docker", append([]string{"inspect", "--format", MountedVolumes}, ids...)...); err == nil {
+		seen := map[string]bool{}
+		for _, name := range strings.Fields(out) {
+			if anonymousVolume.MatchString(name) && !seen[name] {
+				seen[name] = true
+				volumes = append(volumes, name)
+			}
+		}
+	}
+	if _, err := run(ctx, "", "docker", append([]string{"rm", "-f", "-v"}, ids...)...); err != nil {
+		return err
+	}
+	if len(volumes) > 0 {
+		_, _ = run(ctx, "", "docker", append([]string{"volume", "rm"}, volumes...)...)
+	}
+	return nil
+}
+
 // A holder that stops mid-build leaves its containers running (novox/hq issue 306).
 //
 // A build's containers are removed by the build itself, by a kill, or by the next delivery of the same
@@ -106,7 +140,7 @@ func RemoveLeftBehind(ctx context.Context, run Runner) (int, error) {
 	if len(ids) == 0 {
 		return 0, nil
 	}
-	if _, err := run(ctx, "", "docker", append([]string{"rm", "-f"}, ids...)...); err != nil {
+	if err := removeWithVolumes(ctx, run, ids); err != nil {
 		return 0, err
 	}
 	return len(ids), nil