Read volume sizes only when asked and remove in bounded batches, so docker_volumes_remove_anonymous answers on a machine with over a thousand volumes (issue 1796)

modules/docker/cmd/docker-tools/docker.go

@@ -944,16 +944,9 @@ func (c *Client) Volumes(ctx context.Context, unmountedOnly, sizes bool) (map[st
 	}
 	size := map[string]string{}
 	if sizes {
-		out, err := c.docker(ctx, "system", "df", "--verbose", "--format", "{{json .}}")
-		if err != nil {
+		if size, err = c.volumeSizes(ctx); err != nil {
 			return nil, err
 		}
-		var v struct{ Volumes []map[string]any }
-		if err := json.Unmarshal([]byte(out), &v); err == nil {
-			for _, vol := range v.Volumes {
-				size[fmt.Sprint(vol["Name"])] = fmt.Sprint(vol["Size"])
-			}
-		}
 	}
 	vols := []map[string]any{}
 	if len(lines(names)) > 0 {
@@ -1216,11 +1209,30 @@ func (c *Client) DaemonConfig(ctx context.Context) (map[string]any, error) {
 	return answer, nil
 }
 
+// volumeSizes is every volume's size by name, from docker system df -v, which walks every volume:
+// on a machine with a thousand of them it does not answer within a call's bound.
+func (c *Client) volumeSizes(ctx context.Context) (map[string]string, error) {
+	size := map[string]string{}
+	out, err := c.docker(ctx, "system", "df", "--verbose", "--format", "{{json .}}")
+	if err != nil {
+		return nil, err
+	}
+	var v struct{ Volumes []map[string]any }
+	if err := json.Unmarshal([]byte(out), &v); err == nil {
+		for _, vol := range v.Volumes {
+			size[fmt.Sprint(vol["Name"])] = fmt.Sprint(vol["Size"])
+		}
+	}
+	return size, nil
+}
+
 // RemoveVolumesAsk is what docker_volumes_remove_anonymous is asked to take.
 type RemoveVolumesAsk struct {
 	DryRun        bool
 	Why           string
 	CreatedBefore *time.Time
+	Sizes         bool
+	Limit         int // how many to remove in one call; 0 is 200
 }
 
 // RemoveAnonymousVolumes removes the leftover anonymous volumes (novox/hq issue 1796): those docker
@@ -1232,11 +1244,10 @@ func (c *Client) RemoveAnonymousVolumes(ctx context.Context, a RemoveVolumesAsk)
 	if !a.DryRun && strings.TrimSpace(a.Why) == "" {
 		return nil, fmt.Errorf("why is required when dry_run is false")
 	}
-	all, err := c.Volumes(ctx, false, true)
+	all, err := c.Volumes(ctx, false, false)
 	if err != nil {
 		return nil, err
 	}
-	sizes := map[string]string{}
 	candidates := []string{}
 	skipped := []map[string]any{}
 	skip := func(name, why string) { skipped = append(skipped, map[string]any{"name": name, "why": why}) }
@@ -1253,7 +1264,18 @@ func (c *Client) RemoveAnonymousVolumes(ctx context.Context, a RemoveVolumesAsk)
 			skip(name, "not created before "+a.CreatedBefore.Format(time.RFC3339)+" (created "+fmt.Sprint(v["created"])+")")
 		default:
 			candidates = append(candidates, name)
-			sizes[name] = fmt.Sprint(v["size"])
+		}
+	}
+	answer := map[string]any{"dry_run": a.DryRun, "candidates": len(candidates)}
+	// Sizes are asked for, never needed: the listing walks every volume and may not answer, and
+	// then the call answers without them. Nothing below depends on it.
+	sizes := map[string]string{}
+	if a.Sizes {
+		if got, err := c.volumeSizes(ctx); err != nil {
+			answer["sizes_unknown"] = err.Error()
+		} else {
+			sizes = got
+			answer["sizes"] = sizes
 		}
 	}
 	sum := func(names []string) int64 {
@@ -1263,24 +1285,46 @@ func (c *Client) RemoveAnonymousVolumes(ctx context.Context, a RemoveVolumesAsk)
 		}
 		return t
 	}
-	answer := map[string]any{"dry_run": a.DryRun, "skipped": skipped, "sizes": sizes}
 	if a.DryRun {
+		answer["skipped"] = skipped
 		answer["would_remove"] = candidates
-		answer["would_free_bytes"] = sum(candidates)
-		answer["note"] = "nothing was removed: call again with dry_run false and why to remove them"
+		if len(sizes) > 0 {
+			answer["would_free_bytes"] = sum(candidates)
+		}
+		answer["note"] = "nothing was removed: call again with dry_run false and why to remove them (limit volumes at most per call)"
 		return answer, nil
 	}
+	limit := a.Limit
+	if limit <= 0 {
+		limit = 200
+	}
+	take := candidates
+	if len(take) > limit {
+		take = candidates[:limit]
+	}
 	removed := []string{}
-	for _, name := range candidates {
-		if _, err := c.docker(ctx, "volume", "rm", name); err != nil {
-			skip(name, "docker refused: "+err.Error())
+	// Chunks of a hundred in one `docker volume rm`, so a thousand volumes are ten calls; a chunk
+	// docker refuses any of is done again one by one, so what it refuses is named.
+	for i := 0; i < len(take); i += 100 {
+		chunk := take[i:min(i+100, len(take))]
+		if _, err := c.docker(ctx, append([]string{"volume", "rm"}, chunk...)...); err == nil {
+			removed = append(removed, chunk...)
 			continue
 		}
-		removed = append(removed, name)
+		for _, name := range chunk {
+			if _, err := c.docker(ctx, "volume", "rm", name); err != nil {
+				skip(name, "docker refused: "+err.Error())
+				continue
+			}
+			removed = append(removed, name)
+		}
 	}
 	answer["skipped"] = skipped
 	answer["removed"], answer["count"], answer["why"] = removed, len(removed), a.Why
-	answer["freed_bytes"] = sum(removed)
+	answer["remaining"] = len(candidates) - len(take)
+	if len(sizes) > 0 {
+		answer["freed_bytes"] = sum(removed)
+	}
 	return answer, nil
 }