Fail a backup whose holder cannot be asked at the bound, not retry forever (issue 1066)

cmd/mesh-controller/plan_backups.go

@@ -131,8 +131,8 @@ func loadBackupFacts(ctx context.Context, inv *inventory.Inventory) (backupFacts
 // stepBackups takes one step of a plan's backups and says whether the plan may go on to ask its first tier. It
 // asks what is not asked, reads how the asked ones went, and records it all in p.Backups; it waits (false, nil)
 // while any is pending, and stays stopped (false, nil) once one failed, which part B holds and says. A plan whose
-// moved modules have no backed-up item keeps no record and goes on at once. An error is an ask that could not be
-// made or a read the store refused: the step is tried again, the asks already made kept.
+// moved modules have no backed-up item keeps no record and goes on at once. A holder that cannot be asked is tried again
+// within the bound and then failed; an error is left for what the step itself cannot do.
 func stepBackups(ctx context.Context, p *inventory.Plan, facts backupFacts, now time.Time) (bool, error) {
 	proceed, err := stepBackupsOnce(ctx, p, facts, now)
 	if err == nil {
@@ -164,16 +164,25 @@ func stepBackupsOnce(ctx context.Context, p *inventory.Plan, facts backupFacts,
 		}
 		if a.AskedAt == nil {
 			asked := now
+			if a.TriedAt == nil {
+				a.TriedAt = &asked
+			}
 			if err := askBackupNow(ctx, a.Machine, a.Module); err != nil {
 				var unasked *backupNotAsked
 				if errors.As(err, &unasked) {
-					return false, err
+					// A holder off the bus is tried again within the bound, by the look and never by the
+					// "tried again" error path; past it the backup is failed and the plan held.
+					a.Words = "could not be asked: " + err.Error()
+					if now.Sub(*a.TriedAt) >= backupsWithin {
+						a.State = inventory.BackupFailed
+					}
+					continue
 				}
 				a.State, a.Words = inventory.BackupFailed, err.Error()
-				a.AskedAt = &asked
+				a.AskedAt, a.TriedAt = &asked, nil
 				continue
 			}
-			a.AskedAt = &asked
+			a.AskedAt, a.TriedAt, a.Words = &asked, nil, ""
 			continue
 		}
 		if _, read := nights[a.Machine]; !read {
@@ -355,7 +364,7 @@ func askBackupsAgain(p *inventory.Plan, by, why string) ([]inventory.PlanBackup,
 			continue
 		}
 		again = append(again, *a)
-		a.State, a.AskedAt, a.Words = inventory.BackupPending, nil, ""
+		a.State, a.AskedAt, a.TriedAt, a.Words = inventory.BackupPending, nil, nil, ""
 	}
 	p.Backups.State = backupsStateOf(p.Backups.Asked)
 	p.Note = backupsNote + "asked again by " + by + ": " + why