Hold a plan whose backup failed or stayed silent, say it, and ask again by a verb (issue 1066)

cmd/mesh-controller/plan_backups.go

@@ -7,11 +7,13 @@ import (
 	"fmt"
 	"slices"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/nats-io/nats.go"
 	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
 	"github.com/novox/mesh-controller/internal/inventory"
 	"github.com/novox/mesh-controller/internal/link"
 )
@@ -132,6 +134,14 @@ func loadBackupFacts(ctx context.Context, inv *inventory.Inventory) (backupFacts
 // moved modules have no backed-up item keeps no record and goes on at once. An error is an ask that could not be
 // made or a read the store refused: the step is tried again, the asks already made kept.
 func stepBackups(ctx context.Context, p *inventory.Plan, facts backupFacts, now time.Time) (bool, error) {
+	proceed, err := stepBackupsOnce(ctx, p, facts, now)
+	if err == nil {
+		noteBackups(p)
+	}
+	return proceed, err
+}
+
+func stepBackupsOnce(ctx context.Context, p *inventory.Plan, facts backupFacts, now time.Time) (bool, error) {
 	if p.Backups == nil {
 		b := wantedBackups(*p, facts)
 		if b == nil {
@@ -292,3 +302,122 @@ func backupsFirst(ctx context.Context, inv *inventory.Inventory, p *inventory.Pl
 	}
 	return stepBackups(ctx, p, facts, time.Now().UTC())
 }
+
+// backupsNote begins the note of a plan that waits on or is held by its backups.
+const backupsNote = "backups: "
+
+// noteBackups says in the plan's note what its backups wait on, or, held, which backup failed and in what words and
+// the verb that asks again; a note of its own that the backups ended is cleared. Not the "tried again" note of a
+// failed step: a held plan is not retried (novox/hq ADR 0404 decision 4).
+func noteBackups(p *inventory.Plan) {
+	if p.Backups == nil {
+		return
+	}
+	var pending, failed []string
+	for _, a := range p.Backups.Asked {
+		switch a.State {
+		case inventory.BackupPending:
+			pending = append(pending, a.Module+" on "+a.Machine)
+		case inventory.BackupFailed:
+			failed = append(failed, fmt.Sprintf("%s on %s: %s", a.Module, a.Machine, a.Words))
+		}
+	}
+	switch p.Backups.State {
+	case inventory.BackupPending:
+		p.Note = backupsNote + "waiting for the backup of " + strings.Join(pending, ", ")
+	case inventory.BackupFailed:
+		p.Note = backupsNote + "held — the backup of " + strings.Join(failed, "; ") + "; nothing is asked or sent. " +
+			"`delivery-backups-again` asks it again"
+	default:
+		if strings.HasPrefix(p.Note, backupsNote) {
+			p.Note = ""
+		}
+	}
+}
+
+// waitingOnBackups says a plan has not asked its first tier because its backups are pending or failed: no tier
+// late, as S3 and the lines of `plans` read it.
+func waitingOnBackups(p inventory.Plan) bool {
+	return p.Open() && p.Release == nil && p.Tier == 0 && !askedAny(p) && p.Backups != nil &&
+		p.Backups.State != inventory.BackupDone
+}
+
+// askBackupsAgain clears the failed backups of a held plan, so the next step asks those again and no other. It
+// says which. Refused for a plan with no failed backup.
+func askBackupsAgain(p *inventory.Plan, by, why string) ([]inventory.PlanBackup, error) {
+	if !p.Open() || p.Backups == nil || p.Backups.State != inventory.BackupFailed {
+		return nil, fmt.Errorf("%s has no failed backup: there is nothing to ask again", p.ID)
+	}
+	var again []inventory.PlanBackup
+	for i := range p.Backups.Asked {
+		a := &p.Backups.Asked[i]
+		if a.State != inventory.BackupFailed {
+			continue
+		}
+		again = append(again, *a)
+		a.State, a.AskedAt, a.Words = inventory.BackupPending, nil, ""
+	}
+	p.Backups.State = backupsStateOf(p.Backups.Asked)
+	p.Note = backupsNote + "asked again by " + by + ": " + why
+	return again, nil
+}
+
+// backupsAgain is askBackupsAgain for the plan with this id, kept: the verb `delivery-backups-again`.
+func backupsAgain(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, []inventory.PlanBackup, error) {
+	release, err := inv.HoldPlans(ctx, true)
+	if err != nil {
+		return inventory.Plan{}, nil, err
+	}
+	defer release()
+	p, err := inv.PlanByID(ctx, id)
+	if err != nil {
+		return inventory.Plan{}, nil, err
+	}
+	again, err := askBackupsAgain(&p, by, why)
+	if err != nil {
+		return p, nil, err
+	}
+	if err := inv.SavePlan(ctx, &p); err != nil {
+		return p, nil, err
+	}
+	return p, again, nil
+}
+
+// kindBackupHeld is a delivery plan held for a backup that failed or stayed silent; backupHeldProbe its row.
+const (
+	kindBackupHeld  = "backup-held"
+	backupHeldProbe = "DK"
+)
+
+// observeBackupHolds is one condition per plan held for its backups, with the backups' own words as its evidence.
+func observeBackupHolds(plans []inventory.Plan) []conditions.Observation {
+	var out []conditions.Observation
+	for _, p := range plans {
+		if !p.Open() || p.Backups == nil || p.Backups.State != inventory.BackupFailed {
+			continue
+		}
+		var said, machines []string
+		for _, a := range p.Backups.Asked {
+			if a.State == inventory.BackupFailed {
+				said = append(said, fmt.Sprintf("%s on %s: %s", a.Module, a.Machine, a.Words))
+				machines = append(machines, a.Machine)
+			}
+		}
+		out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.ID, Token: "backup-held",
+			Kind: kindBackupHeld, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
+			Machine: machines[0], Also: machines[1:],
+			Summary: fmt.Sprintf("the delivery %s is held: %d of its backups failed or did not answer, so nothing is "+
+				"asked or sent", p.ID, len(said)),
+			Said: strings.Join(said, "; ")})
+	}
+	return out
+}
+
+// probeBackupHeld reads the open plans for the ones held for their backups.
+func probeBackupHeld(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
+	plans, err := d.open.inventory.OpenPlans(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return observeBackupHolds(plans), nil
+}