Send a plan's tier to each machine once, and blame no module for its machine (hq issue 281)

cmd/mesh-controller/plan_retry.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"slices"
 	"sort"
 	"strings"
 	"time"
@@ -181,7 +182,7 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
 	// **A build that failed its gate is not sent again** (novox/hq ADR 0236): it was put back on its first
 	// machine, and retrying would judge the build the mesh put back, or send the failed one by hand.
 	for _, m := range stopped {
-		if g := p.Modules[m].Gate; g != nil && g.Verdict == inventory.GateFailed {
+		if g := p.Modules[m].Gate; g != nil && g.Verdict == inventory.GateFailed && !noVerdictOnItsBuild(g) {
 			return fmt.Errorf("%s failed its gate on %s (%s) and was put back: a build that failed its gate is not "+
 				"sent again — a newer merge, or `rebuild %s`, makes a new build, judged at the gate again",
 				m, strings.Join(g.Machines, ", "), g.Why, m)
@@ -269,6 +270,9 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
 	if err := retryRefusal(p, plans); err != nil {
 		return "", err
 	}
+	if again := unjudgedAtGate(p); len(failedIn(p)) == 0 && len(again) > 0 {
+		return retryTierWhole(ctx, open, &p, again)
+	}
 	if len(failedIn(p)) == 0 {
 		return retryRollouts(ctx, open, &p)
 	}
@@ -388,17 +392,34 @@ func planHolding(module string, openPlans, recent []inventory.Plan) (inventory.P
 // records that send as the first anew, and sets the plan rolling: from there it goes on as the plan
 // would have — the rest sent once those report they applied it, the next tier after (ADR 0218).
 func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string, error) {
-	var said []string
-	for _, m := range stoppedRollouts(*p) {
-		s := p.Modules[m]
-		sent, err := sendRollout(ctx, open, s.First)
-		if err != nil {
-			return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w",
-				m, strings.Join(s.First, ", "), p.ID, err)
+	// Every machine once, for all the modules stopped there (novox/hq issue 281).
+	stopped := stoppedRollouts(*p)
+	var machines []string
+	for _, m := range stopped {
+		for _, n := range p.Modules[m].First {
+			if !slices.Contains(machines, n) {
+				machines = append(machines, n)
+			}
 		}
-		now := time.Now().UTC()
-		s.First, s.FirstAt, s.Why = sent, &now, ""
-		said = append(said, m+" to "+strings.Join(sent, ", "))
+	}
+	sort.Strings(machines)
+	sent, err := sendRollout(ctx, open, machines)
+	if err != nil {
+		return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w",
+			strings.Join(stopped, ", "), strings.Join(machines, ", "), p.ID, err)
+	}
+	now := time.Now().UTC()
+	var said []string
+	for _, m := range stopped {
+		s := p.Modules[m]
+		var again []string
+		for _, n := range sent {
+			if slices.Contains(s.First, n) {
+				again = append(again, n)
+			}
+		}
+		s.First, s.FirstAt, s.Why = again, &now, ""
+		said = append(said, m+" to "+strings.Join(again, ", "))
 	}
 	p.State = inventory.PlanRolling
 	p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
@@ -408,3 +429,78 @@ func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string
 	return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+
 		"applied, as the plan would have", p.ID, p.Tier, len(p.Tiers), strings.Join(said, "; ")), nil
 }
+
+// noVerdictOnItsBuild says a failed gate said nothing about the module's build (novox/hq issue 281): its
+// send changed nothing of the module there — the machine already ran that build, carried there by an
+// earlier send of the same tier, or one identical to it. Such a module was blamed for its machine.
+func noVerdictOnItsBuild(g *inventory.PlanGate) bool {
+	return g.Rollback == gateUnchanged || (g.From != "" && sameCommit(g.From, g.To))
+}
+
+// unjudgedAtGate is the modules of a plan's current tier stopped at a gate that was no verdict on their
+// build, sorted.
+func unjudgedAtGate(p inventory.Plan) []string {
+	if p.Tier >= len(p.Tiers) {
+		return nil
+	}
+	var out []string
+	for _, m := range p.Tiers[p.Tier] {
+		if s := p.Modules[m]; s != nil && s.Gate != nil && s.Gate.Verdict == inventory.GateFailed && noVerdictOnItsBuild(s.Gate) {
+			out = append(out, m)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// retryTierWhole retries a plan stopped at a gate that judged no build of the module it stopped on
+// (issue 281): that module is asked again under a new id — its old build may be marked failed, and a new
+// verdict is what takes the gate's condition away — and every module of the tier sent first and never
+// passed is sent again, the tier whole, one send per machine, judged again. What passed stays passed.
+func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again []string) (string, error) {
+	inv := open.inventory
+	entries, err := inv.Catalogued(ctx)
+	if err != nil {
+		return "", err
+	}
+	byName := map[string]inventory.Entry{}
+	for _, e := range entries {
+		byName[e.Manifest.Module] = e
+	}
+	var resent []string
+	for _, m := range p.Tiers[p.Tier] {
+		s := p.Modules[m]
+		if s == nil || s.FirstAt == nil || s.SentAt != nil || slices.Contains(again, m) ||
+			(s.Gate != nil && s.Gate.Verdict == inventory.GatePassed) {
+			continue
+		}
+		sendAgain(s)
+		resent = append(resent, m)
+	}
+	var asked []string
+	for _, m := range again {
+		sendAgain(p.Modules[m])
+		askModule(ctx, p, m, byName)
+		if s := p.Modules[m]; s.State == "asked" {
+			asked = append(asked, m+" as "+s.Build)
+		}
+	}
+	if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
+		p.State = inventory.PlanBuilding
+		p.Note = fmt.Sprintf("tier %d retried by hand: %s asked again; %s sent again with the tier", p.Tier,
+			strings.Join(again, ", "), orNone(strings.Join(resent, ", ")))
+	}
+	if err := inv.SavePlan(ctx, p); err != nil {
+		return "", err
+	}
+	if p.State != inventory.PlanBuilding {
+		return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note)
+	}
+	return fmt.Sprintf("%s retried at tier %d of %d: asked %s; %s sent again with the tier, one send per machine, "+
+		"judged again", p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", "), orNone(strings.Join(resent, ", "))), nil
+}
+
+// sendAgain forgets a module's first send, so its plan sends it again.
+func sendAgain(s *inventory.PlanModule) {
+	s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", ""
+}