Put a broken module back at once, and excuse a wait only for a move that added an account group (hq issue 318 review)

cmd/mesh-controller/release.go

@@ -334,6 +334,9 @@ func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
 		notes = append(notes, p.Note)
 	}
 	done := map[string]bool{except: true}
+	for _, m := range g.Returned {
+		done[m] = true // put back at once when it broke
+	}
 	// **What passed on its own keeps its pass** (novox/hq ADR 0254, issue 318): healthy for the passes the
 	// gate asks while another module of the send failed, it is neither put back nor left unjudged — a build
 	// left without a verdict is one more move every other walk would wait for.
@@ -390,6 +393,65 @@ func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *invent
 	return kept
 }
 
+// putBackBroken puts back, at once, every module a gate found broken and has not put back yet (issue 318
+// review): a witness that reverted a core component, or a machine that refused or failed its send, is not
+// left registered at the build that broke for as long as the modules beside it take to be judged — a resend
+// meanwhile would declare it again. The send goes on judging the others to their own verdicts; its own
+// verdict is still failed. lead and leadState are the module a plan's gate is kept on, when it is a plan's.
+// Answers whether anything was put back.
+func putBackBroken(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, lead string,
+	leadState *inventory.PlanModule) bool {
+	var todo []string
+	for _, m := range g.Broken {
+		if !slices.Contains(g.Returned, m) {
+			todo = append(todo, m)
+		}
+	}
+	if len(todo) == 0 || g.Verdict != "" {
+		return false
+	}
+	stateWas, noteWas := p.State, p.Note
+	batched, back := batchingRollbacks(ctx)
+	var said []string
+	for _, m := range todo {
+		g.Returned = append(g.Returned, m)
+		machines := g.Machines
+		var state *inventory.PlanModule
+		if m == lead && leadState != nil {
+			judged := *leadState
+			judged.Gate = nil // its own record of the failure; the send's gate goes on judging
+			state = &judged
+		} else {
+			i := slices.IndexFunc(g.Carried, func(c inventory.CarriedMove) bool { return c.Module == m })
+			if i < 0 {
+				continue
+			}
+			c := g.Carried[i]
+			state = &inventory.PlanModule{Build: c.Build, Previous: c.From, Commit: c.To}
+			if sent, err := sentTheBuild(ctx, open, m, c.To); err == nil && len(sent) > 0 {
+				machines = sent
+			} else {
+				machines = []string{c.Node}
+			}
+		}
+		p.Note = ""
+		gateFailed(batched, open, p, m, state, machines, g.BrokenWhy)
+		if m == lead && leadState != nil {
+			leadState.Why = "put back at once, its send still judged: " + g.BrokenWhy
+		}
+		said = append(said, m)
+	}
+	sendRollbacks(ctx, open, p, back)
+	p.State = stateWas
+	p.Note = noteWas
+	if len(said) > 0 {
+		p.Note = fmt.Sprintf("put back at once: %s (%s); the rest of the send judged to their own verdicts",
+			strings.Join(said, ", "), g.BrokenWhy)
+		fmt.Printf("%s: %s\n", p.ID, p.Note)
+	}
+	return true
+}
+
 // passedAloneWhy is the verdict of a module that passed on its own while its send failed, with its own wait.
 func passedAloneWhy(g *inventory.PlanGate, module string) string {
 	why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[module], g.Why)
@@ -617,6 +679,9 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
 	}
 	switch verdict {
 	case "":
+		if putBackBroken(ctx, open, p, g, "", nil) {
+			return true, nil
+		}
 		note := fmt.Sprintf("judging %s: %s", strings.Join(g.Machines, ", "), gateLine(g))
 		changed := p.Note != note
 		p.Note = note