Send the rest at once after the first passes, and resume a stored gate (hq ADR 0353)

A release now sends every other machine in one step once its first machine
passed, each judged by its own gate; a failing one is put back without
stopping the others, a silent one is left. A send is kept before it is made,
and a machine already sent what a gate is for is not sent again: the gate
counts from that send and takes over another walk's readings of the same
build there.

cmd/mesh-controller/release_plan.go

@@ -595,7 +595,7 @@ func advancePlans(ctx context.Context, open *stores) {
 		return
 	}
 	defer release()
-	// What waits for a gate, released one machine at a time (ADR 0236).
+	// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
 	if _, err := releaseBacklog(ctx, open, ""); err != nil {
 		fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
 	}
@@ -1056,6 +1056,15 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 	runningOf map[string][]string) error {
 	inv := open.inventory
 	before, beforeKnown, _ := inv.SentBuilds(ctx, node)
+	// **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan
+	// before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from
+	// the readings of a gate already judging it there.
+	if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 {
+		if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 {
+			adoptSend(ctx, open, p, node, modules, runningOf)
+			return nil
+		}
+	}
 	owns := make([]inventory.CarriedMove, 0, len(modules))
 	for _, m := range modules {
 		s := p.Modules[m]
@@ -1106,6 +1115,86 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 	return nil
 }
 
+// adoptSend records a first send the machine already had (novox/hq ADR 0353 §6, §7): nothing is sent; the gate counts
+// from the machine's last send, and takes over the readings of an open gate of another walk judging the same build
+// there — whose carried move also says what the machine ran before, for a rollback.
+func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string,
+	runningOf map[string][]string) {
+	inv := open.inventory
+	since := time.Now().UTC()
+	last, found, err := inv.SentTo(ctx, node)
+	if err == nil && found && !last.At.IsZero() {
+		since = last.At
+	}
+	lead := modules[0]
+	other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit)
+	for _, m := range modules {
+		s := p.Modules[m]
+		if m == lead && from != "" {
+			s.Previous = from
+		}
+		at := since
+		s.First, s.FirstAt = []string{node}, &at
+		s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning([]string{node}, runningOf[m]),
+			From: s.Previous, To: s.Commit, Since: &at}
+		if found {
+			s.Gate.Sent = map[string]inventory.SentDeclaration{node: last}
+		}
+		s.GatedBy = ""
+		if m == lead {
+			s.Gate.TakeReadings(other, modules, id)
+		} else {
+			s.GatedBy = lead
+		}
+	}
+	p.State = inventory.PlanRolling
+	p.Note = fmt.Sprintf("tier %d built; %s already runs %s's build: nothing sent again, judged from that send",
+		p.Tier, node, strings.Join(modules, ", "))
+	if id != "" {
+		p.Note += ", going on from the readings of " + id
+	}
+	fmt.Printf("%s: %s\n", p.ID, p.Note)
+}
+
+// judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the
+// walk's id, and what the machine ran before; nil when none does.
+func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit string) (*inventory.PlanGate, string, string) {
+	plans, err := inv.OpenPlans(ctx)
+	if err != nil {
+		return nil, "", ""
+	}
+	usable := func(g *inventory.PlanGate) bool {
+		return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) &&
+			slices.Contains(g.Machines, node)
+	}
+	for _, q := range plans {
+		if q.ID == self {
+			continue
+		}
+		if r := q.Release; r != nil {
+			gates := []*inventory.PlanGate{r.Gate}
+			for _, g := range r.Gates {
+				gates = append(gates, g)
+			}
+			for _, g := range gates {
+				if !usable(g) {
+					continue
+				}
+				for _, c := range g.Carried {
+					if c.Module == module && c.Node == node && sameCommit(c.To, commit) {
+						return g, q.ID, c.From
+					}
+				}
+			}
+			continue
+		}
+		if s := q.Modules[module]; s != nil && usable(s.Gate) && sameCommit(s.Commit, commit) {
+			return s.Gate, q.ID, s.Previous
+		}
+	}
+	return nil, "", ""
+}
+
 // passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
 // on the first of them passed, and its pass was kept for every build it carried (passCarried).
 func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
@@ -1594,9 +1683,19 @@ func plansCommand(ctx context.Context, args []string) error {
 			// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
 			fmt.Printf("  machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
 				orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", ")))
-			if r.Gate != nil {
-				fmt.Printf("  %s\n", gateLine(r.Gate))
-				for _, c := range r.Gate.Carried {
+			if len(r.Failed)+len(r.Silent) > 0 {
+				fmt.Printf("  failed: %s; silent: %s\n", orNone(strings.Join(r.Failed, ", ")), orNone(strings.Join(r.Silent, ", ")))
+			}
+			gates := []*inventory.PlanGate{r.Gate}
+			for _, n := range r.Order {
+				gates = append(gates, r.Gates[n])
+			}
+			for _, g := range gates {
+				if g == nil {
+					continue
+				}
+				fmt.Printf("  %s\n", gateLine(g))
+				for _, c := range g.Carried {
 					fmt.Printf("    %-22s %s → %s\n", c.Module, short(c.From), short(c.To))
 					if c.Recreates != "" {
 						fmt.Printf("    %-22s %s\n", "", c.Recreates)