Merge pull request 'The rest go at once after the first passes, and a gate survives a new controller (hq ADR 0353)' (#312) from fix/the-rest-in-parallel-and-gates-survive-a-new-controller into main

cmd/mesh-controller/gate.go

@@ -1443,6 +1443,12 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error
 	}
 	// A release held after a failed one, while builds still wait for a gate (ADR 0236).
 	out = append(out, backlogObservation()...)
+	// And a rest that went silent (novox/hq ADR 0353 §4).
+	silent, err := silentObservation(ctx, d.open.inventory)
+	if err != nil {
+		return nil, err
+	}
+	out = append(out, silent...)
 	return sortedFound(dedupeObservations(out)), nil
 }
 
@@ -1532,6 +1538,12 @@ func gateLine(g *inventory.PlanGate) string {
 	case inventory.GateFailed:
 		what = "FAILED"
 	}
+	switch {
+	case g.Verdict == "" && g.Pending:
+		what = "kept, not yet sent" // novox/hq ADR 0353 §8
+	case g.Verdict == "" && g.Silent:
+		what = "silent on its send, left" // ADR 0353 §4
+	}
 	if g.Verdict == "" && g.Waiting() {
 		what = "waits for a busy container on " + g.Busy.Machine
 	}

cmd/mesh-controller/held_back.go

@@ -151,7 +151,7 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
 		for _, mv := range f.moves(node, modules, same, known, false) {
 			if planStillToSend(plans, mv.Module, node) == "" {
 				why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
-					"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
+					"plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
 			}
 		}
 		if len(why) > 0 {

cmd/mesh-controller/plain_words.go

@@ -456,6 +456,9 @@ var plainWordings = map[string]func(conditions.Observation) words{
 	"release-held": worded(func(o conditions.Observation) words {
 		return releaseHeldWords(nil, o.Also)
 	}),
+	"release-silent": worded(func(o conditions.Observation) words {
+		return releaseSilentWords(o.Also)
+	}),
 	"facts-stale": worded(func(o conditions.Observation) words {
 		return words{Headline: "Merge checks use outdated facts",
 			Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
@@ -841,6 +844,19 @@ func releaseHeldWords(modules, machines []string) words {
 		Resolved: "Resolved: the held updates are released"}
 }
 
+// releaseSilentWords are the plain words of machines that said nothing on an update they were sent (ADR 0353 §4).
+func releaseSilentWords(machines []string) words {
+	where := "Some machines"
+	if len(machines) > 0 {
+		where = conditions.Capital(namesWords(machines, 4))
+	}
+	return words{Headline: "Machines did not answer an update",
+		Needs: "check that they are awake and on the mesh; the next walk sends them what waits.",
+		Explanation: where + " were sent module updates and said nothing about them within fifteen minutes, so " +
+			"they were left as they were. What they run is not known to be the update.",
+		Resolved: "Resolved: the next walk reached them"}
+}
+
 // reloginNeeds is what an account waiting for its groups needs (ADR 0252).
 func reloginNeeds(node string) string {
 	return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)

cmd/mesh-controller/release.go

@@ -39,8 +39,8 @@ import (
 //     from the same manifest is no move.
 //
 // **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no
-// started plan is walking, the mesh opens one — every such machine, one at a time, the control node last,
-// each sent and judged before the next. A release plan that fails stops the next one opening on its own:
+// started plan is walking, the mesh opens one — every such machine, the first sent and judged, then the rest
+// at once, each judged by its own gate (novox/hq ADR 0353, in place of one at a time). A release plan that fails stops the next one opening on its own:
 // a person releases it again (`upgrade release-backlog --why`), after the condition says why.
 
 // sendScope is what a send may carry that no gate has seen.
@@ -81,6 +81,15 @@ type moveFacts struct {
 	srcs   map[string]map[string]string
 	passed map[string]map[string]bool
 	plans  []inventory.Plan
+	// self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353).
+	self string
+}
+
+type walkingKey struct{}
+
+// withWalking says a release plan is the one sending: its own gates never hold its own sends back.
+func withWalking(ctx context.Context, id string) context.Context {
+	return context.WithValue(ctx, walkingKey{}, id)
 }
 
 func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
@@ -99,6 +108,7 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er
 		return f, err
 	}
 	f.plans, err = inv.OpenPlans(ctx)
+	f.self, _ = ctx.Value(walkingKey{}).(string)
 	return f, err
 }
 
@@ -193,17 +203,24 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string,
 // feet. A release keeps no module records: its open gate's carried moves are its walk.
 func (f moveFacts) walkedBy(module, node, to string) string {
 	for _, p := range f.plans {
-		if !p.Open() {
+		if !p.Open() || (p.Release != nil && p.ID == f.self) {
 			continue
 		}
 		if p.Release != nil {
-			g := p.Release.Gate
-			if g == nil || g.Verdict != "" {
-				continue
+			// The first machine's gate, and every gate of the rest still judging (novox/hq ADR 0353).
+			gates := []*inventory.PlanGate{p.Release.Gate}
+			for _, g := range p.Release.Gates {
+				gates = append(gates, g)
 			}
-			for _, c := range g.Carried {
-				if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
-					return p.ID
+			for _, g := range gates {
+				// A send kept and not yet made walks nothing yet (ADR 0353 §8).
+				if g == nil || g.Verdict != "" || g.Silent || g.Pending {
+					continue
+				}
+				for _, c := range g.Carried {
+					if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
+						return p.ID
+					}
 				}
 			}
 			continue
@@ -303,8 +320,8 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
 		}
 	}
 	if len(refused) > 0 {
-		return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+
-			"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
+		return nil, fmt.Errorf("%w: %s — a walk sends them, one machine first and then the rest at once, each judged (novox/hq ADR "+
+			"0236, ADR 0353); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
 	}
 	return kept, nil
 }
@@ -320,19 +337,35 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
 // And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490):
 // answered for the gate to keep, nil when there was none.
 func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) {
-	inv := open.inventory
-	f, err := readMoveFacts(ctx, inv)
+	moves, err := gatedMoves(ctx, open, node, owns)
+	if err != nil || len(moves) == 0 {
+		return nil, nil, nil, err
+	}
+	sayRecreations(ctx, open, moves)
+	sent, grants, err := sendJudged(ctx, open, node)
 	if err != nil {
 		return nil, nil, nil, err
 	}
+	return moves, sent, grants, nil
+}
+
+// gatedMoves is what a gated send to one machine would carry, with the build each moves to, made of nothing:
+// gatedSend's reading, which a walk also keeps before it sends (novox/hq ADR 0353 §8). Refused the same way
+// while a plan that has started walks one of those builds elsewhere.
+func gatedMoves(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, error) {
+	inv := open.inventory
+	f, err := readMoveFacts(ctx, inv)
+	if err != nil {
+		return nil, err
+	}
 	moves, err := machineMoves(ctx, open, f, node, true)
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, err
 	}
 	// A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module
 	// kept over an older record judged as moving back to it would read the gate as superseded by its own send.
 	if moves, err = withoutKept(ctx, open, node, moves); err != nil {
-		return nil, nil, nil, err
+		return nil, err
 	}
 	own := func(module string) bool {
 		return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
@@ -342,7 +375,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
 			continue
 		}
 		if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
-			return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
+			return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
 				mv.Module, short(mv.To), node, id)
 		}
 	}
@@ -350,7 +383,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
 	// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
 	for _, o := range owns {
 		if id := f.walkedBy(o.Module, node, o.To); id != "" {
-			return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
+			return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
 				o.Module, short(o.To), node, id)
 		}
 	}
@@ -363,30 +396,26 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
 			moves[i].Build = o.Build
 		}
 	}
-	if len(moves) == 0 && len(owns) == 0 {
-		return nil, nil, nil, nil
+	if len(moves) == 0 {
+		return nil, nil
 	}
 	for i := range moves {
 		if moves[i].Build == "" {
 			if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
-				return nil, nil, nil, err
+				return nil, err
 			}
 		}
 	}
 	sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
-	sayRecreations(ctx, open, moves)
-	sent, grants, err := sendJudged(ctx, open, node)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-	return moves, sent, grants, nil
+	return moves, nil
 }
 
 // passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
 func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
 	seen := map[string]bool{}
 	for _, c := range g.Carried {
-		if c.Module == except || c.Build == "" || seen[c.Build] {
+		// A build put back with another machine's failure is not this gate's to pass (novox/hq ADR 0353 §3).
+		if c.Module == except || c.Build == "" || seen[c.Build] || slices.Contains(g.Returned, c.Module) {
 			continue
 		}
 		seen[c.Build] = true
@@ -695,7 +724,7 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
 	p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository,
 		Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{},
 		Release: &inventory.PlanRelease{Order: order, By: by},
-		Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine at a time, each judged", len(names),
+		Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine first, then the rest at once, each judged", len(names),
 			strings.Join(order, ", "))}
 	if err := inv.SavePlan(ctx, &p); err != nil {
 		return nil, err
@@ -704,14 +733,23 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
 	return &p, nil
 }
 
-// advanceRelease takes one step of a release plan: the next machine sent everything waiting there under a
-// gate, or the machine being judged judged once more; the plan done when every machine is.
+// advanceRelease takes one step of a release plan: the first machine sent everything waiting there under a
+// gate and judged; once it passed, every other machine sent at once, each under its own gate (novox/hq ADR
+// 0353 §1), and judged; the plan done when every machine is.
+//
+// **A send is kept before it is made** (ADR 0353 §8): the step that chooses a machine keeps its gate pending with
+// what it will carry, and the next step makes the send. A controller replaced between the send and keeping it
+// finds the gate pending, reads the machine's last send, and neither sends again nor leaves the machine unjudged.
 func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
 	r := p.Release
 	if r.Settings != nil {
 		return advanceSettingsWalk(ctx, open, p)
 	}
+	ctx = withWalking(ctx, p.ID)
 	now := time.Now().UTC()
+	if r.Rest {
+		return advanceRest(ctx, open, p, now)
+	}
 	if r.Gate == nil {
 		heard, err := releaseHeard(ctx, open)
 		if err != nil {
@@ -724,7 +762,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
 				r.Next++
 				continue
 			}
-			moves, sent, grants, err := gatedSend(ctx, open, node, nil)
+			moves, err := gatedMoves(ctx, open, node, nil)
 			if errors.Is(err, errWalkedElsewhere) {
 				note := fmt.Sprintf("waiting before %s: %v", node, err)
 				changed := p.Note != note
@@ -732,23 +770,15 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
 				return changed, nil
 			}
 			if err != nil {
-				return false, fmt.Errorf("sending %s what waits there: %w", node, err)
+				return false, fmt.Errorf("reading what waits on %s: %w", node, err)
 			}
 			if len(moves) == 0 {
 				r.Done = append(r.Done, node)
 				r.Next++
 				continue
 			}
-			r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent),
-				Grants: grants}
-			p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
-			if said := grantsSaid(grants); said != "" {
-				p.Note += "; " + said
-			}
-			if said := recreationsSaid(moves); said != "" {
-				p.Note += "; " + said
-			}
-			fmt.Printf("%s: %s\n", p.ID, p.Note)
+			r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true}
+			p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves))
 			return true, nil
 		}
 		p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
@@ -759,6 +789,19 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
 		return true, nil
 	}
 	g := r.Gate
+	if g.Pending {
+		node := firstOf(g.Machines)
+		out, err := sendKept(ctx, open, p, node, g, now)
+		if err != nil || out == keptWaiting {
+			return out == keptWaiting, err
+		}
+		if out == keptNothing {
+			r.Gate = nil
+			r.Done = append(r.Done, node)
+			r.Next++
+		}
+		return true, nil
+	}
 	verdict, err := judgeMoves(ctx, open, g, nil, now)
 	if err != nil {
 		return false, err
@@ -777,6 +820,11 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
 		r.Done = append(r.Done, firstOf(g.Machines))
 		r.Next++
 		r.Gate = nil
+		// **Then the rest at once** (novox/hq ADR 0353 §1): every other machine heard from, each judged by its own
+		// gate, in place of one machine at a time (ADR 0236 §22).
+		r.Rest = true
+		p.Note = fmt.Sprintf("passed on %s; the rest are sent at once, each judged", firstOf(g.Machines))
+		fmt.Printf("%s: %s\n", p.ID, p.Note)
 		return true, nil
 	case inventory.GateSuperseded:
 		// Another send moved a judged module on the judged machine (novox/hq issue 352): no verdict on what
@@ -797,6 +845,453 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
 	return true, nil
 }
 
+// What sendKept did with a kept send.
+const (
+	keptSent    = iota // sent now
+	keptAlready        // the machine was already sent it: judged from that send, nothing sent
+	keptNothing        // nothing waits there any more: no gate
+	keptWaiting        // another walk judges a build it would carry: it waits
+)
+
+// sendKept makes a send a gate kept before it was made (novox/hq ADR 0353 §8) — unless the machine's last send
+// already carries what the gate is for, made before a controller was replaced and before it was kept: then
+// nothing is sent, and the gate judges from that send (§6).
+func sendKept(ctx context.Context, open *stores, p *inventory.Plan, node string, g *inventory.PlanGate,
+	now time.Time) (int, error) {
+	inv := open.inventory
+	moves, sent, grants, err := gatedSend(ctx, open, node, nil)
+	if errors.Is(err, errWalkedElsewhere) {
+		p.Note = fmt.Sprintf("waiting before %s: %v", node, err)
+		return keptWaiting, nil
+	}
+	if err != nil {
+		return keptSent, fmt.Errorf("sending %s what waits there: %w", node, err)
+	}
+	if len(moves) > 0 {
+		g.Machines, g.Carried, g.Since, g.Grants, g.Pending = sent, moves, &now, grants, false
+		g.Sent = sentNow(ctx, inv, sent)
+		p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
+		if said := grantsSaid(grants); said != "" {
+			p.Note += "; " + said
+		}
+		if said := recreationsSaid(moves); said != "" {
+			p.Note += "; " + said
+		}
+		fmt.Printf("%s: %s\n", p.ID, p.Note)
+		return keptSent, nil
+	}
+	if !alreadySent(ctx, inv, node, g.Carried) {
+		return keptNothing, nil
+	}
+	since := now
+	if s, found, err := inv.SentTo(ctx, node); err == nil && found {
+		if !s.At.IsZero() {
+			since = s.At
+		}
+		g.Sent = map[string]inventory.SentDeclaration{node: s}
+	}
+	g.Since, g.Pending = &since, false
+	p.Note = fmt.Sprintf("%s was already sent the %d build(s) kept for it, at %s: judged from that send, nothing sent again",
+		node, len(g.Carried), since.Local().Format("15:04:05"))
+	fmt.Printf("%s: %s\n", p.ID, p.Note)
+	return keptAlready, nil
+}
+
+// alreadySent says a machine's last send carries every build of the moves: a send made and not kept.
+func alreadySent(ctx context.Context, inv *inventory.Inventory, node string, moves []inventory.CarriedMove) bool {
+	if len(moves) == 0 {
+		return false
+	}
+	sent, known, err := inv.SentBuilds(ctx, node)
+	if err != nil || !known {
+		return false
+	}
+	for _, mv := range moves {
+		if !sameCommit(sent[mv.Module], mv.To) {
+			return false
+		}
+	}
+	return true
+}
+
+// advanceRest takes one step of a release plan's rest (novox/hq ADR 0353): the sends kept for a step made, every
+// machine of the step judged by its own gate, one that fails put back by itself while the others go on to their
+// verdicts, and the next step — or the end — once every gate of the step has its verdict or is silent.
+func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.Time) (bool, error) {
+	r := p.Release
+	// The sends kept for this step, in the plan's order: the control node, last in it, last (§1).
+	var pending []string
+	for _, n := range r.Order {
+		if g := r.Gates[n]; g != nil && g.Pending {
+			pending = append(pending, n)
+		}
+	}
+	if len(pending) > 0 {
+		// **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on
+		// a machine since its step was chosen could reach a second machine in it; the step is chosen again.
+		if grew, err := keptStepGrew(ctx, open, r, pending); err != nil {
+			return false, err
+		} else if grew != "" {
+			for _, n := range pending {
+				delete(r.Gates, n)
+			}
+			opened, waiting, err := openRestStep(ctx, open, p)
+			if err != nil {
+				return true, err
+			}
+			p.Note = "chosen again: " + grew
+			if !opened && waiting != "" {
+				p.Note += "; " + waiting
+			}
+			return true, nil
+		}
+		var sent, already, waiting []string
+		for _, node := range pending {
+			out, err := sendKept(ctx, open, p, node, r.Gates[node], now)
+			if err != nil {
+				return true, err
+			}
+			switch out {
+			case keptSent:
+				sent = append(sent, node)
+			case keptAlready:
+				already = append(already, node)
+			case keptNothing:
+				delete(r.Gates, node)
+				r.Done = append(r.Done, node)
+			case keptWaiting:
+				waiting = append(waiting, node)
+			}
+		}
+		var said []string
+		if len(sent) > 0 {
+			said = append(said, "sent "+strings.Join(sent, ", ")+" at once, each judged there")
+		}
+		if len(already) > 0 {
+			said = append(said, strings.Join(already, ", ")+" already sent it: judged from that send")
+		}
+		if len(waiting) > 0 {
+			said = append(said, "waiting before "+strings.Join(waiting, ", ")+": another walk judges a build it carries")
+		}
+		if len(said) > 0 {
+			p.Note = strings.Join(said, "; ")
+		}
+		return len(sent)+len(already) > 0 || len(waiting) < len(pending), nil
+	}
+	reports, err := open.inventory.LastReports(ctx)
+	if err != nil {
+		return false, err
+	}
+	byNode := map[string]inventory.Reported{}
+	for _, rep := range reports {
+		byNode[rep.Node] = rep
+	}
+	changed, noteWas := false, p.Note
+	var notes, judging []string
+	batched, back := batchingRollbacks(ctx)
+	for _, node := range r.Order {
+		g := r.Gates[node]
+		if g == nil || g.Verdict != "" || g.Silent {
+			continue
+		}
+		// What another machine's failure already put back is not this gate's to judge (ADR 0353 §3).
+		if len(g.Carried) > 0 && len(g.Returned) >= len(g.Carried) {
+			decide(g, inventory.GateFailed, "what it was sent was put back with another machine's failure", now)
+			r.Failed = append(r.Failed, node)
+			changed = true
+			continue
+		}
+		// **A machine that says nothing on its send is left silent, not failed** (ADR 0353 §4): it is not judged
+		// until it reports, and past ApplySilentAfter it is listed and left.
+		if !reportedOnSend(byNode[node], node, g) {
+			if now.Sub(*g.Since) > inventory.ApplySilentAfter {
+				g.Silent = true
+				r.Silent = append(r.Silent, node)
+				changed = true
+				notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node,
+					inventory.ApplySilentAfter))
+				continue
+			}
+			judging = append(judging, node+": not yet reported on its send")
+			continue
+		}
+		verdict, err := judgeMoves(ctx, open, g, nil, now)
+		if err != nil {
+			return changed, err
+		}
+		switch verdict {
+		case "":
+			if putBackBroken(ctx, open, p, g, "", nil) {
+				changed = true
+				notes = append(notes, p.Note)
+			}
+			judging = append(judging, node+": "+gateLine(g))
+		case inventory.GatePassed:
+			passCarried(ctx, open, p, g, "")
+			r.Done = append(r.Done, node)
+			changed = true
+		case inventory.GateSuperseded:
+			// Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on
+			// it; what still waits is released again.
+			r.Done = append(r.Done, node)
+			changed = true
+			notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why))
+		default:
+			// **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other
+			// machine of the step goes on to its own verdict.
+			stateWas := p.State
+			p.Note = ""
+			failCarried(batched, open, p, g, "")
+			p.State = stateWas
+			r.Failed = append(r.Failed, node)
+			changed = true
+			notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note))
+			returnedElsewhere(ctx, open, r, node, g)
+		}
+	}
+	p.Note = ""
+	sendRollbacks(ctx, open, p, back)
+	if p.Note != "" {
+		notes = append(notes, strings.TrimPrefix(p.Note, "; "))
+	}
+	if len(judging) > 0 {
+		note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ")
+		p.Note = note
+		return changed || noteWas != note, nil
+	}
+	// Every gate of the step has its verdict or is silent: the next step, unless a machine failed (§3).
+	if len(r.Failed) == 0 {
+		opened, waiting, err := openRestStep(ctx, open, p)
+		if err != nil {
+			return changed, err
+		}
+		if opened {
+			return true, nil
+		}
+		if waiting != "" {
+			note := strings.Join(append(notes, waiting), "; ")
+			p.Note = note
+			return changed || noteWas != note, nil
+		}
+	}
+	p.Tier = len(p.Tiers)
+	end := fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", ")))
+	if len(r.Silent) > 0 {
+		end += "; silent on their send, left: " + strings.Join(r.Silent, ", ")
+	}
+	if len(r.Skipped) > 0 {
+		end += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
+	}
+	p.State = inventory.PlanDone
+	if len(r.Failed) > 0 {
+		p.State = inventory.PlanFailed
+		end = "failed on " + strings.Join(r.Failed, ", ") + "; " + end
+	} else if len(r.Silent) > 0 && len(r.Gates) == len(r.Silent) {
+		// **A rest that said nothing is no success** (ADR 0353 §4): failure is loud; the condition says it.
+		p.State = inventory.PlanFailed
+		end = "no machine of the rest reported on its send; " + end
+	}
+	p.Note = strings.Join(append(notes, end), "; ")
+	fmt.Printf("%s: %s\n", p.ID, p.Note)
+	return true, nil
+}
+
+// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with;
+// empty when none would.
+func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) {
+	f, err := readMoveFacts(ctx, open.inventory)
+	if err != nil {
+		return "", err
+	}
+	shelf, err := open.inventory.Catalogue(ctx)
+	if err != nil {
+		return "", err
+	}
+	for _, node := range pending {
+		moves, err := gatedMoves(ctx, open, node, nil)
+		if errors.Is(err, errWalkedElsewhere) {
+			continue // sendKept says it waits
+		}
+		if err != nil {
+			return "", err
+		}
+		kept := r.Gates[node].Carried
+		for _, mv := range moves {
+			// The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9).
+			if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool {
+				return c.Module == mv.Module && sameCommit(c.To, mv.To)
+			}) {
+				continue
+			}
+			return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil
+		}
+	}
+	return "", nil
+}
+
+// reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the
+// declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last.
+func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool {
+	if r.At == nil {
+		return false
+	}
+	if s, kept := g.Sent[node]; kept {
+		return s.ReportsOn(r)
+	}
+	return r.Current
+}
+
+// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's
+// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3).
+func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) {
+	for _, c := range g.Carried {
+		if c.Build == "" {
+			continue
+		}
+		if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked {
+			continue
+		}
+		for node, other := range r.Gates {
+			if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) {
+				continue
+			}
+			for _, oc := range other.Carried {
+				if oc.Module == c.Module && sameCommit(oc.To, c.To) {
+					other.Returned = append(other.Returned, c.Module)
+				}
+			}
+		}
+	}
+}
+
+// openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2):
+// every machine left that is heard from and has something waiting, except that a build no gate has passed goes
+// to one machine of the step only. Answers whether a step was kept, or why the walk waits.
+func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, string, error) {
+	r := p.Release
+	heard, err := releaseHeard(ctx, open)
+	if err != nil {
+		return false, "", err
+	}
+	f, err := readMoveFacts(ctx, open.inventory)
+	if err != nil {
+		return false, "", err
+	}
+	waiting := map[string][]inventory.CarriedMove{}
+	var nodes, walked []string
+	for _, node := range r.Order[min(r.Next, len(r.Order)):] {
+		if r.Gates[node] != nil || slices.Contains(r.Done, node) || slices.Contains(r.Skipped, node) ||
+			slices.Contains(r.Failed, node) || slices.Contains(r.Silent, node) {
+			continue
+		}
+		if !heard[node] {
+			r.Skipped = append(r.Skipped, node)
+			continue
+		}
+		moves, err := gatedMoves(ctx, open, node, nil)
+		if errors.Is(err, errWalkedElsewhere) {
+			walked = append(walked, fmt.Sprintf("%s (%v)", node, err))
+			continue
+		}
+		if err != nil {
+			return false, "", fmt.Errorf("reading what waits on %s: %w", node, err)
+		}
+		if len(moves) == 0 {
+			r.Done = append(r.Done, node)
+			continue
+		}
+		waiting[node] = moves
+		nodes = append(nodes, node)
+	}
+	shelf, err := open.inventory.Catalogue(ctx)
+	if err != nil {
+		return false, "", err
+	}
+	step := restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) })
+	if len(step) == 0 {
+		if len(walked) > 0 {
+			return false, "waiting before " + strings.Join(walked, "; "), nil
+		}
+		return false, "", nil
+	}
+	if r.Gates == nil {
+		r.Gates = map[string]*inventory.PlanGate{}
+	}
+	for _, node := range step {
+		r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true}
+	}
+	p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", "))
+	if left := len(nodes) - len(step); left > 0 {
+		p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left)
+	}
+	return true, "", nil
+}
+
+// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the
+// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus.
+func oneAtATime(module string, m catalogue.Manifest) bool {
+	if module == hostModule {
+		return true
+	}
+	return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat })
+}
+
+// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless
+// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's
+// first machine. Pure.
+//
+// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a
+// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus
+// shows only as its silence and no rollback reaches it.
+func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool,
+	alone func(module string) bool) []string {
+	first := map[string]string{}
+	var out []string
+	for _, n := range nodes {
+		var unpassed []string
+		for _, mv := range waiting[n] {
+			if !passed(mv.Module, mv.To) || alone(mv.Module) {
+				unpassed = append(unpassed, mv.Module+"@"+mv.To)
+			}
+		}
+		if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) {
+			continue
+		}
+		for _, k := range unpassed {
+			first[k] = n
+		}
+		out = append(out, n)
+	}
+	return out
+}
+
+// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353
+// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing.
+func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) {
+	recent, err := inv.RecentPlans(ctx, 20)
+	if err != nil {
+		return nil, err
+	}
+	for _, p := range recent {
+		if p.Release == nil || p.Release.Settings != nil || p.Open() {
+			continue
+		}
+		if len(p.Release.Silent) == 0 {
+			return nil, nil
+		}
+		machines := append([]string(nil), p.Release.Silent...)
+		sort.Strings(machines)
+		o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "silent", Kind: "release-silent",
+			Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines,
+			Summary: fmt.Sprintf("%s: %s said nothing on the send of the builds that waited for a gate within %s, and were "+
+				"left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)}
+		w := releaseSilentWords(machines)
+		o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs
+		return []conditions.Observation{o}, nil
+	}
+	return nil, nil
+}
+
 // backlogObservation is what the gate's probe says of a release held after a failure.
 func backlogObservation() []conditions.Observation {
 	if backlogNow.held == "" || len(backlogNow.waiting) == 0 {

cmd/mesh-controller/release_plan.go

@@ -595,7 +595,7 @@ func advancePlans(ctx context.Context, open *stores) {
 		return
 	}
 	defer release()
-	// What waits for a gate, released one machine at a time (ADR 0236).
+	// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
 	if _, err := releaseBacklog(ctx, open, ""); err != nil {
 		fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
 	}
@@ -1056,10 +1056,36 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 	runningOf map[string][]string) error {
 	inv := open.inventory
 	before, beforeKnown, _ := inv.SentBuilds(ctx, node)
+	// **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan
+	// before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from
+	// the readings of a gate already judging it there.
+	// Only a send the machine reported it applied is one it has; one it never reported on is sent, as before
+	// (issue 280: what it ran before is then that build, and a failure leaves it as it was).
+	if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 &&
+		appliedLastSend(ctx, inv, node) {
+		if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 {
+			adoptSend(ctx, open, p, node, modules, before, runningOf)
+			return nil
+		}
+	}
+	// **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced
+	// between the send and keeping it reads it here, and a failure still puts the machine back.
+	kept := false
+	for _, m := range modules {
+		s := p.Modules[m]
+		if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] {
+			s.Previous, kept = before[m], true
+		}
+	}
+	if kept {
+		if err := inv.SavePlan(ctx, p); err != nil {
+			return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err)
+		}
+	}
 	owns := make([]inventory.CarriedMove, 0, len(modules))
 	for _, m := range modules {
 		s := p.Modules[m]
-		owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: before[m], To: s.Commit, Build: s.Build})
+		owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build})
 	}
 	// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
 	// all of it there.
@@ -1074,10 +1100,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 	sentWhat := sentNow(ctx, inv, sent)
 	for _, m := range modules {
 		s := p.Modules[m]
-		// What the first machine ran before: what a failed gate puts back (ADR 0236).
-		if beforeKnown {
-			s.Previous = before[m]
-		}
+		// What the first machine ran before (ADR 0236), kept above before the send.
 		s.First, s.FirstAt = sent, &now
 		s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]),
 			From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat}
@@ -1106,6 +1129,113 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 	return nil
 }
 
+// adoptSend records a first send the machine already had (novox/hq ADR 0353 §6, §7): nothing is sent; the gate counts
+// from the machine's last send, and takes over the readings of an open gate of another walk judging the same build
+// there — whose carried move also says what the machine ran before, for a rollback.
+func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string,
+	before map[string]string, runningOf map[string][]string) {
+	inv := open.inventory
+	since := time.Now().UTC()
+	last, found, err := inv.SentTo(ctx, node)
+	if err == nil && found && !last.At.IsZero() {
+		since = last.At
+	}
+	lead := modules[0]
+	other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build)
+	for _, m := range modules {
+		s := p.Modules[m]
+		// What it ran before: kept by this walk before its send was made (§6); else what the other walk's send
+		// moved it from; else the build itself, so a failure leaves it as it was (issue 280).
+		switch {
+		case s.Previous != "":
+		case m == lead && from != "":
+			s.Previous = from
+		default:
+			s.Previous = before[m]
+		}
+		at := since
+		s.First, s.FirstAt = []string{node}, &at
+		s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning([]string{node}, runningOf[m]),
+			From: s.Previous, To: s.Commit, Since: &at}
+		if found {
+			s.Gate.Sent = map[string]inventory.SentDeclaration{node: last}
+		}
+		s.GatedBy = ""
+		if m == lead {
+			s.Gate.TakeReadings(other, modules, id)
+		} else {
+			s.GatedBy = lead
+		}
+	}
+	p.State = inventory.PlanRolling
+	p.Note = fmt.Sprintf("tier %d built; %s already runs %s's build: nothing sent again, judged from that send",
+		p.Tier, node, strings.Join(modules, ", "))
+	if id != "" {
+		p.Note += ", going on from the readings of " + id
+	}
+	fmt.Printf("%s: %s\n", p.ID, p.Note)
+}
+
+// appliedLastSend says a machine reported it applied the declaration it was sent last.
+func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool {
+	reports, err := inv.LastReports(ctx)
+	if err != nil {
+		return false
+	}
+	for _, r := range reports {
+		if r.Node == node {
+			return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied
+		}
+	}
+	return false
+}
+
+// judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the
+// walk's id, and what the machine ran before; nil when none does.
+func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) {
+	plans, err := inv.OpenPlans(ctx)
+	if err != nil {
+		return nil, "", ""
+	}
+	// The same build, by its id where both name one, else by its commit.
+	same := func(b, c string) bool {
+		if b != "" && build != "" {
+			return b == build
+		}
+		return sameCommit(c, commit)
+	}
+	usable := func(g *inventory.PlanGate) bool {
+		return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) &&
+			slices.Contains(g.Machines, node)
+	}
+	for _, q := range plans {
+		if q.ID == self {
+			continue
+		}
+		if r := q.Release; r != nil {
+			gates := []*inventory.PlanGate{r.Gate}
+			for _, g := range r.Gates {
+				gates = append(gates, g)
+			}
+			for _, g := range gates {
+				if !usable(g) {
+					continue
+				}
+				for _, c := range g.Carried {
+					if c.Module == module && c.Node == node && same(c.Build, c.To) {
+						return g, q.ID, c.From
+					}
+				}
+			}
+			continue
+		}
+		if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) {
+			return s.Gate, q.ID, s.Previous
+		}
+	}
+	return nil, "", ""
+}
+
 // passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
 // on the first of them passed, and its pass was kept for every build it carried (passCarried).
 func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
@@ -1594,9 +1724,19 @@ func plansCommand(ctx context.Context, args []string) error {
 			// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
 			fmt.Printf("  machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
 				orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", ")))
-			if r.Gate != nil {
-				fmt.Printf("  %s\n", gateLine(r.Gate))
-				for _, c := range r.Gate.Carried {
+			if len(r.Failed)+len(r.Silent) > 0 {
+				fmt.Printf("  failed: %s; silent: %s\n", orNone(strings.Join(r.Failed, ", ")), orNone(strings.Join(r.Silent, ", ")))
+			}
+			gates := []*inventory.PlanGate{r.Gate}
+			for _, n := range r.Order {
+				gates = append(gates, r.Gates[n])
+			}
+			for _, g := range gates {
+				if g == nil {
+					continue
+				}
+				fmt.Printf("  %s\n", gateLine(g))
+				for _, c := range g.Carried {
 					fmt.Printf("    %-22s %s → %s\n", c.Module, short(c.From), short(c.To))
 					if c.Recreates != "" {
 						fmt.Printf("    %-22s %s\n", "", c.Recreates)

cmd/mesh-controller/rest_at_once_test.go

@@ -0,0 +1,372 @@
+package main
+
+import (
+	"context"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// novox/hq ADR 0353: after its first machine passes, a build goes to every other machine at once, each judged by
+// its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them —
+// a machine already sent what a send is for is not sent again, and its judging goes on from what was read.
+//
+// Seen 2026-10-14: release-1791930194184991192 sent ace, g14, shanks and the control node one after another in
+// eleven minutes; the new controller its last send started sent the control node again the controller build it
+// had just applied, and judged it from its first reading.
+
+// aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others.
+func aBacklogOnThree(t *testing.T) *backlogMesh {
+	t.Helper()
+	b := aBacklog(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	if _, err := inv.AddNode(ctx, "desk"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.Assign(ctx, "desk", "app"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil {
+		t.Fatal(err)
+	}
+	wasHeard := releaseHeard
+	releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
+		return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil
+	}
+	t.Cleanup(func() { releaseHeard = wasHeard })
+	return b
+}
+
+// events records every send and every judging, in order.
+func events(t *testing.T) *[]string {
+	t.Helper()
+	var said []string
+	wasSend, wasGather := sendRollout, gatherGateFacts
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		said = append(said, "send "+strings.Join(names, ","))
+		return wasSend(ctx, open, names)
+	}
+	gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
+		said = append(said, "judge")
+		return wasGather(ctx, open, component)
+	}
+	t.Cleanup(func() { sendRollout, gatherGateFacts = wasSend, wasGather })
+	return &said
+}
+
+// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second
+// was judged to its verdict before the third was sent.
+func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklogOnThree(t)
+	ctx := t.Context()
+	said := events(t)
+	gateEvery = 0
+	for i := 0; i < 12; i++ {
+		advancePlans(ctx, b.open)
+	}
+	p := b.release(t)
+	if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
+		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
+	}
+	desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop")
+	if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) {
+		t.Fatalf("sends and judgings: %v", *said)
+	}
+	if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") {
+		t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said)
+	}
+	if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
+		t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
+	}
+}
+
+// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own
+// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent.
+func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklogOnThree(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	gateEvery = 0
+	b.broken["desk"] = true // desk fails what it is sent in the rest's step
+	advancePlans(ctx, b.open)
+	for i := 0; i < 12 && b.release(t).Open(); i++ {
+		advancePlans(ctx, b.open)
+	}
+	p := b.release(t)
+	if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
+		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
+	}
+	if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) {
+		t.Fatalf("laptop was never sent: %v", b.sent)
+	}
+	if !slices.Contains(p.Release.Done, "laptop") {
+		t.Fatalf("laptop has no verdict of its own: %+v", p.Release)
+	}
+	// late waited on laptop alone and was healthy there: its pass is kept, whatever desk did.
+	if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
+		t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
+	}
+	// app failed on desk: marked, and put back wherever it was sent (ADR 0236).
+	if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
+		t.Fatal("app's build is not marked failed")
+	}
+	if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" {
+		t.Fatalf("desk was not put back: %v", sent)
+	}
+}
+
+// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that
+// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading.
+func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklog(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once
+	release := b.release(t)
+	judging := release.Release.Gate
+	if judging == nil || judging.Pending || judging.Passes == 0 {
+		t.Fatalf("the release is not judging anchor: %+v", judging)
+	}
+	built := time.Now().UTC()
+	plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
+		Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
+		Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
+			Build: "build-app-c2"}}}
+	if err := inv.SavePlan(ctx, &plan); err != nil {
+		t.Fatal(err)
+	}
+	advancePlans(ctx, b.open)
+	if len(b.sent) != 1 {
+		t.Fatalf("anchor was sent again what it runs: %v", b.sent)
+	}
+	p, err := inv.PlanByID(ctx, "plan-app")
+	if err != nil {
+		t.Fatal(err)
+	}
+	g := p.Modules["app"].Gate
+	if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID {
+		t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging)
+	}
+	if p.Modules["app"].Previous != "c1" {
+		t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous)
+	}
+}
+
+// A walk's first send that was made and not kept — the controller replaced between the two — is not made again
+// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6).
+func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	g := aGateMesh(t)
+	ctx := t.Context()
+	// The old controller sends anchor app c2 and is replaced before it keeps the send.
+	made := sendRollout
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		if _, err := made(ctx, open, names); err != nil {
+			return nil, err
+		}
+		return nil, context.Canceled
+	}
+	advancePlans(ctx, g.open)
+	sendRollout = made
+	if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil {
+		t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent)
+	}
+	last, _, err := g.open.inventory.SentTo(ctx, "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	advancePlans(ctx, g.open)
+	if len(g.sent) != 1 {
+		t.Fatalf("the send was made again: %v", g.sent)
+	}
+	s := g.plan(t).Modules["app"]
+	if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) {
+		t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate)
+	}
+	// What anchor ran before was kept before the send, so a failure still puts it back (§5).
+	if s.Previous != "c1" || s.Gate.From != "c1" {
+		t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From)
+	}
+}
+
+// A release's send kept before it was made, and made, is judged by the controller that takes over without being
+// made again (ADR 0353 §8).
+func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklog(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	p, err := releaseBacklog(ctx, b.open, "")
+	if err != nil || p == nil {
+		t.Fatalf("no release opened: %v", err)
+	}
+	if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending {
+		t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
+	}
+	if err := inv.SavePlan(ctx, p); err != nil {
+		t.Fatal(err)
+	}
+	// The send is made, and the controller is replaced before it keeps it.
+	if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
+		t.Fatal(err)
+	}
+	advancePlans(ctx, b.open)
+	if len(b.sent) != 1 {
+		t.Fatalf("anchor was sent again: %v", b.sent)
+	}
+	g := b.release(t).Release.Gate
+	if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
+		t.Fatalf("anchor is not judged on the send made: %+v", g)
+	}
+}
+
+// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits
+// for the next step, and one whose every build passed goes now (ADR 0353 §2).
+func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) {
+	waiting := map[string][]inventory.CarriedMove{
+		"desk":   {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
+		"laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
+		"shanks": {{Module: "app", To: "c2"}},
+		"novox":  {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}},
+	}
+	passed := func(module, commit string) bool { return module == "app" && commit == "c2" }
+	never := func(string) bool { return false }
+	step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never)
+	if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
+		t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step)
+	}
+	if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) {
+		t.Fatalf("a machine whose every build passed waits: %v", step)
+	}
+}
+
+// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step
+// carries either to one machine only, while what else passed goes everywhere at once.
+func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) {
+	waiting := map[string][]inventory.CarriedMove{
+		"desk":   {{Module: hostModule, To: "h2"}},
+		"laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}},
+		"shanks": {{Module: "app", To: "c2"}},
+		"novox":  {{Module: "dns", To: "d2"}},
+		"ace":    {{Module: "dns", To: "d2"}},
+	}
+	resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}}
+	alone := func(m string) bool {
+		if m == "dns" {
+			return oneAtATime(m, resolver)
+		}
+		return oneAtATime(m, catalogue.Manifest{Module: m})
+	}
+	step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting,
+		func(string, string) bool { return true }, alone)
+	if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
+		t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step)
+	}
+	if oneAtATime("app", catalogue.Manifest{Module: "app"}) {
+		t.Fatal("a leaf module is held to one machine at a time")
+	}
+}
+
+// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent.
+func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) {
+	b := aBacklogOnThree(t)
+	ctx := t.Context()
+	gateEvery = 0
+	b.broken["anchor"] = true
+	advancePlans(ctx, b.open)
+	for i := 0; i < 6 && b.release(t).Open(); i++ {
+		advancePlans(ctx, b.open)
+	}
+	if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest {
+		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
+	}
+	for _, s := range b.sent {
+		if !slices.Equal(s, []string{"anchor"}) {
+			t.Fatalf("sent %v: the rest after a failed first machine", b.sent)
+		}
+	}
+}
+
+// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is
+// ordered after every other.
+func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklogOnThree(t)
+	ctx := t.Context()
+	register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"})
+	if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil {
+		t.Fatal(err)
+	}
+	said := events(t)
+	gateEvery = 0
+	for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ {
+		advancePlans(ctx, b.open)
+	}
+	if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) {
+		t.Fatalf("the order is %v", p.Release.Order)
+	}
+	laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor")
+	if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") {
+		t.Fatalf("the control node is not sent last in the rest's step: %v", *said)
+	}
+}
+
+// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
+// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4).
+func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklog(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	reporting := sendRollout
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
+			b.sent = append(b.sent, names)
+			current, _ := open.inventory.CurrentBuilds(ctx)
+			return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
+				map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
+		}
+		return reporting(ctx, open, names)
+	}
+	t.Cleanup(func() { sendRollout = reporting })
+	gateEvery = 0
+	for i := 0; i < 8; i++ {
+		advancePlans(ctx, b.open)
+	}
+	p := b.release(t)
+	g := p.Release.Gates["laptop"]
+	if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
+		t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
+	}
+	since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
+	g.Since = &since
+	if err := inv.SavePlan(ctx, &p); err != nil {
+		t.Fatal(err)
+	}
+	advancePlans(ctx, b.open)
+	p = b.release(t)
+	if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
+		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
+	}
+	if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
+		t.Fatal("a silent machine's build was marked failed")
+	}
+	if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
+		t.Fatalf("laptop was put back: %v", sent)
+	}
+	obs, err := silentObservation(ctx, inv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
+		t.Fatalf("the silence is not said: %+v", obs)
+	}
+}

cmd/mesh-controller/upgrades.go

@@ -79,7 +79,7 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
 	// `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it
 	// now waits for a gate like any other build, and the release plan walks it through the machines, one
 	// at a time, each judged.
-	fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine at a time, each judged "+
+	fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine first and then the rest at once, each judged "+
 		"(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on))
 	return nil
 }

cmd/mesh-controller/walk_said.go

@@ -177,6 +177,12 @@ func walkReached(p inventory.Plan, running func(string) []string) map[string][]s
 			reached = appendNew(reached, r.Gate.Machines...)
 			carried(r.Gate)
 		}
+		for _, g := range r.Gates {
+			if g != nil && !g.Pending {
+				reached = appendNew(reached, g.Machines...)
+				carried(g)
+			}
+		}
 		for _, tier := range p.Tiers {
 			for _, m := range tier {
 				for _, n := range running(m) {

internal/inventory/plans.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"slices"
 	"strings"
 	"time"
 
@@ -391,6 +392,65 @@ type PlanGate struct {
 	// reports its module's container left running because it said busy, the judging waits and has not
 	// started; once it reports the container replaced, Since is moved to then. Nil when nothing waited.
 	Busy *GateBusy `json:"busy,omitempty"`
+	// Pending says the send this gate judges is kept and not yet made (novox/hq ADR 0353 §8): a controller that
+	// takes over reads the machine's last send before making it, and never sends it twice.
+	Pending bool `json:"pending,omitempty"`
+	// Silent says the machine reported nothing on its send within ApplySilentAfter (novox/hq ADR 0353 §4): left,
+	// neither failed nor put back.
+	Silent bool `json:"silent,omitempty"`
+	// TookOver names the gate whose readings this one took over (novox/hq ADR 0353 §7): another walk's judging of
+	// the same build on the same machine, which this gate goes on from instead of starting from zero.
+	TookOver string `json:"took_over,omitempty"`
+}
+
+// TakeReadings makes this gate go on from another's readings of the same build on the same machine (novox/hq ADR
+// 0353 §7): when the judging began, when each machine applied, the passes counted and when, and what each found.
+// A module's own count is taken only for the modules this gate judges.
+func (g *PlanGate) TakeReadings(from *PlanGate, modules []string, id string) {
+	if from == nil {
+		return
+	}
+	if from.Since != nil {
+		since := *from.Since
+		g.Since = &since
+	}
+	for n, s := range from.Sent {
+		if slices.Contains(g.Machines, n) {
+			if g.Sent == nil {
+				g.Sent = map[string]SentDeclaration{}
+			}
+			g.Sent[n] = s
+		}
+	}
+	for n, at := range from.Applied {
+		if slices.Contains(g.Machines, n) {
+			if g.Applied == nil {
+				g.Applied = map[string]time.Time{}
+			}
+			g.Applied[n] = at
+		}
+	}
+	g.Passes, g.Faulted, g.HealthySeen, g.Last = from.Passes, from.Faulted, from.HealthySeen, from.Last
+	if from.LastPass != nil {
+		at := *from.LastPass
+		g.LastPass = &at
+	}
+	for _, m := range modules {
+		if n, counted := from.Healthy[m]; counted {
+			if g.Healthy == nil {
+				g.Healthy = map[string]int{}
+			}
+			g.Healthy[m] = n
+		}
+		if at, counted := from.HealthyAt[m]; counted {
+			if g.HealthyAt == nil {
+				g.HealthyAt = map[string]time.Time{}
+			}
+			g.HealthyAt[m] = at
+		}
+	}
+	g.Readings = append([]GateReading(nil), from.Readings...)
+	g.TookOver = id
 }
 
 // GateBusy is a first-node gate's wait for a busy container (novox/hq ADR 0335 §4).
@@ -477,9 +537,17 @@ type CarriedMove struct {
 type PlanRelease struct {
 	Order []string `json:"order"`
 	Next  int      `json:"next"`
-	// Gate is the machine being judged; nil between machines.
+	// Gate is the first machine being judged; nil between machines.
 	Gate *PlanGate `json:"gate,omitempty"`
-	Done []string  `json:"done,omitempty"`
+	// Rest says the first machine passed (novox/hq ADR 0353 §1): every other machine is sent at once, each judged by
+	// its own gate in Gates, by machine — a build no gate passed going to one machine of a step only (§2).
+	Rest  bool                 `json:"rest,omitempty"`
+	Gates map[string]*PlanGate `json:"gates,omitempty"`
+	// Failed are the machines of the rest whose gate failed, and Silent those that said nothing on their send
+	// within ApplySilentAfter (ADR 0353 §3 and §4).
+	Failed []string `json:"failed,omitempty"`
+	Silent []string `json:"silent,omitempty"`
+	Done   []string `json:"done,omitempty"`
 	// Skipped are the machines not heard from when their turn came, left as they were.
 	Skipped []string `json:"skipped,omitempty"`
 	// By is the person who released it, empty when the mesh did.

testdata/store-tests

@@ -61,3 +61,11 @@
 ./cmd/mesh-controller TestAProviderElsewhereServesItsOwnComposedName
 ./cmd/mesh-controller TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne
 ./internal/inventory TestAReportOfABusyContainerIsKeptAndReadBack
+./cmd/mesh-controller TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses
+./cmd/mesh-controller TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn
+./cmd/mesh-controller TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere
+./cmd/mesh-controller TestAFirstSendMadeAndNotKeptIsNotSentAgain
+./cmd/mesh-controller TestAReleaseSendKeptAndMadeIsNotMadeAgain
+./cmd/mesh-controller TestAFirstMachineThatFailsStillStopsTheWalk
+./cmd/mesh-controller TestTheControlNodeIsSentLastInAStep
+./cmd/mesh-controller TestARestMachineSilentPastItsBoundIsLeftAndSaid