Bring F3 up to its release branch, with both sides' store tests (part 3 of 14)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

cmd/mesh-controller/release_plan.go

@@ -251,6 +251,20 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool)
 	return out
 }
 
+// noMoveWhy is what a plan says of a module of its tier whose build was made from the source every machine
+// running it runs (novox/hq issue 280): nothing is sent for it.
+// noMovePrefix opens it, and is what noMove reads: one prefix for the writer and the reader.
+const (
+	noMovePrefix = "no move"
+	noMoveWhy    = noMovePrefix + ": made from the source every machine running it runs"
+)
+
+// noMove is whether the plan marked the module no move and sent it nothing. Read from what the plan says,
+// so a plan kept before the mark had a name of its own is read the same (novox/hq issue 648).
+func noMove(s *inventory.PlanModule) bool {
+	return s != nil && s.FirstAt == nil && s.Gate == nil && strings.HasPrefix(s.Why, noMovePrefix)
+}
+
 // applied says whether every machine running the module has reported since the module was built.
 // appliedEach is applied with a moment of its own for each machine: the reports that count are the ones
 // after that machine was sent the build (novox/hq issue 256).
@@ -595,7 +609,7 @@ func advancePlans(ctx context.Context, open *stores) {
 		return
 	}
 	defer release()
-	// What waits for a gate, released one machine at a time (ADR 0236).
+	// What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353).
 	if _, err := releaseBacklog(ctx, open, ""); err != nil {
 		fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err)
 	}
@@ -805,7 +819,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 			} else if same {
 				now := time.Now().UTC()
 				state.SentAt = &now
-				state.Why = "no move: made from the source every machine running it runs"
+				state.Why = noMoveWhy
 				fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m)
 				continue
 			}
@@ -995,6 +1009,25 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 			if state.BuiltAt != nil {
 				since = *state.BuiltAt
 			}
+			// **A build that is no move is waited on by the send that last carried it** (novox/hq issue 648).
+			// Nothing was sent for it, so no report follows its marking or its build: a machine that does not
+			// happen to report again held the walk for ever. Each machine runs it once it answered its last
+			// send — the one whose builds made it no move — whenever that was.
+			if noMove(state) {
+				byNode := map[string]inventory.Reported{}
+				for _, r := range reports {
+					byNode[r.Node] = r
+				}
+				if ok, on := appliedEach(m, func(node string) time.Time {
+					if s := byNode[node].Sent; s != nil {
+						return *s
+					}
+					return time.Time{}
+				}, running, reports); !ok {
+					waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
+				}
+				continue
+			}
 			// **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218)
 			// a module is sent twice — the first machine, then the rest — and SentAt is the second.
 			// Asked of every machine, the first machine's report, made between the two sends, read as
@@ -1056,10 +1089,36 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 	runningOf map[string][]string) error {
 	inv := open.inventory
 	before, beforeKnown, _ := inv.SentBuilds(ctx, node)
+	// **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan
+	// before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from
+	// the readings of a gate already judging it there.
+	// Only a send the machine reported it applied is one it has; one it never reported on is sent, as before
+	// (issue 280: what it ran before is then that build, and a failure leaves it as it was).
+	if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 &&
+		appliedLastSend(ctx, inv, node) {
+		if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 {
+			adoptSend(ctx, open, p, node, modules, before, runningOf)
+			return nil
+		}
+	}
+	// **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced
+	// between the send and keeping it reads it here, and a failure still puts the machine back.
+	kept := false
+	for _, m := range modules {
+		s := p.Modules[m]
+		if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] {
+			s.Previous, kept = before[m], true
+		}
+	}
+	if kept {
+		if err := inv.SavePlan(ctx, p); err != nil {
+			return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err)
+		}
+	}
 	owns := make([]inventory.CarriedMove, 0, len(modules))
 	for _, m := range modules {
 		s := p.Modules[m]
-		owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: before[m], To: s.Commit, Build: s.Build})
+		owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build})
 	}
 	// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
 	// all of it there.
@@ -1074,10 +1133,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 	sentWhat := sentNow(ctx, inv, sent)
 	for _, m := range modules {
 		s := p.Modules[m]
-		// What the first machine ran before: what a failed gate puts back (ADR 0236).
-		if beforeKnown {
-			s.Previous = before[m]
-		}
+		// What the first machine ran before (ADR 0236), kept above before the send.
 		s.First, s.FirstAt = sent, &now
 		s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]),
 			From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat}
@@ -1106,6 +1162,113 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 	return nil
 }
 
+// adoptSend records a first send the machine already had (novox/hq ADR 0353 §6, §7): nothing is sent; the gate counts
+// from the machine's last send, and takes over the readings of an open gate of another walk judging the same build
+// there — whose carried move also says what the machine ran before, for a rollback.
+func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string,
+	before map[string]string, runningOf map[string][]string) {
+	inv := open.inventory
+	since := time.Now().UTC()
+	last, found, err := inv.SentTo(ctx, node)
+	if err == nil && found && !last.At.IsZero() {
+		since = last.At
+	}
+	lead := modules[0]
+	other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build)
+	for _, m := range modules {
+		s := p.Modules[m]
+		// What it ran before: kept by this walk before its send was made (§6); else what the other walk's send
+		// moved it from; else the build itself, so a failure leaves it as it was (issue 280).
+		switch {
+		case s.Previous != "":
+		case m == lead && from != "":
+			s.Previous = from
+		default:
+			s.Previous = before[m]
+		}
+		at := since
+		s.First, s.FirstAt = []string{node}, &at
+		s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning([]string{node}, runningOf[m]),
+			From: s.Previous, To: s.Commit, Since: &at}
+		if found {
+			s.Gate.Sent = map[string]inventory.SentDeclaration{node: last}
+		}
+		s.GatedBy = ""
+		if m == lead {
+			s.Gate.TakeReadings(other, modules, id)
+		} else {
+			s.GatedBy = lead
+		}
+	}
+	p.State = inventory.PlanRolling
+	p.Note = fmt.Sprintf("tier %d built; %s already runs %s's build: nothing sent again, judged from that send",
+		p.Tier, node, strings.Join(modules, ", "))
+	if id != "" {
+		p.Note += ", going on from the readings of " + id
+	}
+	fmt.Printf("%s: %s\n", p.ID, p.Note)
+}
+
+// appliedLastSend says a machine reported it applied the declaration it was sent last.
+func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool {
+	reports, err := inv.LastReports(ctx)
+	if err != nil {
+		return false
+	}
+	for _, r := range reports {
+		if r.Node == node {
+			return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied
+		}
+	}
+	return false
+}
+
+// judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the
+// walk's id, and what the machine ran before; nil when none does.
+func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) {
+	plans, err := inv.OpenPlans(ctx)
+	if err != nil {
+		return nil, "", ""
+	}
+	// The same build, by its id where both name one, else by its commit.
+	same := func(b, c string) bool {
+		if b != "" && build != "" {
+			return b == build
+		}
+		return sameCommit(c, commit)
+	}
+	usable := func(g *inventory.PlanGate) bool {
+		return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) &&
+			slices.Contains(g.Machines, node)
+	}
+	for _, q := range plans {
+		if q.ID == self {
+			continue
+		}
+		if r := q.Release; r != nil {
+			gates := []*inventory.PlanGate{r.Gate}
+			for _, g := range r.Gates {
+				gates = append(gates, g)
+			}
+			for _, g := range gates {
+				if !usable(g) {
+					continue
+				}
+				for _, c := range g.Carried {
+					if c.Module == module && c.Node == node && same(c.Build, c.To) {
+						return g, q.ID, c.From
+					}
+				}
+			}
+			continue
+		}
+		if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) {
+			return s.Gate, q.ID, s.Previous
+		}
+	}
+	return nil, "", ""
+}
+
 // passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate
 // on the first of them passed, and its pass was kept for every build it carried (passCarried).
 func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) {
@@ -1594,9 +1757,19 @@ func plansCommand(ctx context.Context, args []string) error {
 			// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
 			fmt.Printf("  machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
 				orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", ")))
-			if r.Gate != nil {
-				fmt.Printf("  %s\n", gateLine(r.Gate))
-				for _, c := range r.Gate.Carried {
+			if len(r.Failed)+len(r.Silent) > 0 {
+				fmt.Printf("  failed: %s; silent: %s\n", orNone(strings.Join(r.Failed, ", ")), orNone(strings.Join(r.Silent, ", ")))
+			}
+			gates := []*inventory.PlanGate{r.Gate}
+			for _, n := range r.Order {
+				gates = append(gates, r.Gates[n])
+			}
+			for _, g := range gates {
+				if g == nil {
+					continue
+				}
+				fmt.Printf("  %s\n", gateLine(g))
+				for _, c := range g.Carried {
 					fmt.Printf("    %-22s %s → %s\n", c.Module, short(c.From), short(c.To))
 					if c.Recreates != "" {
 						fmt.Printf("    %-22s %s\n", "", c.Recreates)

cmd/mesh-controller/rest_at_once_test.go

@@ -0,0 +1,372 @@
+package main
+
+import (
+	"context"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// novox/hq ADR 0353: after its first machine passes, a build goes to every other machine at once, each judged by
+// its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them —
+// a machine already sent what a send is for is not sent again, and its judging goes on from what was read.
+//
+// Seen 2026-10-14: release-1791930194184991192 sent ace, g14, shanks and the control node one after another in
+// eleven minutes; the new controller its last send started sent the control node again the controller build it
+// had just applied, and judged it from its first reading.
+
+// aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others.
+func aBacklogOnThree(t *testing.T) *backlogMesh {
+	t.Helper()
+	b := aBacklog(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	if _, err := inv.AddNode(ctx, "desk"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.Assign(ctx, "desk", "app"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil {
+		t.Fatal(err)
+	}
+	wasHeard := releaseHeard
+	releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
+		return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil
+	}
+	t.Cleanup(func() { releaseHeard = wasHeard })
+	return b
+}
+
+// events records every send and every judging, in order.
+func events(t *testing.T) *[]string {
+	t.Helper()
+	var said []string
+	wasSend, wasGather := sendRollout, gatherGateFacts
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		said = append(said, "send "+strings.Join(names, ","))
+		return wasSend(ctx, open, names)
+	}
+	gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
+		said = append(said, "judge")
+		return wasGather(ctx, open, component)
+	}
+	t.Cleanup(func() { sendRollout, gatherGateFacts = wasSend, wasGather })
+	return &said
+}
+
+// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second
+// was judged to its verdict before the third was sent.
+func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklogOnThree(t)
+	ctx := t.Context()
+	said := events(t)
+	gateEvery = 0
+	for i := 0; i < 12; i++ {
+		advancePlans(ctx, b.open)
+	}
+	p := b.release(t)
+	if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
+		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
+	}
+	desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop")
+	if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) {
+		t.Fatalf("sends and judgings: %v", *said)
+	}
+	if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") {
+		t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said)
+	}
+	if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
+		t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
+	}
+}
+
+// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own
+// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent.
+func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklogOnThree(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	gateEvery = 0
+	b.broken["desk"] = true // desk fails what it is sent in the rest's step
+	advancePlans(ctx, b.open)
+	for i := 0; i < 12 && b.release(t).Open(); i++ {
+		advancePlans(ctx, b.open)
+	}
+	p := b.release(t)
+	if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
+		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
+	}
+	if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) {
+		t.Fatalf("laptop was never sent: %v", b.sent)
+	}
+	if !slices.Contains(p.Release.Done, "laptop") {
+		t.Fatalf("laptop has no verdict of its own: %+v", p.Release)
+	}
+	// late waited on laptop alone and was healthy there: its pass is kept, whatever desk did.
+	if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
+		t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
+	}
+	// app failed on desk: marked, and put back wherever it was sent (ADR 0236).
+	if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
+		t.Fatal("app's build is not marked failed")
+	}
+	if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" {
+		t.Fatalf("desk was not put back: %v", sent)
+	}
+}
+
+// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that
+// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading.
+func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklog(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once
+	release := b.release(t)
+	judging := release.Release.Gate
+	if judging == nil || judging.Pending || judging.Passes == 0 {
+		t.Fatalf("the release is not judging anchor: %+v", judging)
+	}
+	built := time.Now().UTC()
+	plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
+		Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
+		Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
+			Build: "build-app-c2"}}}
+	if err := inv.SavePlan(ctx, &plan); err != nil {
+		t.Fatal(err)
+	}
+	advancePlans(ctx, b.open)
+	if len(b.sent) != 1 {
+		t.Fatalf("anchor was sent again what it runs: %v", b.sent)
+	}
+	p, err := inv.PlanByID(ctx, "plan-app")
+	if err != nil {
+		t.Fatal(err)
+	}
+	g := p.Modules["app"].Gate
+	if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID {
+		t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging)
+	}
+	if p.Modules["app"].Previous != "c1" {
+		t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous)
+	}
+}
+
+// A walk's first send that was made and not kept — the controller replaced between the two — is not made again
+// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6).
+func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	g := aGateMesh(t)
+	ctx := t.Context()
+	// The old controller sends anchor app c2 and is replaced before it keeps the send.
+	made := sendRollout
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		if _, err := made(ctx, open, names); err != nil {
+			return nil, err
+		}
+		return nil, context.Canceled
+	}
+	advancePlans(ctx, g.open)
+	sendRollout = made
+	if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil {
+		t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent)
+	}
+	last, _, err := g.open.inventory.SentTo(ctx, "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	advancePlans(ctx, g.open)
+	if len(g.sent) != 1 {
+		t.Fatalf("the send was made again: %v", g.sent)
+	}
+	s := g.plan(t).Modules["app"]
+	if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) {
+		t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate)
+	}
+	// What anchor ran before was kept before the send, so a failure still puts it back (§5).
+	if s.Previous != "c1" || s.Gate.From != "c1" {
+		t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From)
+	}
+}
+
+// A release's send kept before it was made, and made, is judged by the controller that takes over without being
+// made again (ADR 0353 §8).
+func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklog(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	p, err := releaseBacklog(ctx, b.open, "")
+	if err != nil || p == nil {
+		t.Fatalf("no release opened: %v", err)
+	}
+	if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending {
+		t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
+	}
+	if err := inv.SavePlan(ctx, p); err != nil {
+		t.Fatal(err)
+	}
+	// The send is made, and the controller is replaced before it keeps it.
+	if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
+		t.Fatal(err)
+	}
+	advancePlans(ctx, b.open)
+	if len(b.sent) != 1 {
+		t.Fatalf("anchor was sent again: %v", b.sent)
+	}
+	g := b.release(t).Release.Gate
+	if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
+		t.Fatalf("anchor is not judged on the send made: %+v", g)
+	}
+}
+
+// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits
+// for the next step, and one whose every build passed goes now (ADR 0353 §2).
+func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) {
+	waiting := map[string][]inventory.CarriedMove{
+		"desk":   {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
+		"laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
+		"shanks": {{Module: "app", To: "c2"}},
+		"novox":  {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}},
+	}
+	passed := func(module, commit string) bool { return module == "app" && commit == "c2" }
+	never := func(string) bool { return false }
+	step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never)
+	if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
+		t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step)
+	}
+	if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) {
+		t.Fatalf("a machine whose every build passed waits: %v", step)
+	}
+}
+
+// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step
+// carries either to one machine only, while what else passed goes everywhere at once.
+func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) {
+	waiting := map[string][]inventory.CarriedMove{
+		"desk":   {{Module: hostModule, To: "h2"}},
+		"laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}},
+		"shanks": {{Module: "app", To: "c2"}},
+		"novox":  {{Module: "dns", To: "d2"}},
+		"ace":    {{Module: "dns", To: "d2"}},
+	}
+	resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}}
+	alone := func(m string) bool {
+		if m == "dns" {
+			return oneAtATime(m, resolver)
+		}
+		return oneAtATime(m, catalogue.Manifest{Module: m})
+	}
+	step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting,
+		func(string, string) bool { return true }, alone)
+	if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
+		t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step)
+	}
+	if oneAtATime("app", catalogue.Manifest{Module: "app"}) {
+		t.Fatal("a leaf module is held to one machine at a time")
+	}
+}
+
+// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent.
+func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) {
+	b := aBacklogOnThree(t)
+	ctx := t.Context()
+	gateEvery = 0
+	b.broken["anchor"] = true
+	advancePlans(ctx, b.open)
+	for i := 0; i < 6 && b.release(t).Open(); i++ {
+		advancePlans(ctx, b.open)
+	}
+	if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest {
+		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
+	}
+	for _, s := range b.sent {
+		if !slices.Equal(s, []string{"anchor"}) {
+			t.Fatalf("sent %v: the rest after a failed first machine", b.sent)
+		}
+	}
+}
+
+// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is
+// ordered after every other.
+func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklogOnThree(t)
+	ctx := t.Context()
+	register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"})
+	if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil {
+		t.Fatal(err)
+	}
+	said := events(t)
+	gateEvery = 0
+	for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ {
+		advancePlans(ctx, b.open)
+	}
+	if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) {
+		t.Fatalf("the order is %v", p.Release.Order)
+	}
+	laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor")
+	if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") {
+		t.Fatalf("the control node is not sent last in the rest's step: %v", *said)
+	}
+}
+
+// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
+// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4).
+func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklog(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	reporting := sendRollout
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
+			b.sent = append(b.sent, names)
+			current, _ := open.inventory.CurrentBuilds(ctx)
+			return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
+				map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
+		}
+		return reporting(ctx, open, names)
+	}
+	t.Cleanup(func() { sendRollout = reporting })
+	gateEvery = 0
+	for i := 0; i < 8; i++ {
+		advancePlans(ctx, b.open)
+	}
+	p := b.release(t)
+	g := p.Release.Gates["laptop"]
+	if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
+		t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
+	}
+	since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
+	g.Since = &since
+	if err := inv.SavePlan(ctx, &p); err != nil {
+		t.Fatal(err)
+	}
+	advancePlans(ctx, b.open)
+	p = b.release(t)
+	if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
+		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
+	}
+	if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
+		t.Fatal("a silent machine's build was marked failed")
+	}
+	if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
+		t.Fatalf("laptop was put back: %v", sent)
+	}
+	obs, err := silentObservation(ctx, inv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
+		t.Fatalf("the silence is not said: %+v", obs)
+	}
+}

cmd/mesh-controller/seatverbs.go

@@ -457,6 +457,28 @@ func (a *verbArguments) commandLine() ([]string, error) {
 			return nil, err
 		}
 		return []string{"forget-checks", str("owner"), str("repo"), str("number")}, nil
+	case "try":
+		// A branch tried before its pull request (novox/hq issue 686): asked, or read by the id it answered.
+		if id := str("id"); id != "" {
+			return []string{"try", "--id", id}, nil
+		}
+		if a.given["repository"] == "" && a.given["ref"] == "" {
+			// Neither shape named: the command says its usage, which names both, as `build` does.
+			return []string{"try"}, nil
+		}
+		if err := need("repository", "ref"); err != nil {
+			return nil, err
+		}
+		argv := []string{"try", str("repository"), str("ref")}
+		for _, flag := range []string{"base", "tests"} {
+			if v := str(flag); v != "" {
+				argv = append(argv, "--"+flag, v)
+			}
+		}
+		if on("store-only") {
+			argv = append(argv, "--store-only")
+		}
+		return argv, nil
 	case "cancel", "kill":
 		if err := need("id"); err != nil {
 			return nil, err
@@ -1018,6 +1040,8 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co
 	"delivery": true,
 	// The build queue, each ask with its position among the waiting (novox/hq issue 455).
 	"queue": true,
+	// A branch tried before its pull request (novox/hq issue 686).
+	"try": true,
 	// A plan followed (novox/hq issue 229): `plans wait` answers JSON; its other forms say text, and so answer no data.
 	"plans": true}
 
@@ -1308,6 +1332,10 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
 					link.AnswersWithin(ctx, bound+planWaitStart)
 				}
 			}
+			if verb == "try" && len(argv) > 1 && argv[1] == "--id" {
+				// Reading a try holds its answer for the verdict, up to its bound (novox/hq issue 686, 229).
+				link.AnswersWithin(ctx, tryWait+planWaitStart)
+			}
 			if answer, read := readHere(ctx, argv); read {
 				return answer, nil
 			}