Keep a walk's sends to what was kept, and say a silent rest (hq ADR 0353 review)

A release's own gates held its other sends back for ever; an adopted send
lost what the machine ran before; a kept send could grow a build no gate
passed; a take-over matched by commit alone; a rest that went silent ended
done. The node-engine and the resolver still go one machine at a time, since
a fault that cuts a machine off shows only as silence.

cmd/mesh-controller/gate.go

@@ -1443,6 +1443,8 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error
 	}
 	// A release held after a failed one, while builds still wait for a gate (ADR 0236).
 	out = append(out, backlogObservation()...)
+	// And a rest that went silent (novox/hq ADR 0353 §4).
+	out = append(out, silentObservation(ctx, d.open.inventory)...)
 	return sortedFound(dedupeObservations(out)), nil
 }
 

cmd/mesh-controller/plain_words.go

@@ -456,6 +456,9 @@ var plainWordings = map[string]func(conditions.Observation) words{
 	"release-held": worded(func(o conditions.Observation) words {
 		return releaseHeldWords(nil, o.Also)
 	}),
+	"release-silent": worded(func(o conditions.Observation) words {
+		return releaseSilentWords(o.Also)
+	}),
 	"facts-stale": worded(func(o conditions.Observation) words {
 		return words{Headline: "Merge checks use outdated facts",
 			Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
@@ -841,6 +844,19 @@ func releaseHeldWords(modules, machines []string) words {
 		Resolved: "Resolved: the held updates are released"}
 }
 
+// releaseSilentWords are the plain words of machines that said nothing on an update they were sent (ADR 0353 §4).
+func releaseSilentWords(machines []string) words {
+	where := "Some machines"
+	if len(machines) > 0 {
+		where = conditions.Capital(namesWords(machines, 4))
+	}
+	return words{Headline: "Machines did not answer an update",
+		Needs: "check that they are awake and on the mesh; the next walk sends them what waits.",
+		Explanation: where + " were sent module updates and said nothing about them within fifteen minutes, so " +
+			"they were left as they were. What they run is not known to be the update.",
+		Resolved: "Resolved: the next walk reached them"}
+}
+
 // reloginNeeds is what an account waiting for its groups needs (ADR 0252).
 func reloginNeeds(node string) string {
 	return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)

cmd/mesh-controller/release.go

@@ -81,6 +81,15 @@ type moveFacts struct {
 	srcs   map[string]map[string]string
 	passed map[string]map[string]bool
 	plans  []inventory.Plan
+	// self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353).
+	self string
+}
+
+type walkingKey struct{}
+
+// withWalking says a release plan is the one sending: its own gates never hold its own sends back.
+func withWalking(ctx context.Context, id string) context.Context {
+	return context.WithValue(ctx, walkingKey{}, id)
 }
 
 func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
@@ -99,6 +108,7 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er
 		return f, err
 	}
 	f.plans, err = inv.OpenPlans(ctx)
+	f.self, _ = ctx.Value(walkingKey{}).(string)
 	return f, err
 }
 
@@ -193,7 +203,7 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string,
 // feet. A release keeps no module records: its open gate's carried moves are its walk.
 func (f moveFacts) walkedBy(module, node, to string) string {
 	for _, p := range f.plans {
-		if !p.Open() {
+		if !p.Open() || (p.Release != nil && p.ID == f.self) {
 			continue
 		}
 		if p.Release != nil {
@@ -203,7 +213,8 @@ func (f moveFacts) walkedBy(module, node, to string) string {
 				gates = append(gates, g)
 			}
 			for _, g := range gates {
-				if g == nil || g.Verdict != "" || g.Silent {
+				// A send kept and not yet made walks nothing yet (ADR 0353 §8).
+				if g == nil || g.Verdict != "" || g.Silent || g.Pending {
 					continue
 				}
 				for _, c := range g.Carried {
@@ -734,6 +745,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
 	if r.Settings != nil {
 		return advanceSettingsWalk(ctx, open, p)
 	}
+	ctx = withWalking(ctx, p.ID)
 	now := time.Now().UTC()
 	if r.Rest {
 		return advanceRest(ctx, open, p, now)
@@ -915,6 +927,24 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
 		}
 	}
 	if len(pending) > 0 {
+		// **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on
+		// a machine since its step was chosen could reach a second machine in it; the step is chosen again.
+		if grew, err := keptStepGrew(ctx, open, r, pending); err != nil {
+			return false, err
+		} else if grew != "" {
+			for _, n := range pending {
+				delete(r.Gates, n)
+			}
+			opened, waiting, err := openRestStep(ctx, open, p)
+			if err != nil {
+				return true, err
+			}
+			p.Note = "chosen again: " + grew
+			if !opened && waiting != "" {
+				p.Note += "; " + waiting
+			}
+			return true, nil
+		}
 		var sent, already, waiting []string
 		for _, node := range pending {
 			out, err := sendKept(ctx, open, p, node, r.Gates[node], now)
@@ -1056,12 +1086,44 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
 	if len(r.Failed) > 0 {
 		p.State = inventory.PlanFailed
 		end = "failed on " + strings.Join(r.Failed, ", ") + "; " + end
+	} else if len(r.Silent) > 0 && len(r.Gates) == len(r.Silent) {
+		// **A rest that said nothing is no success** (ADR 0353 §4): failure is loud; the condition says it.
+		p.State = inventory.PlanFailed
+		end = "no machine of the rest reported on its send; " + end
 	}
 	p.Note = strings.Join(append(notes, end), "; ")
 	fmt.Printf("%s: %s\n", p.ID, p.Note)
 	return true, nil
 }
 
+// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with;
+// empty when none would.
+func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) {
+	f, err := readMoveFacts(ctx, open.inventory)
+	if err != nil {
+		return "", err
+	}
+	for _, node := range pending {
+		moves, err := gatedMoves(ctx, open, node, nil)
+		if errors.Is(err, errWalkedElsewhere) {
+			continue // sendKept says it waits
+		}
+		if err != nil {
+			return "", err
+		}
+		kept := r.Gates[node].Carried
+		for _, mv := range moves {
+			if f.gated(mv.Module, mv.To) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool {
+				return c.Module == mv.Module && sameCommit(c.To, mv.To)
+			}) {
+				continue
+			}
+			return fmt.Sprintf("%s's build %s, which passed no gate, came to wait on %s", mv.Module, short(mv.To), node), nil
+		}
+	}
+	return "", nil
+}
+
 // reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the
 // declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last.
 func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool {
@@ -1136,7 +1198,11 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s
 		waiting[node] = moves
 		nodes = append(nodes, node)
 	}
-	step := restStep(nodes, waiting, f.gated)
+	shelf, err := open.inventory.Catalogue(ctx)
+	if err != nil {
+		return false, "", err
+	}
+	step := restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) })
 	if len(step) == 0 {
 		if len(walked) > 0 {
 			return false, "waiting before " + strings.Join(walked, "; "), nil
@@ -1156,16 +1222,30 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s
 	return true, "", nil
 }
 
+// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the
+// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus.
+func oneAtATime(module string, m catalogue.Manifest) bool {
+	if module == hostModule {
+		return true
+	}
+	return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat })
+}
+
 // restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless
 // a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's
 // first machine. Pure.
-func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool) []string {
+//
+// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a
+// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus
+// shows only as its silence and no rollback reaches it.
+func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool,
+	alone func(module string) bool) []string {
 	first := map[string]string{}
 	var out []string
 	for _, n := range nodes {
 		var unpassed []string
 		for _, mv := range waiting[n] {
-			if !passed(mv.Module, mv.To) {
+			if !passed(mv.Module, mv.To) || alone(mv.Module) {
 				unpassed = append(unpassed, mv.Module+"@"+mv.To)
 			}
 		}
@@ -1180,6 +1260,33 @@ func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed
 	return out
 }
 
+// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353
+// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing.
+func silentObservation(ctx context.Context, inv *inventory.Inventory) []conditions.Observation {
+	recent, err := inv.RecentPlans(ctx, 20)
+	if err != nil {
+		return nil
+	}
+	for _, p := range recent {
+		if p.Release == nil || p.Release.Settings != nil || p.Open() {
+			continue
+		}
+		if len(p.Release.Silent) == 0 {
+			return nil
+		}
+		machines := append([]string(nil), p.Release.Silent...)
+		sort.Strings(machines)
+		o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "silent", Kind: "release-silent",
+			Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines,
+			Summary: fmt.Sprintf("%s: %s said nothing on the send of the builds that waited for a gate within %s, and were "+
+				"left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)}
+		w := releaseSilentWords(machines)
+		o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs
+		return []conditions.Observation{o}
+	}
+	return nil
+}
+
 // backlogObservation is what the gate's probe says of a release held after a failure.
 func backlogObservation() []conditions.Observation {
 	if backlogNow.held == "" || len(backlogNow.waiting) == 0 {

cmd/mesh-controller/release_plan.go

@@ -1059,16 +1059,33 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 	// **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan
 	// before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from
 	// the readings of a gate already judging it there.
-	if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 {
+	// Only a send the machine reported it applied is one it has; one it never reported on is sent, as before
+	// (issue 280: what it ran before is then that build, and a failure leaves it as it was).
+	if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 &&
+		appliedLastSend(ctx, inv, node) {
 		if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 {
-			adoptSend(ctx, open, p, node, modules, runningOf)
+			adoptSend(ctx, open, p, node, modules, before, runningOf)
 			return nil
 		}
 	}
+	// **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced
+	// between the send and keeping it reads it here, and a failure still puts the machine back.
+	kept := false
+	for _, m := range modules {
+		s := p.Modules[m]
+		if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] {
+			s.Previous, kept = before[m], true
+		}
+	}
+	if kept {
+		if err := inv.SavePlan(ctx, p); err != nil {
+			return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err)
+		}
+	}
 	owns := make([]inventory.CarriedMove, 0, len(modules))
 	for _, m := range modules {
 		s := p.Modules[m]
-		owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: before[m], To: s.Commit, Build: s.Build})
+		owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build})
 	}
 	// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
 	// all of it there.
@@ -1083,10 +1100,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 	sentWhat := sentNow(ctx, inv, sent)
 	for _, m := range modules {
 		s := p.Modules[m]
-		// What the first machine ran before: what a failed gate puts back (ADR 0236).
-		if beforeKnown {
-			s.Previous = before[m]
-		}
+		// What the first machine ran before (ADR 0236), kept above before the send.
 		s.First, s.FirstAt = sent, &now
 		s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]),
 			From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat}
@@ -1119,7 +1133,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 // from the machine's last send, and takes over the readings of an open gate of another walk judging the same build
 // there — whose carried move also says what the machine ran before, for a rollback.
 func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string,
-	runningOf map[string][]string) {
+	before map[string]string, runningOf map[string][]string) {
 	inv := open.inventory
 	since := time.Now().UTC()
 	last, found, err := inv.SentTo(ctx, node)
@@ -1127,11 +1141,17 @@ func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 		since = last.At
 	}
 	lead := modules[0]
-	other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit)
+	other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build)
 	for _, m := range modules {
 		s := p.Modules[m]
-		if m == lead && from != "" {
+		// What it ran before: kept by this walk before its send was made (§6); else what the other walk's send
+		// moved it from; else the build itself, so a failure leaves it as it was (issue 280).
+		switch {
+		case s.Previous != "":
+		case m == lead && from != "":
 			s.Previous = from
+		default:
+			s.Previous = before[m]
 		}
 		at := since
 		s.First, s.FirstAt = []string{node}, &at
@@ -1156,13 +1176,34 @@ func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string
 	fmt.Printf("%s: %s\n", p.ID, p.Note)
 }
 
+// appliedLastSend says a machine reported it applied the declaration it was sent last.
+func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool {
+	reports, err := inv.LastReports(ctx)
+	if err != nil {
+		return false
+	}
+	for _, r := range reports {
+		if r.Node == node {
+			return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied
+		}
+	}
+	return false
+}
+
 // judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the
 // walk's id, and what the machine ran before; nil when none does.
-func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit string) (*inventory.PlanGate, string, string) {
+func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) {
 	plans, err := inv.OpenPlans(ctx)
 	if err != nil {
 		return nil, "", ""
 	}
+	// The same build, by its id where both name one, else by its commit.
+	same := func(b, c string) bool {
+		if b != "" && build != "" {
+			return b == build
+		}
+		return sameCommit(c, commit)
+	}
 	usable := func(g *inventory.PlanGate) bool {
 		return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) &&
 			slices.Contains(g.Machines, node)
@@ -1181,14 +1222,14 @@ func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, n
 					continue
 				}
 				for _, c := range g.Carried {
-					if c.Module == module && c.Node == node && sameCommit(c.To, commit) {
+					if c.Module == module && c.Node == node && same(c.Build, c.To) {
 						return g, q.ID, c.From
 					}
 				}
 			}
 			continue
 		}
-		if s := q.Modules[module]; s != nil && usable(s.Gate) && sameCommit(s.Commit, commit) {
+		if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) {
 			return s.Gate, q.ID, s.Previous
 		}
 	}

cmd/mesh-controller/rest_at_once_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/inventory"
 )
 
@@ -165,9 +166,18 @@ func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
 	judgeLeavesAsTheCore(t)
 	g := aGateMesh(t)
 	ctx := t.Context()
-	// The old controller sent anchor app c2 and was replaced before it kept the send.
-	if _, err := sendRollout(ctx, g.open, []string{"anchor"}); err != nil {
-		t.Fatal(err)
+	// The old controller sends anchor app c2 and is replaced before it keeps the send.
+	made := sendRollout
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		if _, err := made(ctx, open, names); err != nil {
+			return nil, err
+		}
+		return nil, context.Canceled
+	}
+	advancePlans(ctx, g.open)
+	sendRollout = made
+	if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil {
+		t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent)
 	}
 	last, _, err := g.open.inventory.SentTo(ctx, "anchor")
 	if err != nil {
@@ -181,6 +191,10 @@ func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
 	if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) {
 		t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate)
 	}
+	// What anchor ran before was kept before the send, so a failure still puts it back (§5).
+	if s.Previous != "c1" || s.Gate.From != "c1" {
+		t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From)
+	}
 }
 
 // A release's send kept before it was made, and made, is judged by the controller that takes over without being
@@ -224,11 +238,131 @@ func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) {
 		"novox":  {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}},
 	}
 	passed := func(module, commit string) bool { return module == "app" && commit == "c2" }
-	step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed)
+	never := func(string) bool { return false }
+	step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never)
 	if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
 		t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step)
 	}
-	if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }); !slices.Equal(step, []string{"laptop"}) {
+	if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) {
 		t.Fatalf("a machine whose every build passed waits: %v", step)
 	}
 }
+
+// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step
+// carries either to one machine only, while what else passed goes everywhere at once.
+func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) {
+	waiting := map[string][]inventory.CarriedMove{
+		"desk":   {{Module: hostModule, To: "h2"}},
+		"laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}},
+		"shanks": {{Module: "app", To: "c2"}},
+		"novox":  {{Module: "dns", To: "d2"}},
+		"ace":    {{Module: "dns", To: "d2"}},
+	}
+	resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}}
+	alone := func(m string) bool {
+		if m == "dns" {
+			return oneAtATime(m, resolver)
+		}
+		return oneAtATime(m, catalogue.Manifest{Module: m})
+	}
+	step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting,
+		func(string, string) bool { return true }, alone)
+	if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
+		t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step)
+	}
+	if oneAtATime("app", catalogue.Manifest{Module: "app"}) {
+		t.Fatal("a leaf module is held to one machine at a time")
+	}
+}
+
+// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent.
+func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) {
+	b := aBacklogOnThree(t)
+	ctx := t.Context()
+	gateEvery = 0
+	b.broken["anchor"] = true
+	for i := 0; i < 6 && b.release(t).Open(); i++ {
+		advancePlans(ctx, b.open)
+	}
+	if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest {
+		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
+	}
+	for _, s := range b.sent {
+		if !slices.Equal(s, []string{"anchor"}) {
+			t.Fatalf("sent %v: the rest after a failed first machine", b.sent)
+		}
+	}
+}
+
+// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is
+// ordered after every other.
+func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklogOnThree(t)
+	ctx := t.Context()
+	register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"})
+	if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil {
+		t.Fatal(err)
+	}
+	said := events(t)
+	gateEvery = 0
+	for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ {
+		advancePlans(ctx, b.open)
+	}
+	if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) {
+		t.Fatalf("the order is %v", p.Release.Order)
+	}
+	laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor")
+	if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") {
+		t.Fatalf("the control node is not sent last in the rest's step: %v", *said)
+	}
+}
+
+// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
+// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4).
+func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	b := aBacklog(t)
+	ctx := t.Context()
+	inv := b.open.inventory
+	reporting := sendRollout
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
+			b.sent = append(b.sent, names)
+			current, _ := open.inventory.CurrentBuilds(ctx)
+			return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
+				map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
+		}
+		return reporting(ctx, open, names)
+	}
+	t.Cleanup(func() { sendRollout = reporting })
+	gateEvery = 0
+	for i := 0; i < 8; i++ {
+		advancePlans(ctx, b.open)
+	}
+	p := b.release(t)
+	g := p.Release.Gates["laptop"]
+	if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
+		t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
+	}
+	since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
+	g.Since = &since
+	if err := inv.SavePlan(ctx, &p); err != nil {
+		t.Fatal(err)
+	}
+	advancePlans(ctx, b.open)
+	p = b.release(t)
+	if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
+		t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
+	}
+	if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
+		t.Fatal("a silent machine's build was marked failed")
+	}
+	if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
+		t.Fatalf("laptop was put back: %v", sent)
+	}
+	obs := silentObservation(ctx, inv)
+	if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
+		t.Fatalf("the silence is not said: %+v", obs)
+	}
+}

testdata/store-tests

@@ -66,3 +66,6 @@
 ./cmd/mesh-controller TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere
 ./cmd/mesh-controller TestAFirstSendMadeAndNotKeptIsNotSentAgain
 ./cmd/mesh-controller TestAReleaseSendKeptAndMadeIsNotMadeAgain
+./cmd/mesh-controller TestAFirstMachineThatFailsStillStopsTheWalk
+./cmd/mesh-controller TestTheControlNodeIsSentLastInAStep
+./cmd/mesh-controller TestARestMachineSilentPastItsBoundIsLeftAndSaid