Keep a walk's sends to what was kept, and say a silent rest (hq ADR 0353 review)

A release's own gates held its other sends back for ever; an adopted send
lost what the machine ran before; a kept send could grow a build no gate
passed; a take-over matched by commit alone; a rest that went silent ended
done. The node-engine and the resolver still go one machine at a time, since
a fault that cuts a machine off shows only as silence.

cmd/mesh-controller/release.go

@@ -81,6 +81,15 @@ type moveFacts struct {
 	srcs   map[string]map[string]string
 	passed map[string]map[string]bool
 	plans  []inventory.Plan
+	// self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353).
+	self string
+}
+
+type walkingKey struct{}
+
+// withWalking says a release plan is the one sending: its own gates never hold its own sends back.
+func withWalking(ctx context.Context, id string) context.Context {
+	return context.WithValue(ctx, walkingKey{}, id)
 }
 
 func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
@@ -99,6 +108,7 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er
 		return f, err
 	}
 	f.plans, err = inv.OpenPlans(ctx)
+	f.self, _ = ctx.Value(walkingKey{}).(string)
 	return f, err
 }
 
@@ -193,7 +203,7 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string,
 // feet. A release keeps no module records: its open gate's carried moves are its walk.
 func (f moveFacts) walkedBy(module, node, to string) string {
 	for _, p := range f.plans {
-		if !p.Open() {
+		if !p.Open() || (p.Release != nil && p.ID == f.self) {
 			continue
 		}
 		if p.Release != nil {
@@ -203,7 +213,8 @@ func (f moveFacts) walkedBy(module, node, to string) string {
 				gates = append(gates, g)
 			}
 			for _, g := range gates {
-				if g == nil || g.Verdict != "" || g.Silent {
+				// A send kept and not yet made walks nothing yet (ADR 0353 §8).
+				if g == nil || g.Verdict != "" || g.Silent || g.Pending {
 					continue
 				}
 				for _, c := range g.Carried {
@@ -734,6 +745,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
 	if r.Settings != nil {
 		return advanceSettingsWalk(ctx, open, p)
 	}
+	ctx = withWalking(ctx, p.ID)
 	now := time.Now().UTC()
 	if r.Rest {
 		return advanceRest(ctx, open, p, now)
@@ -915,6 +927,24 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
 		}
 	}
 	if len(pending) > 0 {
+		// **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on
+		// a machine since its step was chosen could reach a second machine in it; the step is chosen again.
+		if grew, err := keptStepGrew(ctx, open, r, pending); err != nil {
+			return false, err
+		} else if grew != "" {
+			for _, n := range pending {
+				delete(r.Gates, n)
+			}
+			opened, waiting, err := openRestStep(ctx, open, p)
+			if err != nil {
+				return true, err
+			}
+			p.Note = "chosen again: " + grew
+			if !opened && waiting != "" {
+				p.Note += "; " + waiting
+			}
+			return true, nil
+		}
 		var sent, already, waiting []string
 		for _, node := range pending {
 			out, err := sendKept(ctx, open, p, node, r.Gates[node], now)
@@ -1056,12 +1086,44 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
 	if len(r.Failed) > 0 {
 		p.State = inventory.PlanFailed
 		end = "failed on " + strings.Join(r.Failed, ", ") + "; " + end
+	} else if len(r.Silent) > 0 && len(r.Gates) == len(r.Silent) {
+		// **A rest that said nothing is no success** (ADR 0353 §4): failure is loud; the condition says it.
+		p.State = inventory.PlanFailed
+		end = "no machine of the rest reported on its send; " + end
 	}
 	p.Note = strings.Join(append(notes, end), "; ")
 	fmt.Printf("%s: %s\n", p.ID, p.Note)
 	return true, nil
 }
 
+// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with;
+// empty when none would.
+func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) {
+	f, err := readMoveFacts(ctx, open.inventory)
+	if err != nil {
+		return "", err
+	}
+	for _, node := range pending {
+		moves, err := gatedMoves(ctx, open, node, nil)
+		if errors.Is(err, errWalkedElsewhere) {
+			continue // sendKept says it waits
+		}
+		if err != nil {
+			return "", err
+		}
+		kept := r.Gates[node].Carried
+		for _, mv := range moves {
+			if f.gated(mv.Module, mv.To) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool {
+				return c.Module == mv.Module && sameCommit(c.To, mv.To)
+			}) {
+				continue
+			}
+			return fmt.Sprintf("%s's build %s, which passed no gate, came to wait on %s", mv.Module, short(mv.To), node), nil
+		}
+	}
+	return "", nil
+}
+
 // reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the
 // declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last.
 func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool {
@@ -1136,7 +1198,11 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s
 		waiting[node] = moves
 		nodes = append(nodes, node)
 	}
-	step := restStep(nodes, waiting, f.gated)
+	shelf, err := open.inventory.Catalogue(ctx)
+	if err != nil {
+		return false, "", err
+	}
+	step := restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) })
 	if len(step) == 0 {
 		if len(walked) > 0 {
 			return false, "waiting before " + strings.Join(walked, "; "), nil
@@ -1156,16 +1222,30 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s
 	return true, "", nil
 }
 
+// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the
+// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus.
+func oneAtATime(module string, m catalogue.Manifest) bool {
+	if module == hostModule {
+		return true
+	}
+	return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat })
+}
+
 // restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless
 // a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's
 // first machine. Pure.
-func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool) []string {
+//
+// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a
+// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus
+// shows only as its silence and no rollback reaches it.
+func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool,
+	alone func(module string) bool) []string {
 	first := map[string]string{}
 	var out []string
 	for _, n := range nodes {
 		var unpassed []string
 		for _, mv := range waiting[n] {
-			if !passed(mv.Module, mv.To) {
+			if !passed(mv.Module, mv.To) || alone(mv.Module) {
 				unpassed = append(unpassed, mv.Module+"@"+mv.To)
 			}
 		}
@@ -1180,6 +1260,33 @@ func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed
 	return out
 }
 
+// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353
+// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing.
+func silentObservation(ctx context.Context, inv *inventory.Inventory) []conditions.Observation {
+	recent, err := inv.RecentPlans(ctx, 20)
+	if err != nil {
+		return nil
+	}
+	for _, p := range recent {
+		if p.Release == nil || p.Release.Settings != nil || p.Open() {
+			continue
+		}
+		if len(p.Release.Silent) == 0 {
+			return nil
+		}
+		machines := append([]string(nil), p.Release.Silent...)
+		sort.Strings(machines)
+		o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "silent", Kind: "release-silent",
+			Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines,
+			Summary: fmt.Sprintf("%s: %s said nothing on the send of the builds that waited for a gate within %s, and were "+
+				"left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)}
+		w := releaseSilentWords(machines)
+		o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs
+		return []conditions.Observation{o}
+	}
+	return nil
+}
+
 // backlogObservation is what the gate's probe says of a release held after a failure.
 func backlogObservation() []conditions.Observation {
 	if backlogNow.held == "" || len(backlogNow.waiting) == 0 {