Say a read error of a silent rest, and keep the node-engine one at a time in a kept step (hq ADR 0353)

An empty answer on a read error would clear release-silent unseen.

cmd/mesh-controller/gate.go

@@ -1444,7 +1444,11 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error
 	// A release held after a failed one, while builds still wait for a gate (ADR 0236).
 	out = append(out, backlogObservation()...)
 	// And a rest that went silent (novox/hq ADR 0353 §4).
-	out = append(out, silentObservation(ctx, d.open.inventory)...)
+	silent, err := silentObservation(ctx, d.open.inventory)
+	if err != nil {
+		return nil, err
+	}
+	out = append(out, silent...)
 	return sortedFound(dedupeObservations(out)), nil
 }
 

cmd/mesh-controller/release.go

@@ -1103,6 +1103,10 @@ func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, p
 	if err != nil {
 		return "", err
 	}
+	shelf, err := open.inventory.Catalogue(ctx)
+	if err != nil {
+		return "", err
+	}
 	for _, node := range pending {
 		moves, err := gatedMoves(ctx, open, node, nil)
 		if errors.Is(err, errWalkedElsewhere) {
@@ -1113,12 +1117,13 @@ func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, p
 		}
 		kept := r.Gates[node].Carried
 		for _, mv := range moves {
-			if f.gated(mv.Module, mv.To) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool {
+			// The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9).
+			if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool {
 				return c.Module == mv.Module && sameCommit(c.To, mv.To)
 			}) {
 				continue
 			}
-			return fmt.Sprintf("%s's build %s, which passed no gate, came to wait on %s", mv.Module, short(mv.To), node), nil
+			return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil
 		}
 	}
 	return "", nil
@@ -1262,17 +1267,17 @@ func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed
 
 // silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353
 // §4): machines that never reported on their send, left as they were — loud, never a success said by nothing.
-func silentObservation(ctx context.Context, inv *inventory.Inventory) []conditions.Observation {
+func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) {
 	recent, err := inv.RecentPlans(ctx, 20)
 	if err != nil {
-		return nil
+		return nil, err
 	}
 	for _, p := range recent {
 		if p.Release == nil || p.Release.Settings != nil || p.Open() {
 			continue
 		}
 		if len(p.Release.Silent) == 0 {
-			return nil
+			return nil, nil
 		}
 		machines := append([]string(nil), p.Release.Silent...)
 		sort.Strings(machines)
@@ -1282,9 +1287,9 @@ func silentObservation(ctx context.Context, inv *inventory.Inventory) []conditio
 				"left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)}
 		w := releaseSilentWords(machines)
 		o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs
-		return []conditions.Observation{o}
+		return []conditions.Observation{o}, nil
 	}
-	return nil
+	return nil, nil
 }
 
 // backlogObservation is what the gate's probe says of a release held after a failure.

cmd/mesh-controller/rest_at_once_test.go

@@ -281,6 +281,7 @@ func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) {
 	ctx := t.Context()
 	gateEvery = 0
 	b.broken["anchor"] = true
+	advancePlans(ctx, b.open)
 	for i := 0; i < 6 && b.release(t).Open(); i++ {
 		advancePlans(ctx, b.open)
 	}
@@ -361,7 +362,10 @@ func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
 	if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
 		t.Fatalf("laptop was put back: %v", sent)
 	}
-	obs := silentObservation(ctx, inv)
+	obs, err := silentObservation(ctx, inv)
+	if err != nil {
+		t.Fatal(err)
+	}
 	if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
 		t.Fatalf("the silence is not said: %+v", obs)
 	}