Keep builds on the bus under their module, so no lookup reads the bucket whole (issue 1121, ADR 0427)

Milestone 5 moved 8,599 build records onto the bus under bare build ids, each
carrying its module's whole manifest. Every build lookup then read the whole
bucket, and a delivery plan's send did that per module per machine while it
held the plans and the machine's declaration: the send of build-agent never
finished, and every other act and the status waited behind it.

Builds now live under <module>.build.<id> (requests under req.<id>, id.<id>
names a build's module), with pointer keys newest, worked, held,
commit.<sha> and source.<reuse key> that move only to a newer build and are
read as hints, repaired when wrong. Manifests are kept once in the same
bucket under manifest.sha256-<hex>, so no grant the running bus lacks is
needed. The old keys are laid out once at start, idempotently, and left
unread. The calls a plan look makes on the silent clocks while it holds the
plans are bounded, since the vendored kv.Keys ignores its context.

cmd/mesh-controller/plan_looks.go

@@ -412,6 +412,7 @@ func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time
 		byTimer()
 		return nil
 	}
+	c = boundedClock{c}
 	held, err := c.Held(ctx)
 	if err != nil {
 		byTimer()
@@ -435,6 +436,51 @@ func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time
 	return first
 }
 
+// silentCallBound is how long one call to the silent clock may take while the look holds the plans (novox/hq issue 1121).
+// The vendored key listing reads its watcher without the context, so a consumer lost when the broker reloaded its user
+// list left the look waiting for good, with the plans and every node held.
+var silentCallBound = 20 * time.Second
+
+// boundedClock is a clock whose every call is answered within silentCallBound, or fails: a call that never returns is
+// left behind on its own goroutine.
+type boundedClock struct{ silentClock }
+
+func boundedCall[T any](ctx context.Context, what string, call func(context.Context) (T, error)) (T, error) {
+	ctx, cancel := context.WithTimeout(ctx, silentCallBound)
+	defer cancel()
+	type answer struct {
+		v   T
+		err error
+	}
+	done := make(chan answer, 1)
+	go func() { v, err := call(ctx); done <- answer{v, err} }()
+	select {
+	case a := <-done:
+		return a.v, a.err
+	case <-ctx.Done():
+		var zero T
+		return zero, fmt.Errorf("%s: no answer from the bus within %s: %w", what, silentCallBound, ctx.Err())
+	}
+}
+
+func (b boundedClock) Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error {
+	_, err := boundedCall(ctx, "arming a wait", func(ctx context.Context) (struct{}, error) {
+		return struct{}{}, b.silentClock.Arm(ctx, w, ttl, sent)
+	})
+	return err
+}
+
+func (b boundedClock) Held(ctx context.Context) (map[silentWait]time.Time, error) {
+	return boundedCall(ctx, "listing the waits", b.silentClock.Held)
+}
+
+func (b boundedClock) Reported(ctx context.Context, w silentWait) error {
+	_, err := boundedCall(ctx, "dropping a wait", func(ctx context.Context) (struct{}, error) {
+		return struct{}{}, b.silentClock.Reported(ctx, w)
+	})
+	return err
+}
+
 var silentTimers = struct {
 	mu     sync.Mutex
 	timers map[silentWait]time.Time

cmd/mesh-controller/plan_looks_test.go

@@ -179,3 +179,32 @@ func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) {
 
 // A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is
 // looked at: the controller follows its own plan-moved (issue 974, round 1).
+
+// A clock whose bus answers nothing, ever: a consumer lost when the broker reloaded its user list, a key listing that
+// never reaches its end.
+type hungClock struct{ silentClock }
+
+func (hungClock) Held(context.Context) (map[silentWait]time.Time, error) {
+	select {} // never answers, whatever the context says (the vendored Keys reads its watcher without it)
+}
+
+// issue 1121: the look holds the plans while it keeps the silent keys, so a key listing that never ends held the plans
+// and every node for good. Every call to the clock has a bound, and the wait is looked at by a timer of its own.
+func TestAClockThatNeverAnswersDoesNotHoldTheLookForever(t *testing.T) {
+	was := silentCallBound
+	silentCallBound = 50 * time.Millisecond
+	t.Cleanup(func() { silentCallBound = was })
+	w := silentWait{Plan: "plan-hung", Tier: 0, Node: "ace"}
+	sent := time.Now().UTC()
+	t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() })
+	done := make(chan error, 1)
+	go func() { done <- keepSilentKeys(t.Context(), hungClock{}, map[silentWait]time.Time{w: sent}, sent) }()
+	select {
+	case err := <-done:
+		if err == nil {
+			t.Fatal("a clock that never answered was reported as fine")
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("the look is still waiting for a clock that never answers")
+	}
+}