Merge fix/1121-holds-released: delivery 2 on the builds' new layout (issue 1121)

Delivery 2 deleted the move and chain_moved, so the layout of the builds under their modules is made by the controller's start (LayBuildsOut), recorded by the key layout.done in the builds bucket, retried every half minute with an urgent condition while it fails; the chain on the bus is refused until it exists. Its source readers use the module and pointer reads.

cmd/mesh-controller/chain.go

@@ -42,6 +42,9 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
 		return err
 	}
 	inv.UseChain(c)
+	inv.RequireLayout()
+	// And the builds laid out under their modules, once, before they are read (issue 1121).
+	go layBuildsOutSoon(ctx, inv)
 	// And the requests the move of delivery 1 found never handed over, asked again once (askAgain), once it is serving.
 	go askAgainSoon(ctx, inv)
 	// And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds).
@@ -130,3 +133,39 @@ func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) {
 
 // chainMoveRetry is how long the copy waits before it tries again.
 const chainMoveRetry = 30 * time.Second
+
+// kindBuildsNotLaidOut is the condition of the layout of the builds under their modules that has not succeeded: until it
+// has, no build is read, and a send that needs one is refused with the reason.
+const kindBuildsNotLaidOut = "builds-not-laid-out"
+
+// layBuildsOutSoon lays the builds out under their modules as soon as this controller holds the lease (the chain refuses a
+// write before), and tries again every half minute while it fails, loudly: each failure is printed, and the first raises an
+// urgent condition until it has worked. It is idempotent (LayBuildsOut).
+func layBuildsOutSoon(ctx context.Context, inv *inventory.Inventory) {
+	failed := false
+	for {
+		err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) })
+		o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindBuildsNotLaidOut, Token: kindBuildsNotLaidOut,
+			Kind: kindBuildsNotLaidOut, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
+			Source: "chain", Summary: "the builds are not laid out under their modules, so none is read: "}
+		if err != nil {
+			fmt.Printf("chain: the builds are not laid out under their modules, and none is read until they are: %v\n", err)
+			if keeper := conditionsFrom; keeper != nil && !failed {
+				o.Summary += err.Error()
+				if _, oerr := keeper.Observe(ctx, o); oerr == nil {
+					failed = true
+				}
+			}
+		} else {
+			if keeper := conditionsFrom; keeper != nil && failed {
+				_, _ = keeper.Clear(ctx, o.Key(), "laid out")
+			}
+			return
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(chainMoveRetry):
+		}
+	}
+}

cmd/mesh-controller/plan_looks.go

@@ -412,6 +412,7 @@ func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time
 		byTimer()
 		return nil
 	}
+	c = boundedClock{c}
 	held, err := c.Held(ctx)
 	if err != nil {
 		byTimer()
@@ -435,6 +436,51 @@ func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time
 	return first
 }
 
+// silentCallBound is how long one call to the silent clock may take while the look holds the plans (novox/hq issue 1121).
+// The vendored key listing reads its watcher without the context, so a consumer lost when the broker reloaded its user
+// list left the look waiting for good, with the plans and every node held.
+var silentCallBound = 20 * time.Second
+
+// boundedClock is a clock whose every call is answered within silentCallBound, or fails: a call that never returns is
+// left behind on its own goroutine.
+type boundedClock struct{ silentClock }
+
+func boundedCall[T any](ctx context.Context, what string, call func(context.Context) (T, error)) (T, error) {
+	ctx, cancel := context.WithTimeout(ctx, silentCallBound)
+	defer cancel()
+	type answer struct {
+		v   T
+		err error
+	}
+	done := make(chan answer, 1)
+	go func() { v, err := call(ctx); done <- answer{v, err} }()
+	select {
+	case a := <-done:
+		return a.v, a.err
+	case <-ctx.Done():
+		var zero T
+		return zero, fmt.Errorf("%s: no answer from the bus within %s: %w", what, silentCallBound, ctx.Err())
+	}
+}
+
+func (b boundedClock) Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error {
+	_, err := boundedCall(ctx, "arming a wait", func(ctx context.Context) (struct{}, error) {
+		return struct{}{}, b.silentClock.Arm(ctx, w, ttl, sent)
+	})
+	return err
+}
+
+func (b boundedClock) Held(ctx context.Context) (map[silentWait]time.Time, error) {
+	return boundedCall(ctx, "listing the waits", b.silentClock.Held)
+}
+
+func (b boundedClock) Reported(ctx context.Context, w silentWait) error {
+	_, err := boundedCall(ctx, "dropping a wait", func(ctx context.Context) (struct{}, error) {
+		return struct{}{}, b.silentClock.Reported(ctx, w)
+	})
+	return err
+}
+
 var silentTimers = struct {
 	mu     sync.Mutex
 	timers map[silentWait]time.Time

cmd/mesh-controller/plan_looks_test.go

@@ -179,3 +179,32 @@ func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) {
 
 // A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is
 // looked at: the controller follows its own plan-moved (issue 974, round 1).
+
+// A clock whose bus answers nothing, ever: a consumer lost when the broker reloaded its user list, a key listing that
+// never reaches its end.
+type hungClock struct{ silentClock }
+
+func (hungClock) Held(context.Context) (map[silentWait]time.Time, error) {
+	select {} // never answers, whatever the context says (the vendored Keys reads its watcher without it)
+}
+
+// issue 1121: the look holds the plans while it keeps the silent keys, so a key listing that never ends held the plans
+// and every node for good. Every call to the clock has a bound, and the wait is looked at by a timer of its own.
+func TestAClockThatNeverAnswersDoesNotHoldTheLookForever(t *testing.T) {
+	was := silentCallBound
+	silentCallBound = 50 * time.Millisecond
+	t.Cleanup(func() { silentCallBound = was })
+	w := silentWait{Plan: "plan-hung", Tier: 0, Node: "ace"}
+	sent := time.Now().UTC()
+	t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() })
+	done := make(chan error, 1)
+	go func() { done <- keepSilentKeys(t.Context(), hungClock{}, map[silentWait]time.Time{w: sent}, sent) }()
+	select {
+	case err := <-done:
+		if err == nil {
+			t.Fatal("a clock that never answered was reported as fine")
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("the look is still waiting for a clock that never answers")
+	}
+}