Send the bus's machine before the grants, and only when its user list moved (hq issue 249)

Grants first could hold back the very declaration that lets the controller
issue them. The holder now goes first, then buckets and memberships, then
the rest; a membership that fails holds back only its own machine, and an
announcement whose send stopped at its grants is asked again. Whether the
holder must go first is read from a digest of the user list it was last
sent, not its whole declaration. Migration renumbered to 0058.

cmd/mesh-controller/delivery_order_test.go

@@ -6,6 +6,8 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+
+	"github.com/novox/mesh-controller/internal/link"
 )
 
 // recordingDelivery is a delivery that writes down what was done, in order, and fails where told.
@@ -39,10 +41,11 @@ func ready(names ...string) []readyNode {
 }
 
 // novox/hq issue 249: the grants that come with a module's new declarations are issued before any
-// machine is sent the code that uses them.
+// machine is sent the code that uses them — except the machine holding the bus, whose declaration
+// carries the controller's own right to issue them, and goes first.
 func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) {
 	d := &recordingDelivery{}
-	digests, err := deliver(t.Context(), d, ready("anchor", "laptop"))
+	digests, err := deliver(t.Context(), d, "", ready("anchor", "laptop"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -53,27 +56,71 @@ func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) {
 	if digests["anchor"] != "digest-anchor" || digests["laptop"] != "digest-laptop" {
 		t.Fatalf("the digests sent were not answered: %v", digests)
 	}
+
+	held := &recordingDelivery{}
+	if _, err := deliver(t.Context(), held, "broker", ready("broker", "anchor")); err != nil {
+		t.Fatal(err)
+	}
+	want = []string{"declare broker", "grant broker", "grant anchor", "declare anchor"}
+	if !reflect.DeepEqual(held.did, want) {
+		t.Fatalf("with the bus's machine in the send: %v, wanted %v", held.did, want)
+	}
 }
 
-// A grant that cannot be issued sends nothing and is an error — never "until the next push".
-func TestAGrantThatFailsSendsNothingAndSaysSo(t *testing.T) {
-	d := &recordingDelivery{grantErr: errors.New("the bus refused the membership")}
-	_, err := deliver(t.Context(), d, ready("anchor", "laptop"))
-	if err == nil || !strings.Contains(err.Error(), "the bus refused the membership") {
+// A grant that cannot be issued holds back the machines it concerns and is an error the caller
+// retries on — never "until the next push" — and the bus's own machine is sent regardless, so the
+// grant that would let the controller issue memberships is never held behind them.
+func TestAGrantThatFailsHoldsBackWhatItConcerns(t *testing.T) {
+	// A failure naming no machine (the buckets): everything but the bus's machine.
+	d := &recordingDelivery{grantErr: errors.New("the bus refused the bucket")}
+	_, err := deliver(t.Context(), d, "broker", ready("broker", "anchor", "laptop"))
+	if err == nil || !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "the bus refused the bucket") ||
+		!strings.Contains(err.Error(), "anchor, laptop not sent") {
 		t.Fatalf("a failed grant was not said as the send's failure: %v", err)
 	}
-	for _, did := range d.did {
-		if strings.HasPrefix(did, "declare") {
-			t.Fatalf("code was sent after its grant failed: %v", d.did)
-		}
+	if want := []string{"declare broker", "grant broker", "grant anchor", "grant laptop"}; !reflect.DeepEqual(d.did, want) {
+		t.Fatalf("delivered %v, wanted the bus's machine alone", d.did)
 	}
+
+	// A membership that failed for one machine: that machine alone.
+	one := &recordingDelivery{grantErr: &grantsRefused{nodes: map[string]error{"laptop": errors.New("no")}}}
+	_, err = deliver(t.Context(), one, "", ready("anchor", "laptop"))
+	if !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "laptop not sent") {
+		t.Fatalf("one machine's refused membership was not said: %v", err)
+	}
+	if want := []string{"grant anchor", "grant laptop", "declare anchor"}; !reflect.DeepEqual(one.did, want) {
+		t.Fatalf("delivered %v, wanted anchor sent and laptop held back", one.did)
+	}
+
+	// The announced upgrade that hit it is asked again.
+	if !errors.Is(askAgainOnGrants(err), link.ErrTryAgain) {
+		t.Fatal("an announcement whose send stopped at its grants is not asked again")
+	}
+	if other := errors.New("laptop could not be resolved"); errors.Is(askAgainOnGrants(other), link.ErrTryAgain) {
+		t.Fatal("any failure is asked again, not only a grant's")
+	}
+
 	// Nothing to send is nothing granted either.
 	none := &recordingDelivery{grantErr: errors.New("never asked")}
-	if _, err := deliver(t.Context(), none, nil); err != nil || len(none.did) != 0 {
+	if _, err := deliver(t.Context(), none, "", nil); err != nil || len(none.did) != 0 {
 		t.Fatalf("an empty send granted or failed: %v %v", none.did, err)
 	}
 }
 
+// Whether the bus's machine goes first is read from the user list alone, by its digest.
+func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
+	list := "users: [a, b]"
+	if userListBehind(list, digestOf([]byte(list))) {
+		t.Fatal("the list it was sent reads as behind")
+	}
+	if !userListBehind(list, digestOf([]byte("users: [a]"))) || !userListBehind(list, "") {
+		t.Fatal("a changed or never-sent list reads as current")
+	}
+	if userListBehind("", "") {
+		t.Fatal("a machine sent no list reads as behind")
+	}
+}
+
 // The machine holding the bus goes first: its declaration carries the user list the new grants are
 // checked against. Among the machines it is moved to the front; not among them it is added only
 // when it is behind.

cmd/mesh-controller/hold_test.go

@@ -23,11 +23,11 @@ func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
 
 	for name, round := range map[string]func() error{
 		"a body that cannot be marshalled": func() error {
-			_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine)
+			_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "")
 			return err
 		},
 		"a send that fails": func() error {
-			_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing)
+			_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "")
 			return err
 		},
 	} {

cmd/mesh-controller/plan.go

@@ -398,7 +398,7 @@ func declarationWith(ctx context.Context, open *stores, node string,
 		return sendable{}, err
 	}
 	return sendable{Resources: composed.Resources, Adoption: adoption,
-		Received: composed.Received, Mesh: with.Mesh,
+		Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers,
 		LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut}, nil
 }
 
@@ -1341,6 +1341,20 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
 	//
 	// Asked of what this push resolves to rather than of the seat's holder mesh-wide: the file is a
 	// resource of that module, so the question is whether it is here.
+	list, missing, err := busUserList(ctx, inv, onThisNode)
+	if len(missing) > 0 {
+		fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
+			"for: %s. Each is a user that cannot connect until one is issued\n",
+			len(missing), strings.Join(missing, ", "))
+	}
+	return list, err
+}
+
+// busUserList is composeBusUsers without saying anything: the list, and the users left out of it
+// for want of a credential. Asked on every send to decide whether the machine holding the bus must
+// go first (novox/hq issue 249), where saying the same missing users each time would bury them.
+func busUserList(ctx context.Context, inv *inventory.Inventory,
+	onThisNode []catalogue.Manifest) (string, []string, error) {
 	holdsTheBus := false
 	for _, m := range onThisNode {
 		if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
@@ -1348,37 +1362,33 @@ func composeBusUsers(ctx context.Context, inv *inventory.Inventory,
 		}
 	}
 	if !holdsTheBus {
-		return "", nil
+		return "", nil, nil
 	}
 
 	records, err := inv.BusRecords(ctx)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
 	users, err := broker.Users(records)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
 	kept, err := inv.BusUsers(ctx)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
 	hashes := make(map[string]string, len(kept))
 	for name, u := range kept {
 		hashes[name] = u.PasswordHash
 	}
 	filled, missing := broker.WithPasswords(users, hashes)
-	if len(missing) > 0 {
-		fmt.Printf("the bus's user list leaves out %d user(s) the mesh has minted no credential "+
-			"for: %s. Each is a user that cannot connect until one is issued\n",
-			len(missing), strings.Join(missing, ", "))
-	}
 	if len(filled) == 0 {
-		return "", fmt.Errorf(
+		return "", missing, fmt.Errorf(
 			"this machine runs the bus and not one user has a credential, so the composed list " +
 				"would refuse every connection in the mesh")
 	}
-	return broker.ComposeAccounts(filled)
+	list, err := broker.ComposeAccounts(filled)
+	return list, missing, err
 }
 
 // providerModuleOf is which module answers a need on the providing node: the one in this node's

cmd/mesh-controller/push.go

@@ -402,7 +402,7 @@ func pushCommand(ctx context.Context, args []string) error {
 	// Each machine's memberships first, then the declarations (novox/hq issue 249, ADR 0160): a push
 	// is the one most operators run, and on 2026-10-01 it was the one path that issued none.
 	bus := overTheBus{open: open, server: server, signer: ident}
-	sentDigest, err := deliver(ctx, bus, sending)
+	sentDigest, err := deliver(ctx, bus, holder, sending)
 	if err != nil {
 		return err
 	}
@@ -477,7 +477,7 @@ func pushCommand(ctx context.Context, args []string) error {
 					}
 					return declared, err
 				},
-				bus)
+				bus, holder)
 			refusals = append(refusals, refused...)
 			if err != nil {
 				return err
@@ -607,7 +607,7 @@ func composeEach(names []string, allot func(node string) (int64, error),
 // still sent. Their memberships go before their declarations, as every send's do (issue 249).
 func sendRound(ctx context.Context, open *stores, names []string,
 	compose func(held context.Context, node string) (sendable, error),
-	d delivery) ([]string, error) {
+	d delivery, holder string) ([]string, error) {
 	held, release, err := holdNodes(ctx, open, names)
 	if err != nil {
 		return nil, err
@@ -616,7 +616,7 @@ func sendRound(ctx context.Context, open *stores, names []string,
 	sending, refused := composeEach(names, allotting(held, open.inventory), func(node string) (sendable, error) {
 		return compose(held, node)
 	})
-	if _, err := deliver(held, d, sending); err != nil {
+	if _, err := deliver(held, d, holder, sending); err != nil {
 		return refused, err
 	}
 	return refused, nil
@@ -632,7 +632,28 @@ type delivery interface {
 	declare(ctx context.Context, s readyNode, body []byte) (string, error)
 }
 
-// deliver sends the machines their declarations, **the grants first** (novox/hq issue 249).
+// errGrants marks a send that stopped because what the machines' modules may do could not be issued
+// (novox/hq issue 249). Nothing about the machines is wrong; asked again, it is likely to work, so an
+// announcement that hits it is held and asked again.
+var errGrants = errors.New("what the machines' modules may do on the bus could not be issued, and code " +
+	"sent before its grants is refused there")
+
+// grantsRefused is a grant that failed for some machines and not others: their memberships could not
+// be issued, by machine, and only those machines are held back.
+type grantsRefused struct{ nodes map[string]error }
+
+func (g *grantsRefused) Error() string {
+	names := make([]string, 0, len(g.nodes))
+	for n := range g.nodes {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	return fmt.Sprintf("the memberships of %s could not be issued; the first: %v",
+		strings.Join(names, ", "), g.nodes[names[0]])
+}
+
+// deliver sends the machines their declarations: **the machine holding the bus, then the grants,
+// then the rest** (novox/hq issue 249).
 //
 // A merge gave a module a new state; its bundle reached every machine within a minute, and the
 // machines' permissions on the bus did not include the state until somebody pushed by hand: the code
@@ -642,31 +663,82 @@ type delivery interface {
 // the bus (internal/link/bus.go), so issued first it waits for the runtime that arrives after it. A
 // runtime still on the old code merely holds a grant it does not use yet.
 //
-// **A grant that cannot be issued sends nothing**, and says so as an error. It used to be said and
-// passed over — "the machines keep what they derive until the next push" — which reported a rollout
-// as done that had delivered code its machines could not run, and left the remedy to a push nobody
-// knew to make. Returned, the caller's rollout is not marked sent and is tried again.
-func deliver(ctx context.Context, d delivery, sending []readyNode) (map[string]string, error) {
+// **The holder's declaration before the grants, though.** The bus's user list travels in it, and the
+// controller's own right to publish memberships and raise buckets is in that list (the precedent of
+// issue 183): grants first, and a grant the controller is not yet allowed to make would hold the very
+// declaration that allows it — a lock only a hand on the broker could open. A runtime already running
+// on that machine follows a membership issued after its declaration, as it always has.
+//
+// **A grant that cannot be issued holds back what it concerns, and says so as an error.** It used to
+// be said and passed over — "the machines keep what they derive until the next push" — which reported
+// a rollout done that had delivered code its machines could not run. A membership that failed holds
+// back its own machine; a failure that names no machine (the buckets) holds back every machine but the
+// holder, already sent. The error carries errGrants, so the caller's rollout is not marked sent and is
+// tried again.
+//
+// The grants are issued, not waited on: a membership is a retained message the runtime reads when it
+// comes, and the bus answers its publication; nothing here waits for a runtime to have read one.
+func deliver(ctx context.Context, d delivery, holder string, sending []readyNode) (map[string]string, error) {
 	digests := map[string]string{}
 	if len(sending) == 0 {
 		return digests, nil
 	}
-	if err := d.grant(ctx, sending); err != nil {
-		return digests, fmt.Errorf("nothing was sent: what the machines' modules may do on the bus "+
-			"could not be issued, and code sent before its grants is refused there (novox/hq issue 249): %w", err)
-	}
-	for _, s := range sending {
+	send := func(s readyNode) error {
 		// The number is inside the signed bytes, so a replayed older declaration cannot borrow a
 		// newer one's (novox/hq 04-ISSUES/107); it was taken when the composition began (issue 204).
 		body, err := s.declared.Body()
 		if err != nil {
-			return digests, err
+			return err
 		}
 		digest, err := d.declare(ctx, s, body)
 		if err != nil {
-			return digests, err
+			return err
 		}
 		digests[s.node] = digest
+		return nil
+	}
+	var rest []readyNode
+	for _, s := range sending {
+		if holder != "" && s.node == holder {
+			if err := send(s); err != nil {
+				return digests, err
+			}
+			continue
+		}
+		rest = append(rest, s)
+	}
+	held := map[string]error{}
+	if err := d.grant(ctx, sending); err != nil {
+		var some *grantsRefused
+		if !errors.As(err, &some) {
+			var names []string
+			for _, s := range rest {
+				names = append(names, s.node)
+			}
+			if len(names) == 0 {
+				return digests, fmt.Errorf("%w: %w", errGrants, err)
+			}
+			return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(names, ", "), err)
+		}
+		held = some.nodes
+	}
+	var notSent []string
+	for _, s := range rest {
+		if _, refused := held[s.node]; refused {
+			notSent = append(notSent, s.node)
+			continue
+		}
+		if err := send(s); err != nil {
+			return digests, err
+		}
+	}
+	if len(notSent) > 0 {
+		return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(notSent, ", "),
+			&grantsRefused{nodes: held})
+	}
+	if len(held) > 0 {
+		// Only the holder's own memberships failed, and it was sent before them.
+		return digests, fmt.Errorf("%w: %w", errGrants, &grantsRefused{nodes: held})
 	}
 	return digests, nil
 }
@@ -694,6 +766,16 @@ func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (stri
 	if err != nil {
 		return "", err
 	}
+	if s.declared.BusUsers != "" {
+		// And the user list it carried, so the next send reads whether it must go first from the
+		// list alone (novox/hq issue 249). On the same outliving context as the send's record.
+		kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+		err := b.open.inventory.RecordSentBusUsers(kept, s.node, digestOf([]byte(s.declared.BusUsers)))
+		cancel()
+		if err != nil {
+			return "", err
+		}
+	}
 	fmt.Printf("%ssent %s %d resource(s)\n", b.indent, s.node, len(s.declared.Resources))
 	return digest, nil
 }
@@ -722,14 +804,13 @@ func brokerFirst(names []string, holder string, behind bool) []string {
 }
 
 // brokerBehind is the machine holding the bus — the one whose declaration carries the user list —
-// and, when it is not among the machines named, whether what it should be differs from what it was
-// last sent.
+// and, when it is not among the machines named, whether the user list it would be sent now differs
+// from the one it was last sent (novox/hq issue 249).
 //
-// **Its whole declaration, because that is what is recorded** (novox/hq issue 249). The mesh keeps
-// a digest of what each machine was last sent and not of the user list inside it (ADR 0043: the
-// list is composed on each push, never kept), so "the user list changed" is read as "the bus's
-// machine is behind". That is the safe direction: a machine sent what it should be is never wrong,
-// and whatever else it was behind on is what any push would have sent it.
+// **The user list alone, not the whole declaration.** Read from the whole declaration, any change
+// pending on that machine — an upgrade its policy records rather than rolls out — went with every
+// send anywhere, and a module running there always put it in its first wave. A digest of the list
+// last sent is kept for this (ADR 0043: the list is composed on each push, never kept itself).
 func brokerBehind(ctx context.Context, open *stores, names []string) (string, bool, error) {
 	inv := open.inventory
 	holders, err := seatHolders(ctx, inv)
@@ -753,23 +834,26 @@ func brokerBehind(ctx context.Context, open *stores, names []string) (string, bo
 			return h.Node, false, nil
 		}
 	}
-	node, err := inv.NodeByName(ctx, h.Node)
+	plan, _, err := planFor(ctx, open, h.Node)
 	if err != nil {
-		return "", false, err
-	}
-	would, err := wouldSend(ctx, open, []inventory.Node{node})
-	if err != nil {
-		return "", false, err
-	}
-	if would[h.Node] == "" {
 		// It cannot be worked out: sending it would refuse the whole send, and `plan` says why.
 		return h.Node, false, nil
 	}
-	sent, err := inv.Outstanding(ctx, h.Node)
+	list, _, err := busUserList(ctx, inv, plan.Modules)
+	if err != nil {
+		return h.Node, false, nil
+	}
+	sent, err := inv.SentBusUsers(ctx, h.Node)
 	if err != nil {
 		return "", false, err
 	}
-	return h.Node, would[h.Node] != sent, nil
+	return h.Node, userListBehind(list, sent), nil
+}
+
+// userListBehind is whether the user list composed now is not the one last sent, by its digest. An
+// empty list composed is never behind: there is nothing for it to carry.
+func userListBehind(now, sentDigest string) bool {
+	return now != "" && digestOf([]byte(now)) != sentDigest
 }
 
 // couldNotBeResolved is what a push ends with when some machines could not be worked out.
@@ -867,7 +951,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er
 	// composition. **Issued before the declarations** (novox/hq issue 249): the runtime the
 	// membership is for arrives with the declaration, and a membership waits for it on the bus; the
 	// code arriving first was refused its own state until somebody pushed.
-	if _, err := deliver(ctx, overTheBus{open: open, server: server, signer: ident, indent: "  "}, sending); err != nil {
+	if _, err := deliver(ctx, overTheBus{open: open, server: server, signer: ident, indent: "  "}, holder, sending); err != nil {
 		return nil, err
 	}
 	sent := make([]string, 0, len(sending))
@@ -898,9 +982,10 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
 	// state: its bundle asked for a bucket that did not exist. Idempotent and cheap.
 	//
 	// **A failure here is the send's failure** (novox/hq issue 249). It was said and the push stood,
-	// because the declarations were already away; they are sent after this now, and a module whose
-	// state does not exist is a module that fails its start, so nothing is sent and the caller tries
-	// again rather than reporting the rollout done.
+	// because the declarations were already away; they are sent after this now — all but the bus's
+	// own machine, sent before it (deliver) — and a module whose state does not exist is a module
+	// that fails its start, so they are not sent and the caller tries again rather than reporting the
+	// rollout done.
 	buckets, err := open.inventory.DeclaredBuckets(ctx)
 	if err != nil {
 		return fmt.Errorf("the modules' state could not be read, so no bucket was asserted: %w", err)
@@ -909,8 +994,8 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
 		return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
 	}
 	// Every membership is tried, and the first failure named once.
-	issued, failed := 0, 0
-	var first error
+	issued := 0
+	refused := map[string]error{}
 	for _, s := range sent {
 		node := s.node
 		for _, d := range records.Assigned[node] {
@@ -931,10 +1016,9 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
 				return err
 			}
 			if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil {
-				if first == nil {
-					first = err
+				if refused[node] == nil {
+					refused[node] = fmt.Errorf("%s: %w", d.Module, err)
 				}
-				failed++
 				continue
 			}
 			issued++
@@ -943,10 +1027,10 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
 	if issued > 0 {
 		fmt.Printf("  issued %d membership(s)\n", issued)
 	}
-	if failed > 0 {
-		// Returned, never passed over (novox/hq issue 249): the declarations that need these are not
-		// sent, and the rollout that asked is tried again rather than waiting for a push.
-		return fmt.Errorf("%d membership(s) could not be issued; the first: %w", failed, first)
+	if len(refused) > 0 {
+		// Returned, never passed over (novox/hq issue 249): the declarations of the machines they are
+		// for are not sent, and the rollout that asked is tried again rather than waiting for a push.
+		return &grantsRefused{nodes: refused}
 	}
 	return nil
 }

cmd/mesh-controller/sendable.go

@@ -31,6 +31,10 @@ type sendable struct {
 	// the same composition as its received files, and every machine's private-network address.
 	Received map[string]map[string][]catalogue.Contribution
 	Mesh     []string
+	// BusUsers is the bus's user list this declaration carries, empty for every machine but the one
+	// holding the bus; not sent apart from the file it is in. Its digest is recorded once sent, so
+	// whether that machine must go first is read from the list alone (novox/hq issue 249).
+	BusUsers string
 	// LeftOut is every module of the machine's set left out of this declaration because a stored
 	// setting cannot compose with its definition (novox/hq ADR 0163, rule 6), sorted. The host
 	// keeps that module's held things and touches none of its containers; a machine is told

cmd/mesh-controller/upgrades.go

@@ -65,13 +65,17 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
 	if plans, err := inv.OpenPlans(ctx); err != nil {
 		return notNow(err)
 	} else if id := rolledOutByAPlan(plans, u.Module); id != "" {
-		fmt.Printf("%s moved to %s; %s rolls it out to %s\n", u.Module, shortCommit(u.Commit), id, readableList(on))
+		// Said with its remedy: a plan that ends without sending it — failed, or closed by hand — leaves
+		// these machines behind, which `status` lists and `push --behind` sends (novox/hq issue 249).
+		fmt.Printf("%s moved to %s; %s rolls it out to %s — if that plan ends without sending it, "+
+			"`status` lists them as behind and `push --behind` sends it\n",
+			u.Module, shortCommit(u.Commit), id, readableList(on))
 		return nil
 	}
 	if decision.Together {
 		fmt.Printf("%s moved to %s; sending %s together\n",
 			u.Module, shortCommit(u.Commit), readableList(on))
-		return sendTo(ctx, f.open, on)
+		return askAgainOnGrants(sendTo(ctx, f.open, on))
 	}
 	// One at a time, and stopping at the first that fails.
 	//
@@ -82,18 +86,28 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
 		u.Module, shortCommit(u.Commit), readableList(on))
 	for _, node := range on {
 		if err := sendTo(ctx, f.open, []string{node}); err != nil {
-			return fmt.Errorf("%s did not take %s, so the machines after it were left alone: %w",
-				node, u.Module, err)
+			return askAgainOnGrants(fmt.Errorf("%s did not take %s, so the machines after it were left alone: %w",
+				node, u.Module, err))
 		}
 	}
 	return nil
 }
 
+// askAgainOnGrants marks a send that stopped at its grants as one to ask again (novox/hq issue 249):
+// an announcement handled by a send whose memberships could not be issued is held and redelivered,
+// rather than taken as handled with the machines left on the old version.
+func askAgainOnGrants(err error) error {
+	if err != nil && errors.Is(err, errGrants) && !errors.Is(err, link.ErrTryAgain) {
+		return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
+	}
+	return err
+}
+
 // rolledOutByAPlan is the open plan that will send a module's machines its new build — one holding
 // the module that has not finished sending it — or empty when none will (novox/hq issue 249).
 func rolledOutByAPlan(plans []inventory.Plan, module string) string {
 	for _, p := range plans {
-		if s, holds := p.Modules[module]; p.Open() && holds && (s == nil || s.SentAt == nil) {
+		if s, holds := p.Modules[module]; p.Open() && holds && (s == nil || (s.SentAt == nil && s.State != "failed")) {
 			return p.ID
 		}
 	}

internal/inventory/migrations/0058-a-newer-plan-supersedes-an-older.sql

@@ -12,3 +12,11 @@
 -- is superseded by the next plan of its repository, whichever branch — nothing is lost by it, since
 -- what it had not built is folded into the plan that supersedes it.
 alter table release_plan add column branch text not null default '';
+
+-- And the bus's user list the machine holding the bus was last sent, as a digest (novox/hq issue
+-- 249). A module's new grants are refused by the bus until its user list says them, so that machine
+-- is sent first whenever the list it would be sent differs from the one it was. Read from its whole
+-- declaration, every pending change on it — a recorded upgrade the operator chose not to roll out —
+-- went with every send anywhere. A digest and never the list (ADR 0043: the list is composed on each
+-- push, never kept). Empty for a machine never sent one, which reads as behind once.
+alter table node add column sent_bus_users text not null default '';

internal/inventory/nodes.go

@@ -921,6 +921,26 @@ func (i *Inventory) RecordSent(ctx context.Context, node, digest string) error {
 	return err
 }
 
+// RecordSentBusUsers keeps a digest of the bus's user list a machine was just sent, by its name
+// (novox/hq issue 249): whether the machine holding the bus must go first is whether this differs
+// from the list composed now.
+func (i *Inventory) RecordSentBusUsers(ctx context.Context, name, digest string) error {
+	_, err := i.store.Pool().Exec(ctx,
+		`update node set sent_bus_users = $2 where name = $1`, name, digest)
+	return err
+}
+
+// SentBusUsers is the digest of the bus's user list a machine was last sent, empty for none.
+func (i *Inventory) SentBusUsers(ctx context.Context, name string) (string, error) {
+	var sent string
+	err := i.store.Pool().QueryRow(ctx,
+		`select sent_bus_users from node where name = $1`, name).Scan(&sent)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return "", nil
+	}
+	return sent, err
+}
+
 // Outstanding is the digest of the declaration a machine was last sent, by its name, and empty
 // for one that has never been sent anything.
 //

internal/inventory/sent_bus_users_test.go

@@ -0,0 +1,25 @@
+package inventory
+
+import "testing"
+
+// novox/hq issue 249: the digest of the user list a machine was last sent is kept, by its name, and
+// is empty for a machine never sent one.
+func TestTheUserListAMachineWasSentIsKept(t *testing.T) {
+	inv := ForTest(t)
+	ctx := t.Context()
+	if _, err := inv.AddNode(ctx, "anchor"); err != nil {
+		t.Fatal(err)
+	}
+	if sent, err := inv.SentBusUsers(ctx, "anchor"); err != nil || sent != "" {
+		t.Fatalf("a machine never sent a list has %q: %v", sent, err)
+	}
+	if err := inv.RecordSentBusUsers(ctx, "anchor", "abc"); err != nil {
+		t.Fatal(err)
+	}
+	if sent, err := inv.SentBusUsers(ctx, "anchor"); err != nil || sent != "abc" {
+		t.Fatalf("the list sent was not kept: %q %v", sent, err)
+	}
+	if sent, err := inv.SentBusUsers(ctx, "nobody"); err != nil || sent != "" {
+		t.Fatalf("a machine the mesh does not know: %q %v", sent, err)
+	}
+}