Issue grants before code, and the bus's machine first (hq issue 249)

A module's new state reached every machine before the permissions to use
it, which came only with a later push. Memberships and buckets now go
before declarations, the machine holding mesh-broker goes first when its
user list must change, and a grant that fails sends nothing and is an
error so the rollout is retried.

cmd/mesh-controller/delivery_order_test.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// recordingDelivery is a delivery that writes down what was done, in order, and fails where told.
+type recordingDelivery struct {
+	did        []string
+	grantErr   error
+	declareErr error
+}
+
+func (r *recordingDelivery) grant(_ context.Context, sending []readyNode) error {
+	for _, s := range sending {
+		r.did = append(r.did, "grant "+s.node)
+	}
+	return r.grantErr
+}
+
+func (r *recordingDelivery) declare(_ context.Context, s readyNode, _ []byte) (string, error) {
+	if r.declareErr != nil {
+		return "", r.declareErr
+	}
+	r.did = append(r.did, "declare "+s.node)
+	return "digest-" + s.node, nil
+}
+
+func ready(names ...string) []readyNode {
+	var out []readyNode
+	for _, n := range names {
+		out = append(out, readyNode{node: n, declared: sendable{Resources: []map[string]any{{"id": "x"}}}})
+	}
+	return out
+}
+
+// novox/hq issue 249: the grants that come with a module's new declarations are issued before any
+// machine is sent the code that uses them.
+func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) {
+	d := &recordingDelivery{}
+	digests, err := deliver(t.Context(), d, ready("anchor", "laptop"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []string{"grant anchor", "grant laptop", "declare anchor", "declare laptop"}
+	if !reflect.DeepEqual(d.did, want) {
+		t.Fatalf("delivered in the order %v, wanted %v", d.did, want)
+	}
+	if digests["anchor"] != "digest-anchor" || digests["laptop"] != "digest-laptop" {
+		t.Fatalf("the digests sent were not answered: %v", digests)
+	}
+}
+
+// A grant that cannot be issued sends nothing and is an error — never "until the next push".
+func TestAGrantThatFailsSendsNothingAndSaysSo(t *testing.T) {
+	d := &recordingDelivery{grantErr: errors.New("the bus refused the membership")}
+	_, err := deliver(t.Context(), d, ready("anchor", "laptop"))
+	if err == nil || !strings.Contains(err.Error(), "the bus refused the membership") {
+		t.Fatalf("a failed grant was not said as the send's failure: %v", err)
+	}
+	for _, did := range d.did {
+		if strings.HasPrefix(did, "declare") {
+			t.Fatalf("code was sent after its grant failed: %v", d.did)
+		}
+	}
+	// Nothing to send is nothing granted either.
+	none := &recordingDelivery{grantErr: errors.New("never asked")}
+	if _, err := deliver(t.Context(), none, nil); err != nil || len(none.did) != 0 {
+		t.Fatalf("an empty send granted or failed: %v %v", none.did, err)
+	}
+}
+
+// The machine holding the bus goes first: its declaration carries the user list the new grants are
+// checked against. Among the machines it is moved to the front; not among them it is added only
+// when it is behind.
+func TestTheMachineHoldingTheBusIsSentFirst(t *testing.T) {
+	for _, c := range []struct {
+		what   string
+		names  []string
+		holder string
+		behind bool
+		want   []string
+	}{
+		{"among them", []string{"ace", "g14", "novox"}, "novox", false, []string{"novox", "ace", "g14"}},
+		{"not among them, behind", []string{"ace", "g14"}, "novox", true, []string{"novox", "ace", "g14"}},
+		{"not among them, current", []string{"ace", "g14"}, "novox", false, []string{"ace", "g14"}},
+		{"nothing holds the bus", []string{"ace", "g14"}, "", true, []string{"ace", "g14"}},
+		{"only it", []string{"novox"}, "novox", false, []string{"novox"}},
+	} {
+		if got := brokerFirst(c.names, c.holder, c.behind); !reflect.DeepEqual(got, c.want) {
+			t.Errorf("%s: sent in the order %v, wanted %v", c.what, got, c.want)
+		}
+	}
+}