mesh-grants: compose the user list from the running and the trunk manifest, and write a grant only once the loaded list carries it, so L3's parts never use a subject the bus has not loaded (issue 774)

modules/mesh-grants/grants_test.go

@@ -481,8 +481,8 @@ func TestGrants(t *testing.T) {
 				[]string{"a.mod-x", "a.nats", "b.mod-y", "controller", "node.a", "node.b"}, "enrolment tokens"},
 			{"a manifest missing is a gap and its user is left out", func(b *fakeBus) { delete(b.states[grants.ReadModules], "mod-y") },
 				[]string{"a.mod-x", "a.nats", "controller", "node.a", "node.b"}, "manifest of mod-y"},
-			{"a want's built manifest is composed, not the trunk's", func(b *fakeBus) {
-				built := strings.Replace(xManifest, `"history":3}]`, `"history":3},{"name":"newer"}]`, 1)
+			{"a want's built manifest is composed, not the trunk's; the list's records carry both (issue 774)", func(b *fakeBus) {
+				built := strings.Replace(xManifest, `{"name":"kept","history":3}`, `{"name":"newer"}`, 1)
 				digest := delivery.ManifestDigest([]byte(built))
 				b.put(grants.ReadManifests, digest, delivery.Manifest{V: delivery.Version, Module: "mod-x", JSON: built})
 				fp := "src2:" + strings.Repeat("cd", 32)
@@ -510,8 +510,22 @@ func TestGrants(t *testing.T) {
 				}
 				if strings.HasPrefix(row.name, "a want's") {
 					x := users[slices.IndexFunc(users, func(u grants.User) bool { return u.User == "a.mod-x" })]
-					if !slices.Contains(x.Publish, "$KV.mod-x_newer.>") {
-						t.Errorf("a.mod-x is not granted the state its wanted build declares: %v", x.Publish)
+					if !slices.Contains(x.Publish, "$KV.mod-x_newer.>") || slices.Contains(x.Publish, "$KV.mod-x_kept.>") {
+						t.Errorf("a.mod-x is not granted the state its wanted build declares alone: %v", x.Publish)
+					}
+					// The list composes what runs and what the trunk names: a subject only the trunk's manifest names is in
+					// it before the want moves to the trunk's build (issue 774).
+					listRecs, _ := grants.ListRecords(in)
+					listed, err := grants.ComposeUsers(listRecs)
+					if err != nil {
+						t.Fatal(err)
+					}
+					if got := names(listed); !slices.Equal(got, row.users) {
+						t.Errorf("the list's users = %v, want %v", got, row.users)
+					}
+					lx := listed[slices.IndexFunc(listed, func(u grants.User) bool { return u.User == "a.mod-x" })]
+					if !slices.Contains(lx.Publish, "$KV.mod-x_newer.>") || !slices.Contains(lx.Publish, "$KV.mod-x_kept.>") {
+						t.Errorf("the list does not grant a.mod-x both its running and its trunk manifest's state: %v", lx.Publish)
 					}
 				}
 			})
@@ -1921,7 +1935,7 @@ func TestGrants(t *testing.T) {
 	// 774/L3a (novox/hq ADR 0547 §5, ADR 0561 §2): the part writes grants.<module>.<fp> in shadow, once each and only for a
 	// module whose accounts the vault holds a hash for, says grant-written for each, deletes one only when a module has no
 	// account with a hash and every read was made, and the tool says every difference from the controller's.
-	t.Run("grants: written in shadow once each, only for hashed accounts, deleted only when read whole, compared with the controller's", func(t *testing.T) {
+	t.Run("grants: written in shadow once each, only for hashed accounts, deleted only when read whole, compared with the controller's; after take-over once the loaded list carries them", func(t *testing.T) {
 		b := fixture()
 		b.loaded = shadowOf(t, b)
 		p := partOn(b)
@@ -1979,6 +1993,45 @@ func TestGrants(t *testing.T) {
 		if res := p.Compose(); !slices.Equal(res.Grants.Deleted, []string{key("mod-y")}) || len(held()) != 2 {
 			t.Errorf("a module with no hashed account kept its grant: %+v, held %v", res.Grants, held())
 		}
+
+		// Issue 774: after take-over a new grant is written only once the list the bus loaded carries it. mod-x moves to a
+		// new trunk fingerprint while the list is held (users of the loaded list have no hash): it waits, said per key.
+		busKey, err := seal.GenerateKeyPair()
+		if err != nil {
+			t.Fatal(err)
+		}
+		loads := func(source, digest string) {
+			b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
+				UserList: &delivery.UserListWritten{Source: source, Revision: 8, Wanted: digest, Digest: digest, Users: 6, Key: busKey.PublicKey}})
+		}
+		loads("a.nats.src2", "sha256:aa")
+		p.Objects = &fakeObjects{}
+		sw, _ := json.Marshal(grants.Switch{V: 1, State: grants.TookOver, Acts: []grants.Act{}})
+		if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil {
+			t.Fatal(err)
+		}
+		fp2 := "src2:" + strings.Repeat("cd", 32)
+		key2, _ := delivery.PartGrantKey("mod-x", fp2)
+		b.put(grants.ReadModules, "mod-x", delivery.Module{V: delivery.Version, FP2: fp2, Manifest: delivery.ManifestDigest([]byte(xManifest)), Repository: "novox/x", Commit: "c"})
+		res = p.Compose()
+		if _, written := b.own[grants.StateGrants][key2]; written || res.Sealed == nil || !strings.Contains(res.Sealed.Why, "held") ||
+			len(res.Grants.Waits) != 1 || !strings.HasPrefix(res.Grants.Waits[0], key2+": waits for the bus to load") {
+			t.Fatalf("a held list let a new grant through: sealed %+v, grants %+v", res.Sealed, res.Grants)
+		}
+		// Every user hashed: the list is sealed, and the grants still wait while the bus has another list loaded.
+		for _, u := range shadowOf(t, b) {
+			hash(u.User)
+		}
+		res = p.Compose()
+		sealed, _, _ := p.Sealed()
+		if _, written := b.own[grants.StateGrants][key2]; written || sealed.Digest == "" || len(res.Grants.Waits) != 2 {
+			t.Fatalf("a grant was written before the bus loaded the list carrying it: sealed %+v, grants %+v", sealed, res.Grants)
+		}
+		// The bus loads the sealed list: both written, nothing waits.
+		loads(delivery.GrantsUserListSource, sealed.Digest)
+		if res := p.Compose(); len(res.Grants.Waits) != 0 || !slices.Equal(res.Grants.Written, []string{key2, key("mod-y")}) {
+			t.Errorf("once the bus loaded the list carrying them: %+v", res.Grants)
+		}
 	})
 
 	// Issue 1126 (C5): a module whose source commit has not moved since a build of the old form, with a done key of the

modules/mesh-grants/internal/grants/grants.go

@@ -34,6 +34,34 @@ type GrantsWritten struct {
 	// `<module>: <why>`: the assigner's sixth rule wants none of it (MissingGrants). Never passed over in silence: once
 	// the part's grants are the ones read, it is the condition mesh-grants.grant-missing.
 	Missing []string `json:"missing,omitempty"`
+	// Waits is each grant composed and not written, `<key>: <why>`: after take-over a grant is new or changed only once the
+	// list the bus loaded carries it (issue 774, GrantWaits). Read with grants_grants (last_pass).
+	Waits []string `json:"waits,omitempty"`
+}
+
+// GrantWaits says why a grant composed now, new or changed, is not written yet, or "" when it may be (issue 774). The
+// assigner moves a module's want to the fingerprint once its grant is there, so a grant is the word that the bus's user list
+// in force already carries the module's subjects at that fingerprint. After take-over that is so when the list the bus
+// node says it loaded is the grants part's own and is the list composed now (composed: its digest), which ListRecords
+// composes from what runs and what the trunk names; a list held, not yet sealed, or not yet loaded is not it. In shadow
+// nobody reads the part's grants and the controller's list is in force, so nothing waits. Pure.
+func GrantWaits(tookOver bool, loaded *delivery.UserListWritten, composed string) string {
+	switch {
+	case !tookOver:
+		return ""
+	case composed == "":
+		return "waits: the user list could not be composed, so no list carries it"
+	case loaded == nil:
+		return "waits for the bus node to say which user list it loaded"
+	case loaded.Source != delivery.GrantsUserListSource:
+		return "waits for the bus to load the grants part's list " + short(composed) + ": it loaded " + loaded.Source + "'s"
+	case loaded.Error != "":
+		return "waits for the bus to load the list " + short(composed) + ": its node could not write it: " + loaded.Error
+	case loaded.Digest != composed:
+		return "waits for the bus to load the list " + short(composed) + ", which carries it: it has " + short(loaded.Digest) +
+			" loaded (a list held, or sealed and not yet loaded)"
+	}
+	return ""
 }
 
 // GrantMissingCondition is the key of the warning raised while a placed module has no grant (issue 1126).
@@ -175,7 +203,7 @@ func grantsReadFailed(in Inputs) string {
 
 // writeGrants writes each grant whose key is absent or differs and says grant-written for each, and deletes the key of a
 // module that has none now.
-func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records) {
+func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records, composed string) {
 	if why := grantsReadFailed(in); why != "" {
 		res.Grants = &GrantsWritten{Kept: []string{"no grant is written or deleted in a pass that could not read " + why}}
 		return
@@ -192,6 +220,7 @@ func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records) {
 	done := &GrantsWritten{Wanted: len(want), Missing: MissingGrants(in, want, drop, skip)}
 	res.Grants = done
 	p.sayMissing(done.Missing)
+	waits := GrantWaits(p.SwitchState() == TookOver, in.Loaded, composed)
 	for _, key := range sortedKeys(want) {
 		g := want[key]
 		data, err := delivery.EncodeControllerGrant(g)
@@ -207,6 +236,10 @@ func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records) {
 				continue
 			}
 		}
+		if waits != "" {
+			done.Waits = append(done.Waits, key+": "+waits)
+			continue
+		}
 		if _, err := p.Store.Put(StateGrants, key, data); err != nil {
 			res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateGrants, key, err))
 			continue

modules/mesh-grants/internal/grants/part.go

@@ -233,22 +233,26 @@ func (p *Part) Compose() Result {
 	for _, g := range in.Refused {
 		res.Refused = append(res.Refused, g.Input+": "+g.Why)
 	}
-	recs, left := Records(in)
+	// recs say what runs (memberships, consumers); listRecs what the user list and the grants carry: what runs and what
+	// the trunk names, so the list grants a build's subjects before the want moves to it (issue 774).
+	recs, _ := Records(in)
+	listRecs, left := ListRecords(in)
 	holdings := HoldingsOf(in)
-	res.RootNotFree = WithRootFree(&recs, p.Call)
+	res.RootNotFree = WithRootFree(&listRecs, p.Call)
+	recs.RootFree = listRecs.RootFree
 	st := UserListState{V: 1, Mode: p.SwitchState(), LeftOut: left, Loaded: in.Loaded, BusNode: in.BusNode,
 		Gaps: append(append([]Gap(nil), in.Gaps...), MembershipGaps...), At: p.now()}
 	if st.LeftOut == nil {
 		st.LeftOut = []Gap{}
 	}
-	users, err := ComposeUsers(recs)
+	users, err := ComposeUsers(listRecs)
 	if err != nil {
 		res.Errors = append(res.Errors, "the shadow list could not be composed: "+err.Error())
 	}
 	users = WithHashes(users, in.Hashes)
 	st.Digest, st.Users = Digest(users), len(users)
 	loaded, err := LoadedUsers(p.Call)
-	st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(recs)))
+	st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(listRecs)))
 	if err != nil {
 		st.Compare.Unread = err.Error()
 	}
@@ -273,14 +277,20 @@ func (p *Part) Compose() Result {
 	}
 	// The accounts the vault mints from, said in the state only it reads (ADR 0561 §2).
 	if p.Store != nil {
-		p.writeAccounts(&res, in, recs)
+		p.writeAccounts(&res, in, listRecs)
 	}
-	// The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5).
+	list, n, leftOut, err := ComposeList(listRecs, in.Hashes)
+	composed := ""
+	if err == nil {
+		composed = delivery.UserListDigest(list)
+	}
+	// The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5); after take-over
+	// each new one only once the bus loaded a list that carries it (issue 774).
 	if p.Store != nil {
-		p.writeGrants(&res, in, recs)
+		p.writeGrants(&res, in, listRecs, composed)
 	}
 	// The list itself, sealed to the bus node and named in `user-list.sealed` (ADR 0547 clause 7).
-	if list, n, leftOut, err := ComposeList(recs, in.Hashes); err != nil {
+	if err != nil {
 		res.Errors = append(res.Errors, "the user list could not be composed whole: "+err.Error())
 	} else {
 		// The users the bus loaded: what a first list after take-over must keep where the part has not yet written one.
@@ -293,11 +303,11 @@ func (p *Part) Compose() Result {
 	}
 	ms := Memberships(recs)
 	res.Memberships = len(ms)
-	composed := make([]membership.Membership, 0, len(ms))
+	members := make([]membership.Membership, 0, len(ms))
 	for _, m := range ms {
-		composed = append(composed, m.Membership)
+		members = append(members, m.Membership)
 	}
-	seated := holdings.ByMemberships(composed...)
+	seated := holdings.ByMemberships(members...)
 	for _, key := range sortedKeys(ms) {
 		m, now := publish(p.Publish, ms[key], holdings, seated)
 		switch {

modules/mesh-grants/internal/grants/records.go

@@ -2,6 +2,7 @@ package grants
 
 import (
 	"fmt"
+	"reflect"
 	"slices"
 	"strings"
 
@@ -14,7 +15,15 @@ import (
 // BusRecords, read the same way from Inputs instead of the controller's store). Pure. An assigned module whose manifest
 // the indexer does not hold is left out and named, where the controller refuses the whole composition: the shadow
 // composes what it can and says what it could not.
-func Records(in Inputs) (buscompose.Records, []Gap) {
+func Records(in Inputs) (buscompose.Records, []Gap) { return records(in, false) }
+
+// ListRecords is Records for the user list and the grants (issue 774): a module whose wanted build's manifest is not the
+// trunk's is declared with what both grant (unionDeclared), so the list carries the trunk manifest's subjects before the
+// assigner moves the want to it, and a grant at the trunk's fingerprint waits for nothing the want's move would bring.
+// Memberships and consumers stay with Records: they say what runs. Pure.
+func ListRecords(in Inputs) (buscompose.Records, []Gap) { return records(in, true) }
+
+func records(in Inputs, union bool) (buscompose.Records, []Gap) {
 	var gaps []Gap
 	seats := map[string]catalogue.SeatDeclaration{}
 	declarers := map[string]string{}
@@ -59,21 +68,29 @@ func Records(in Inputs) (buscompose.Records, []Gap) {
 				gaps = append(gaps, Gap{Input: module + " on " + node, Why: "left out of the shadow list: no manifest to derive what it may say from"})
 				continue
 			}
-			d := declaredFor(m, seats, declarers)
+			declare := func(m catalogue.Manifest) buscompose.Declared {
+				d := declaredFor(m, seats, declarers)
+				if len(in.Placed) > 0 {
+					d.Holds = heldWherePlaced(d.Holds, in.Placed, node, m.Module)
+				}
+				for _, sr := range catalogue.ReadsGranted(m, here, node) {
+					d.KeyedReads = append(d.KeyedReads, buscompose.KeyedRead{Read: sr.Read, Key: sr.Key})
+				}
+				d.Hears, d.HearsAStream, d.Consumes = hearsOf(m, in.Manifests, node, values, d.Consumes)
+				return d
+			}
+			d := declare(m)
+			trunk, onTrunk := in.Manifests[module]
 			// What a node composing a consumer reads of it is the trunk manifest's served keys (frombus reads the indexer's
 			// manifest of another node's module), and what runs may still be another build: both are granted (issue 1126).
-			if trunk, ok := in.Manifests[module]; ok {
+			if onTrunk {
 				d.Served = slices.Compact(slices.Sorted(slices.Values(append(slices.Clone(d.Served), catalogue.ServedSettings(trunk)...))))
 			}
-			if len(in.Placed) > 0 {
-				d.Holds = heldWherePlaced(d.Holds, in.Placed, node, m.Module)
+			if union && onTrunk && !reflect.DeepEqual(m, trunk) {
+				d = unionDeclared(d, declare(trunk))
 			}
-			for _, sr := range catalogue.ReadsGranted(m, here, node) {
-				d.KeyedReads = append(d.KeyedReads, buscompose.KeyedRead{Read: sr.Read, Key: sr.Key})
-			}
-			d.Hears, d.HearsAStream, d.Consumes = hearsOf(m, in.Manifests, node, values, d.Consumes)
 			out.Assigned[node] = append(out.Assigned[node], d)
-			if m.Instances == catalogue.InstancesInterchangeable {
+			if m.Instances == catalogue.InstancesInterchangeable || union && onTrunk && trunk.Instances == catalogue.InstancesInterchangeable {
 				out.Interchangeable[m.Module] = true
 			}
 		}
@@ -356,3 +373,48 @@ func sameSeat(a, b string) bool {
 	sb, okb := catalogue.SeatNamed(b)
 	return oka && okb && sa.Name == sb.Name
 }
+
+// unionDeclared is what either declaration grants (issue 774): every list of b's entries a lacks appended to a's, a
+// process of both merged by its name, a flag that adds a grant set where either sets it, and an account kept where
+// either has one (NoAccount only where both say none). a's names and RunsAs stand. Pure.
+func unionDeclared(a, b buscompose.Declared) buscompose.Declared {
+	out := a
+	unionFields(reflect.ValueOf(&out).Elem(), reflect.ValueOf(b))
+	out.NoAccount = a.NoAccount && b.NoAccount
+	procs := slices.Clone(a.Processes)
+	for _, bp := range b.Processes {
+		i := slices.IndexFunc(procs, func(ap buscompose.ProcessPrincipal) bool { return ap.Process == bp.Process })
+		if i < 0 {
+			procs = append(procs, bp)
+			continue
+		}
+		merged := procs[i]
+		unionFields(reflect.ValueOf(&merged).Elem(), reflect.ValueOf(bp))
+		procs[i] = merged
+	}
+	out.Processes = procs
+	return out
+}
+
+// unionFields appends to each slice field of dst the elements of src's it lacks, and sets each bool field src sets.
+func unionFields(dst, src reflect.Value) {
+	for i := 0; i < dst.NumField(); i++ {
+		f, g := dst.Field(i), src.Field(i)
+		switch f.Kind() {
+		case reflect.Slice:
+			merged := reflect.AppendSlice(reflect.MakeSlice(f.Type(), 0, f.Len()+g.Len()), f)
+			for j := 0; j < g.Len(); j++ {
+				held := false
+				for k := 0; k < merged.Len() && !held; k++ {
+					held = reflect.DeepEqual(merged.Index(k).Interface(), g.Index(j).Interface())
+				}
+				if !held {
+					merged = reflect.Append(merged, g.Index(j))
+				}
+			}
+			f.Set(merged)
+		case reflect.Bool:
+			f.SetBool(f.Bool() || g.Bool())
+		}
+	}
+}