diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 694a03e..81dfbe9 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -481,8 +481,8 @@ func TestGrants(t *testing.T) { []string{"a.mod-x", "a.nats", "b.mod-y", "controller", "node.a", "node.b"}, "enrolment tokens"}, {"a manifest missing is a gap and its user is left out", func(b *fakeBus) { delete(b.states[grants.ReadModules], "mod-y") }, []string{"a.mod-x", "a.nats", "controller", "node.a", "node.b"}, "manifest of mod-y"}, - {"a want's built manifest is composed, not the trunk's", func(b *fakeBus) { - built := strings.Replace(xManifest, `"history":3}]`, `"history":3},{"name":"newer"}]`, 1) + {"a want's built manifest is composed, not the trunk's; the list's records carry both (issue 774)", func(b *fakeBus) { + built := strings.Replace(xManifest, `{"name":"kept","history":3}`, `{"name":"newer"}`, 1) digest := delivery.ManifestDigest([]byte(built)) b.put(grants.ReadManifests, digest, delivery.Manifest{V: delivery.Version, Module: "mod-x", JSON: built}) fp := "src2:" + strings.Repeat("cd", 32) @@ -510,8 +510,22 @@ func TestGrants(t *testing.T) { } if strings.HasPrefix(row.name, "a want's") { x := users[slices.IndexFunc(users, func(u grants.User) bool { return u.User == "a.mod-x" })] - if !slices.Contains(x.Publish, "$KV.mod-x_newer.>") { - t.Errorf("a.mod-x is not granted the state its wanted build declares: %v", x.Publish) + if !slices.Contains(x.Publish, "$KV.mod-x_newer.>") || slices.Contains(x.Publish, "$KV.mod-x_kept.>") { + t.Errorf("a.mod-x is not granted the state its wanted build declares alone: %v", x.Publish) + } + // The list composes what runs and what the trunk names: a subject only the trunk's manifest names is in + // it before the want moves to the trunk's build (issue 774). + listRecs, _ := grants.ListRecords(in) + listed, err := grants.ComposeUsers(listRecs) + if err != nil { + t.Fatal(err) + } + if got := names(listed); !slices.Equal(got, row.users) { + t.Errorf("the list's users = %v, want %v", got, row.users) + } + lx := listed[slices.IndexFunc(listed, func(u grants.User) bool { return u.User == "a.mod-x" })] + if !slices.Contains(lx.Publish, "$KV.mod-x_newer.>") || !slices.Contains(lx.Publish, "$KV.mod-x_kept.>") { + t.Errorf("the list does not grant a.mod-x both its running and its trunk manifest's state: %v", lx.Publish) } } }) @@ -1921,7 +1935,7 @@ func TestGrants(t *testing.T) { // 774/L3a (novox/hq ADR 0547 §5, ADR 0561 §2): the part writes grants.. in shadow, once each and only for a // module whose accounts the vault holds a hash for, says grant-written for each, deletes one only when a module has no // account with a hash and every read was made, and the tool says every difference from the controller's. - t.Run("grants: written in shadow once each, only for hashed accounts, deleted only when read whole, compared with the controller's", func(t *testing.T) { + t.Run("grants: written in shadow once each, only for hashed accounts, deleted only when read whole, compared with the controller's; after take-over once the loaded list carries them", func(t *testing.T) { b := fixture() b.loaded = shadowOf(t, b) p := partOn(b) @@ -1979,6 +1993,45 @@ func TestGrants(t *testing.T) { if res := p.Compose(); !slices.Equal(res.Grants.Deleted, []string{key("mod-y")}) || len(held()) != 2 { t.Errorf("a module with no hashed account kept its grant: %+v, held %v", res.Grants, held()) } + + // Issue 774: after take-over a new grant is written only once the list the bus loaded carries it. mod-x moves to a + // new trunk fingerprint while the list is held (users of the loaded list have no hash): it waits, said per key. + busKey, err := seal.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + loads := func(source, digest string) { + b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, + UserList: &delivery.UserListWritten{Source: source, Revision: 8, Wanted: digest, Digest: digest, Users: 6, Key: busKey.PublicKey}}) + } + loads("a.nats.src2", "sha256:aa") + p.Objects = &fakeObjects{} + sw, _ := json.Marshal(grants.Switch{V: 1, State: grants.TookOver, Acts: []grants.Act{}}) + if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil { + t.Fatal(err) + } + fp2 := "src2:" + strings.Repeat("cd", 32) + key2, _ := delivery.PartGrantKey("mod-x", fp2) + b.put(grants.ReadModules, "mod-x", delivery.Module{V: delivery.Version, FP2: fp2, Manifest: delivery.ManifestDigest([]byte(xManifest)), Repository: "novox/x", Commit: "c"}) + res = p.Compose() + if _, written := b.own[grants.StateGrants][key2]; written || res.Sealed == nil || !strings.Contains(res.Sealed.Why, "held") || + len(res.Grants.Waits) != 1 || !strings.HasPrefix(res.Grants.Waits[0], key2+": waits for the bus to load") { + t.Fatalf("a held list let a new grant through: sealed %+v, grants %+v", res.Sealed, res.Grants) + } + // Every user hashed: the list is sealed, and the grants still wait while the bus has another list loaded. + for _, u := range shadowOf(t, b) { + hash(u.User) + } + res = p.Compose() + sealed, _, _ := p.Sealed() + if _, written := b.own[grants.StateGrants][key2]; written || sealed.Digest == "" || len(res.Grants.Waits) != 2 { + t.Fatalf("a grant was written before the bus loaded the list carrying it: sealed %+v, grants %+v", sealed, res.Grants) + } + // The bus loads the sealed list: both written, nothing waits. + loads(delivery.GrantsUserListSource, sealed.Digest) + if res := p.Compose(); len(res.Grants.Waits) != 0 || !slices.Equal(res.Grants.Written, []string{key2, key("mod-y")}) { + t.Errorf("once the bus loaded the list carrying them: %+v", res.Grants) + } }) // Issue 1126 (C5): a module whose source commit has not moved since a build of the old form, with a done key of the diff --git a/modules/mesh-grants/internal/grants/grants.go b/modules/mesh-grants/internal/grants/grants.go index 17c9e93..18c6fc0 100644 --- a/modules/mesh-grants/internal/grants/grants.go +++ b/modules/mesh-grants/internal/grants/grants.go @@ -34,6 +34,34 @@ type GrantsWritten struct { // `: `: the assigner's sixth rule wants none of it (MissingGrants). Never passed over in silence: once // the part's grants are the ones read, it is the condition mesh-grants.grant-missing. Missing []string `json:"missing,omitempty"` + // Waits is each grant composed and not written, `: `: after take-over a grant is new or changed only once the + // list the bus loaded carries it (issue 774, GrantWaits). Read with grants_grants (last_pass). + Waits []string `json:"waits,omitempty"` +} + +// GrantWaits says why a grant composed now, new or changed, is not written yet, or "" when it may be (issue 774). The +// assigner moves a module's want to the fingerprint once its grant is there, so a grant is the word that the bus's user list +// in force already carries the module's subjects at that fingerprint. After take-over that is so when the list the bus +// node says it loaded is the grants part's own and is the list composed now (composed: its digest), which ListRecords +// composes from what runs and what the trunk names; a list held, not yet sealed, or not yet loaded is not it. In shadow +// nobody reads the part's grants and the controller's list is in force, so nothing waits. Pure. +func GrantWaits(tookOver bool, loaded *delivery.UserListWritten, composed string) string { + switch { + case !tookOver: + return "" + case composed == "": + return "waits: the user list could not be composed, so no list carries it" + case loaded == nil: + return "waits for the bus node to say which user list it loaded" + case loaded.Source != delivery.GrantsUserListSource: + return "waits for the bus to load the grants part's list " + short(composed) + ": it loaded " + loaded.Source + "'s" + case loaded.Error != "": + return "waits for the bus to load the list " + short(composed) + ": its node could not write it: " + loaded.Error + case loaded.Digest != composed: + return "waits for the bus to load the list " + short(composed) + ", which carries it: it has " + short(loaded.Digest) + + " loaded (a list held, or sealed and not yet loaded)" + } + return "" } // GrantMissingCondition is the key of the warning raised while a placed module has no grant (issue 1126). @@ -175,7 +203,7 @@ func grantsReadFailed(in Inputs) string { // writeGrants writes each grant whose key is absent or differs and says grant-written for each, and deletes the key of a // module that has none now. -func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records) { +func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records, composed string) { if why := grantsReadFailed(in); why != "" { res.Grants = &GrantsWritten{Kept: []string{"no grant is written or deleted in a pass that could not read " + why}} return @@ -192,6 +220,7 @@ func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records) { done := &GrantsWritten{Wanted: len(want), Missing: MissingGrants(in, want, drop, skip)} res.Grants = done p.sayMissing(done.Missing) + waits := GrantWaits(p.SwitchState() == TookOver, in.Loaded, composed) for _, key := range sortedKeys(want) { g := want[key] data, err := delivery.EncodeControllerGrant(g) @@ -207,6 +236,10 @@ func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records) { continue } } + if waits != "" { + done.Waits = append(done.Waits, key+": "+waits) + continue + } if _, err := p.Store.Put(StateGrants, key, data); err != nil { res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateGrants, key, err)) continue diff --git a/modules/mesh-grants/internal/grants/part.go b/modules/mesh-grants/internal/grants/part.go index 11deb15..1e306be 100644 --- a/modules/mesh-grants/internal/grants/part.go +++ b/modules/mesh-grants/internal/grants/part.go @@ -233,22 +233,26 @@ func (p *Part) Compose() Result { for _, g := range in.Refused { res.Refused = append(res.Refused, g.Input+": "+g.Why) } - recs, left := Records(in) + // recs say what runs (memberships, consumers); listRecs what the user list and the grants carry: what runs and what + // the trunk names, so the list grants a build's subjects before the want moves to it (issue 774). + recs, _ := Records(in) + listRecs, left := ListRecords(in) holdings := HoldingsOf(in) - res.RootNotFree = WithRootFree(&recs, p.Call) + res.RootNotFree = WithRootFree(&listRecs, p.Call) + recs.RootFree = listRecs.RootFree st := UserListState{V: 1, Mode: p.SwitchState(), LeftOut: left, Loaded: in.Loaded, BusNode: in.BusNode, Gaps: append(append([]Gap(nil), in.Gaps...), MembershipGaps...), At: p.now()} if st.LeftOut == nil { st.LeftOut = []Gap{} } - users, err := ComposeUsers(recs) + users, err := ComposeUsers(listRecs) if err != nil { res.Errors = append(res.Errors, "the shadow list could not be composed: "+err.Error()) } users = WithHashes(users, in.Hashes) st.Digest, st.Users = Digest(users), len(users) loaded, err := LoadedUsers(p.Call) - st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(recs))) + st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(listRecs))) if err != nil { st.Compare.Unread = err.Error() } @@ -273,14 +277,20 @@ func (p *Part) Compose() Result { } // The accounts the vault mints from, said in the state only it reads (ADR 0561 §2). if p.Store != nil { - p.writeAccounts(&res, in, recs) + p.writeAccounts(&res, in, listRecs) } - // The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5). + list, n, leftOut, err := ComposeList(listRecs, in.Hashes) + composed := "" + if err == nil { + composed = delivery.UserListDigest(list) + } + // The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5); after take-over + // each new one only once the bus loaded a list that carries it (issue 774). if p.Store != nil { - p.writeGrants(&res, in, recs) + p.writeGrants(&res, in, listRecs, composed) } // The list itself, sealed to the bus node and named in `user-list.sealed` (ADR 0547 clause 7). - if list, n, leftOut, err := ComposeList(recs, in.Hashes); err != nil { + if err != nil { res.Errors = append(res.Errors, "the user list could not be composed whole: "+err.Error()) } else { // The users the bus loaded: what a first list after take-over must keep where the part has not yet written one. @@ -293,11 +303,11 @@ func (p *Part) Compose() Result { } ms := Memberships(recs) res.Memberships = len(ms) - composed := make([]membership.Membership, 0, len(ms)) + members := make([]membership.Membership, 0, len(ms)) for _, m := range ms { - composed = append(composed, m.Membership) + members = append(members, m.Membership) } - seated := holdings.ByMemberships(composed...) + seated := holdings.ByMemberships(members...) for _, key := range sortedKeys(ms) { m, now := publish(p.Publish, ms[key], holdings, seated) switch { diff --git a/modules/mesh-grants/internal/grants/records.go b/modules/mesh-grants/internal/grants/records.go index d07c7ae..a0e575f 100644 --- a/modules/mesh-grants/internal/grants/records.go +++ b/modules/mesh-grants/internal/grants/records.go @@ -2,6 +2,7 @@ package grants import ( "fmt" + "reflect" "slices" "strings" @@ -14,7 +15,15 @@ import ( // BusRecords, read the same way from Inputs instead of the controller's store). Pure. An assigned module whose manifest // the indexer does not hold is left out and named, where the controller refuses the whole composition: the shadow // composes what it can and says what it could not. -func Records(in Inputs) (buscompose.Records, []Gap) { +func Records(in Inputs) (buscompose.Records, []Gap) { return records(in, false) } + +// ListRecords is Records for the user list and the grants (issue 774): a module whose wanted build's manifest is not the +// trunk's is declared with what both grant (unionDeclared), so the list carries the trunk manifest's subjects before the +// assigner moves the want to it, and a grant at the trunk's fingerprint waits for nothing the want's move would bring. +// Memberships and consumers stay with Records: they say what runs. Pure. +func ListRecords(in Inputs) (buscompose.Records, []Gap) { return records(in, true) } + +func records(in Inputs, union bool) (buscompose.Records, []Gap) { var gaps []Gap seats := map[string]catalogue.SeatDeclaration{} declarers := map[string]string{} @@ -59,21 +68,29 @@ func Records(in Inputs) (buscompose.Records, []Gap) { gaps = append(gaps, Gap{Input: module + " on " + node, Why: "left out of the shadow list: no manifest to derive what it may say from"}) continue } - d := declaredFor(m, seats, declarers) + declare := func(m catalogue.Manifest) buscompose.Declared { + d := declaredFor(m, seats, declarers) + if len(in.Placed) > 0 { + d.Holds = heldWherePlaced(d.Holds, in.Placed, node, m.Module) + } + for _, sr := range catalogue.ReadsGranted(m, here, node) { + d.KeyedReads = append(d.KeyedReads, buscompose.KeyedRead{Read: sr.Read, Key: sr.Key}) + } + d.Hears, d.HearsAStream, d.Consumes = hearsOf(m, in.Manifests, node, values, d.Consumes) + return d + } + d := declare(m) + trunk, onTrunk := in.Manifests[module] // What a node composing a consumer reads of it is the trunk manifest's served keys (frombus reads the indexer's // manifest of another node's module), and what runs may still be another build: both are granted (issue 1126). - if trunk, ok := in.Manifests[module]; ok { + if onTrunk { d.Served = slices.Compact(slices.Sorted(slices.Values(append(slices.Clone(d.Served), catalogue.ServedSettings(trunk)...)))) } - if len(in.Placed) > 0 { - d.Holds = heldWherePlaced(d.Holds, in.Placed, node, m.Module) + if union && onTrunk && !reflect.DeepEqual(m, trunk) { + d = unionDeclared(d, declare(trunk)) } - for _, sr := range catalogue.ReadsGranted(m, here, node) { - d.KeyedReads = append(d.KeyedReads, buscompose.KeyedRead{Read: sr.Read, Key: sr.Key}) - } - d.Hears, d.HearsAStream, d.Consumes = hearsOf(m, in.Manifests, node, values, d.Consumes) out.Assigned[node] = append(out.Assigned[node], d) - if m.Instances == catalogue.InstancesInterchangeable { + if m.Instances == catalogue.InstancesInterchangeable || union && onTrunk && trunk.Instances == catalogue.InstancesInterchangeable { out.Interchangeable[m.Module] = true } } @@ -356,3 +373,48 @@ func sameSeat(a, b string) bool { sb, okb := catalogue.SeatNamed(b) return oka && okb && sa.Name == sb.Name } + +// unionDeclared is what either declaration grants (issue 774): every list of b's entries a lacks appended to a's, a +// process of both merged by its name, a flag that adds a grant set where either sets it, and an account kept where +// either has one (NoAccount only where both say none). a's names and RunsAs stand. Pure. +func unionDeclared(a, b buscompose.Declared) buscompose.Declared { + out := a + unionFields(reflect.ValueOf(&out).Elem(), reflect.ValueOf(b)) + out.NoAccount = a.NoAccount && b.NoAccount + procs := slices.Clone(a.Processes) + for _, bp := range b.Processes { + i := slices.IndexFunc(procs, func(ap buscompose.ProcessPrincipal) bool { return ap.Process == bp.Process }) + if i < 0 { + procs = append(procs, bp) + continue + } + merged := procs[i] + unionFields(reflect.ValueOf(&merged).Elem(), reflect.ValueOf(bp)) + procs[i] = merged + } + out.Processes = procs + return out +} + +// unionFields appends to each slice field of dst the elements of src's it lacks, and sets each bool field src sets. +func unionFields(dst, src reflect.Value) { + for i := 0; i < dst.NumField(); i++ { + f, g := dst.Field(i), src.Field(i) + switch f.Kind() { + case reflect.Slice: + merged := reflect.AppendSlice(reflect.MakeSlice(f.Type(), 0, f.Len()+g.Len()), f) + for j := 0; j < g.Len(); j++ { + held := false + for k := 0; k < merged.Len() && !held; k++ { + held = reflect.DeepEqual(merged.Index(k).Interface(), g.Index(j).Interface()) + } + if !held { + merged = reflect.Append(merged, g.Index(j)) + } + } + f.Set(merged) + case reflect.Bool: + f.SetBool(f.Bool() || g.Bool()) + } + } +}