diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index c08d497..c61ccf4 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -1547,7 +1547,7 @@ func TestGrants(t *testing.T) { fp := partOn(b) fp.Objects = p.Objects fp.Asker, fp.Say = fa, func(c asks.Condition) error { - if c.Key != grants.GrantMissingCondition { // its own row below + if c.Key != grants.GrantMissingCondition && c.Key != grants.RefusedCondition { // their own rows fresh = append(fresh, c) } return nil @@ -2445,7 +2445,7 @@ func TestGrants(t *testing.T) { // the composed configuration only by hand (grants_consumer_match), never while the controller holds the lease. var said []asks.Condition p.Say = func(c asks.Condition) error { - if c.Key != grants.GrantMissingCondition { // its own row + if c.Key != grants.GrantMissingCondition && c.Key != grants.RefusedCondition { // their own rows said = append(said, c) } return nil @@ -2629,6 +2629,109 @@ func TestGrants(t *testing.T) { t.Errorf("the kept permissions of the loaded list were not read: %v", res.Grants.Waits) } }) + + // Sweep 2026-10-30 item 5, a kind met twice that day (the engine's acts, then mesh-host_health): one principal the + // composer refuses as a second writer froze the whole list. Now it keeps its grants of the list in force, the rest is + // written, and the refusal is raised by name; one the list in force does not carry stays out, said too. + t.Run("a refused principal keeps its grants of the list in force; the rest of the list is written", func(t *testing.T) { + b := fixture() + const ctl = `{"module":"mesh-controller","version":"1","own-secrets":{"broker":"/var/lib/c/broker"}}` + refusedCtl := strings.Replace(ctl, `}}`, `},"state":[{"name":"declared-config"}]}`, 1) + b.manifestOf("mesh-controller", ctl) + b.assign("b", "mesh-controller") + hash := func(user, h string, gen int) { + b.put(grants.ReadAccounts, user, map[string]any{"user": user, "hash": h, "generation": gen, "fingerprint": fmt.Sprintf("sha256:%d", gen)}) + } + for _, u := range shadowOf(t, b) { + hash(u.User, "$2a$10$first", 1) + } + hash("a.mesh-controller", "$2a$10$first", 1) + busKey, err := seal.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, + UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}}) + p := partOn(b) + objs := &fakeObjects{} + p.Objects = objs + entryOf := func(g delivery.GrantsUserList, user string) string { + t.Helper() + list, err := delivery.OpenUserList(objs.objects[g.Object], g.Digest, func(bx seal.Box) ([]byte, error) { + out, err := seal.Open(bx, busKey.PrivateKey) + return []byte(out), err + }) + if err != nil { + t.Fatalf("the sealed list does not open: %v", err) + } + at := strings.Index(string(list), fmt.Sprintf("{ user: %q,", user)) + if at < 0 { + return "" + } + rest := string(list)[at:] + return rest[:strings.Index(rest, "} }\n")+4] + } + var said []asks.Condition + p.Say = func(c asks.Condition) error { + if c.Key == grants.RefusedCondition { + said = append(said, c) + } + return nil + } + sw, _ := json.Marshal(grants.Switch{V: 1, State: grants.TookOver, Acts: []grants.Act{}}) + if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil { + t.Fatal(err) + } + p.Compose() + inForce, _, _ := p.Sealed() + if inForce.Digest == "" { + t.Fatal("no list written after take-over") + } + entryBefore := entryOf(inForce, "b.mesh-controller") + ctlKey := grants.SealedUserKey(inForce.Digest, "b.mesh-controller") + before, _, ok, _ := b.ownGet(grants.StateUserList, ctlKey) + if !ok { + t.Fatal("the list in force keeps no grants of b.mesh-controller") + } + // b.mesh-controller now declares the controller's mirror as its own state: refused as a second writer. a.mod-x's hash + // changes in the same pass, and a.mesh-controller, refused the same way, is placed new. + b.manifestOf("mesh-controller", refusedCtl) + b.assign("a", "mesh-controller") + hash("a.mod-x", "$2a$10$second", 2) + res := p.Compose() + now, _, _ := p.Sealed() + if now.Digest == inForce.Digest || slices.ContainsFunc(res.Errors, func(e string) bool { return strings.Contains(e, "composed whole") }) { + t.Fatalf("a refused principal froze the list: sealed %+v, errors %v", res.Sealed, res.Errors) + } + after, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(now.Digest, "b.mesh-controller")) + if !ok || string(after) != string(before) { + t.Errorf("b.mesh-controller did not keep its grants of the list in force: %s, was %s", after, before) + } + // The kept user is written as the composer writes one: the bus reads the same entry it loaded. + if got := entryOf(now, "b.mesh-controller"); entryBefore == "" || got != entryBefore { + t.Errorf("the kept entry is written otherwise than the list in force holds it:\n%s\nwas\n%s", got, entryBefore) + } + if entryOf(now, "a.mesh-controller") != "" { + t.Error("a refused principal the list in force did not carry is in the sealed list") + } + if _, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(now.Digest, "a.mesh-controller")); ok { + t.Error("a refused principal the list in force did not carry is in the list") + } + if len(res.PrincipalsRefused) != 2 || len(said) == 0 || said[len(said)-1].State != asks.StateRaised || + !strings.Contains(said[len(said)-1].Summary, "b.mesh-controller (keeps its grants of the list in force)") || + !strings.Contains(said[len(said)-1].Summary, "a.mesh-controller (left out of the list") { + t.Fatalf("the refusals were not said by name: %v, said %+v", res.PrincipalsRefused, said) + } + // Composing again changes nothing and says nothing new; once the manifest composes, the warning is cleared. + n := len(said) + if p.Compose(); len(said) != n { + t.Errorf("an unchanged refusal was said again: %+v", said[n:]) + } + b.manifestOf("mesh-controller", ctl) + if res := p.Compose(); len(res.PrincipalsRefused) != 0 || said[len(said)-1].State != asks.StateCleared { + t.Errorf("the refusal was not cleared once it composes: %v, said %+v", res.PrincipalsRefused, said) + } + }) } type fakeAsker struct { diff --git a/modules/mesh-grants/internal/grants/part.go b/modules/mesh-grants/internal/grants/part.go index 1e83f5c..7b4563a 100644 --- a/modules/mesh-grants/internal/grants/part.go +++ b/modules/mesh-grants/internal/grants/part.go @@ -131,9 +131,12 @@ type Result struct { // Grants is what the pass wrote of its grants (grants.go). Grants *GrantsWritten `json:"grants,omitempty"` // Withheld is what the pass wrote of the hearings it withholds (withheld.go). - Withheld *WithheldWritten `json:"withheld,omitempty"` - Errors []string `json:"errors,omitempty"` - PublishedOrAsserted string `json:"published_or_asserted"` + Withheld *WithheldWritten `json:"withheld,omitempty"` + // PrincipalsRefused is each principal whose composed grants the composer refused, `: `: it keeps its grants + // of the list in force, or stays out where it holds none (refused.go); the condition mesh-grants.principal-refused says it. + PrincipalsRefused []string `json:"principals_refused,omitempty"` + Errors []string `json:"errors,omitempty"` + PublishedOrAsserted string `json:"published_or_asserted"` } // NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships. @@ -186,6 +189,9 @@ type Part struct { // differSaid is the account consumers' differences last said as a condition, once differKnown (consumers.go). differSaid string differKnown bool + // refusedSaid is the refused principals last said as a condition, once refusedKnown (refused.go). + refusedSaid string + refusedKnown bool // missingSaid is the modules without a grant last said as a condition, once missingKnown (grants.go). missingSaid string missingKnown bool @@ -251,11 +257,17 @@ func (p *Part) Compose() Result { if st.LeftOut == nil { st.LeftOut = []Gap{} } - users, err := ComposeUsers(listRecs) + users, refused, err := ComposeEach(listRecs) if err != nil { res.Errors = append(res.Errors, "the shadow list could not be composed: "+err.Error()) } - users = WithHashes(users, in.Hashes) + // A refused principal keeps its grants of the list in force, said once per change (refused.go). + kept, _ := p.keptOf(refused) + for _, user := range sortedKeys(refused) { + res.PrincipalsRefused = append(res.PrincipalsRefused, user+": "+refused[user]) + } + p.sayPrincipalsRefused(refused, kept) + users = WithHashes(withKept(users, kept), in.Hashes) st.Digest, st.Users = Digest(users), len(users) loaded, loadErr := LoadedUsers(p.Call) st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(listRecs))) @@ -289,7 +301,7 @@ func (p *Part) Compose() Result { if p.Store != nil { p.writeWithheld(&res, in, recs) } - list, n, leftOut, err := ComposeList(listRecs, in.Hashes) + list, n, leftOut, dropped, err := ComposeList(listRecs, in.Hashes, refused, kept) // The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5); after take-over // one that adds a permission only once the bus loaded a list that carries it (issue 774). if p.Store != nil { @@ -306,7 +318,7 @@ func (p *Part) Compose() Result { bus = names(loaded) lost = holdings.LastHolders(holdings.ByUsers(loaded), holdings.ByUsers(users)) } - p.writeSealed(&res, list, n, leftOut, in, bus, lost, users) + p.writeSealed(&res, list, n, leftOut, dropped, in, bus, lost, users) } ms := Memberships(recs) res.Memberships = len(ms) diff --git a/modules/mesh-grants/internal/grants/refused.go b/modules/mesh-grants/internal/grants/refused.go new file mode 100644 index 0000000..6d2e8b7 --- /dev/null +++ b/modules/mesh-grants/internal/grants/refused.go @@ -0,0 +1,157 @@ +package grants + +import ( + "fmt" + "slices" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// **One refused principal never freezes the list** (issue 774, sweep 2026-10-30; met twice that day: the engine's acts, +// then mesh-host_health). The composer refuses a principal's grants when they would make it a second writer of a state +// (buscompose.CheckWriters): that refusal used to end the whole composition, so the list in force stood for every principal +// while each change since piled up, and the next list that composed moved 52 users at once. Now each principal is composed +// on its own: one refused keeps the grants it holds in the list in force (the part's own record of that list, sealed +// grants), the rest of the list is composed and written as ever, and the refusal is raised as the warning +// mesh-grants.principal-refused, naming each principal and why, until it composes again. A refused principal the list in +// force does not carry stays out of the list: it never had those grants, and is refused them now. + +// RefusedCondition is the key of the warning raised while a principal's grants are refused. +const RefusedCondition = "mesh-grants.principal-refused" + +// ComposeEach is the shadow list composed principal by principal: every user whose grants compose, and each refused one +// by user with why. err is only for principals that cannot be derived at all (two users of one name). Pure. +func ComposeEach(r buscompose.Records) ([]User, map[string]string, error) { + principals, err := buscompose.Users(r) + if err != nil { + return nil, nil, err + } + out := make([]User, 0, len(principals)) + refused := map[string]string{} + for _, p := range principals { + perms, err := buscompose.PermissionsFor(p) + if err != nil { + refused[p.Username()] = err.Error() + continue + } + out = append(out, User{User: p.Username(), Publish: set(perms.Publish), Subscribe: set(perms.Subscribe), + AllowResponses: perms.AllowResponses, PublishDeny: set(perms.PublishDeny)}) + } + sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User }) + return out, refused, nil +} + +// keptOf is, for each refused user, its grants in the list in force as the part sealed them; the users it holds none of +// (not in the list in force, or that list's grants not kept) are the second answer, sorted. +func (p *Part) keptOf(refused map[string]string) (map[string]User, []string) { + if len(refused) == 0 { + return nil, nil + } + var inForce map[string]User + if g, _, had := p.Sealed(); had && g.Digest != "" && p.Store != nil { + inForce, _ = p.sealedGrantsOf(g.Digest) + } + kept := map[string]User{} + var none []string + for _, user := range sortedKeys(refused) { + if u, ok := inForce[user]; ok { + kept[user] = u + continue + } + none = append(none, user) + } + return kept, none +} + +// withKept is users with each kept user in its place, sorted. +func withKept(users []User, kept map[string]User) []User { + out := slices.Clone(users) + for _, u := range kept { + out = append(out, u) + } + sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User }) + return out +} + +// keptEntry is a user of the accounts block as buscompose.ComposeAccounts writes one, from grants kept rather than +// composed: the same lines, so the bus reads it as it read the list in force. +func keptEntry(u User, hash string) string { + var b strings.Builder + fmt.Fprintf(&b, " { user: %q, password: %q, permissions: {\n", u.User, hash) + if len(u.PublishDeny) > 0 { + fmt.Fprintf(&b, " publish: { allow: [%s], deny: [%s] }\n", quotedList(u.Publish), quotedList(u.PublishDeny)) + } else { + fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quotedList(u.Publish)) + } + fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quotedList(u.Subscribe)) + if u.AllowResponses { + fmt.Fprintf(&b, " allow_responses: { max: 1, ttl: \"%dm\" }\n", int(buscompose.ResponseTTL/time.Minute)) + } + b.WriteString(" } }\n") + return b.String() +} + +func quotedList(in []string) string { + q := make([]string, len(in)) + for i, s := range in { + q[i] = fmt.Sprintf("%q", s) + } + return strings.Join(q, ", ") +} + +// withEntries puts entries at the end of the accounts block's users, before it closes. +func withEntries(list string, entries []string) (string, error) { + if len(entries) == 0 { + return list, nil + } + start := strings.Index(list, " users = [\n") + if start < 0 { + return "", fmt.Errorf("the composed list holds no users block to keep a refused user's grants in") + } + end := strings.Index(list[start:], "\n ]\n") + if end < 0 { + return "", fmt.Errorf("the composed list's users block does not close") + } + at := start + end + 1 + return list[:at] + strings.Join(entries, "") + list[at:], nil +} + +// sayPrincipalsRefused raises RefusedCondition while a principal's grants are refused, naming each with why and what +// became of it, and clears it once none is; said once per set, again when it changes, and at the first pass after a start. +func (p *Part) sayPrincipalsRefused(refused map[string]string, kept map[string]User) { + if p.Say == nil { + return + } + var lines []string + for _, user := range sortedKeys(refused) { + became := "left out of the list: the list in force does not carry it" + if _, ok := kept[user]; ok { + became = "keeps its grants of the list in force" + } + lines = append(lines, user+" ("+became+"): "+refused[user]) + } + why := strings.Join(lines, "; ") + p.holdMu.Lock() + defer p.holdMu.Unlock() + if p.refusedKnown && why == p.refusedSaid { + return + } + c := asks.Condition{Key: RefusedCondition, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}} + if why == "" { + c.State = asks.StateCleared + } else { + c.Severity, c.State = asks.SeverityWarning, asks.StateRaised + c.Summary = fmt.Sprintf("%d principal(s) refused their composed grants: %s", len(refused), why) + c.Headline, c.Needs = "A principal's bus grants are refused", "a fix of the manifest or the writers table" + c.Explanation = asks.NeedsYou + "a fix of what makes it a second writer. The rest of the user list is written as ever; " + + "each refused principal keeps the grants the list in force gives it, or stays out where it holds none: " + why + c.Resolved = "every principal's grants compose again" + } + if p.Say(c) == nil { + p.refusedSaid, p.refusedKnown = why, true + } +} diff --git a/modules/mesh-grants/internal/grants/sealed.go b/modules/mesh-grants/internal/grants/sealed.go index 097e09d..b355b9d 100644 --- a/modules/mesh-grants/internal/grants/sealed.go +++ b/modules/mesh-grants/internal/grants/sealed.go @@ -31,24 +31,41 @@ type Objects interface { DeleteObject(name string) error } -// ComposeList is the bus's user list as the file holds it, with each user's hash from the vault, and the users left out -// for want of one. Pure. -func ComposeList(r buscompose.Records, hashes map[string]HeldHash) ([]byte, int, []string, error) { +// ComposeList is the bus's user list as the file holds it, with each user's hash from the vault; the users left out +// for want of one; and the refused principals (refused, by user) the list carries with their grants kept from the list +// in force (kept), or leaves out where kept holds none (dropped, sorted). Pure. +func ComposeList(r buscompose.Records, hashes map[string]HeldHash, refused map[string]string, kept map[string]User) (list []byte, users int, leftOut, dropped []string, err error) { principals, err := buscompose.Users(r) if err != nil { - return nil, 0, nil, err + return nil, 0, nil, nil, err } held := map[string]string{} for user, h := range hashes { held[user] = h.Hash } - filled, missing := buscompose.WithPasswords(principals, held) - list, err := buscompose.ComposeAccounts(filled) + composing := slices.DeleteFunc(slices.Clone(principals), func(p buscompose.Principal) bool { _, no := refused[p.Username()]; return no }) + filled, missing := buscompose.WithPasswords(composing, held) + text, err := buscompose.ComposeAccounts(filled) if err != nil { - return nil, 0, nil, err + return nil, 0, nil, nil, err + } + var entries []string + for _, user := range sortedKeys(refused) { + u, ok := kept[user] + switch { + case !ok: + dropped = append(dropped, user) + case held[user] == "": + missing = append(missing, user) + default: + entries = append(entries, keptEntry(u, held[user])) + } + } + if text, err = withEntries(text, entries); err != nil { + return nil, 0, nil, nil, err } slices.Sort(missing) - return []byte(list), len(filled), missing, nil + return []byte(text), len(filled) + len(entries), missing, dropped, nil } // Sealed is the part's key `user-list.sealed` as written, and whether there is one. @@ -74,7 +91,7 @@ func followOf(state string) string { // writeSealed puts the list sealed to the bus node and writes the key, each only when what it says changed. It says in // res what it wrote, or why it wrote nothing. -func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string, in Inputs, bus []string, lost []string, composed []User) { +func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut, dropped []string, in Inputs, bus []string, lost []string, composed []User) { cur, _, had := p.Sealed() next := cur next.Follow = followOf(p.SwitchState()) @@ -93,10 +110,10 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string why = "no node runs the bus's module, so the list is sealed to nobody" case key == "": why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)" - case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, bus, lost, digest) != "": + case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, digest) != "": // After take-over the list in force is never replaced by one composed past a failed read, or one that drops a user // of it for want of a hash: that user's module would be locked out of the bus (review of the switch). - why = p.holdsBack(in, leftOut, bus, lost, digest) + why = p.holdsBack(in, leftOut, dropped, bus, lost, digest) held = why p.keepHashHold(res, digest) case cur.Digest != digest || cur.SealedTo != delivery.SealingKeyID(key) || cur.Node != in.BusNode: @@ -356,7 +373,7 @@ func listUsers(list []byte) []string { // loaded list not read holds it too, since that cannot be ruled out). The list in force is the one the part wrote, by the users // it kept of it; where it kept none (a list written before this release), the users the bus loaded (bus, nil when // they could not be read). Empty when it may. -func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string, digest string) string { +func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest string) string { p.hashHold = nil if f := in.ReadFailures(); len(f) > 0 { return "held: the list in force is kept while a read of this pass failed (" + strings.Join(f, "; ") + ")" @@ -387,6 +404,17 @@ func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string, digest string) if len(unhashed) > 0 { return "held: the list in force is kept, since " + strings.Join(unhashed, ", ") + " would be left out for want of a hash in the vault" } + // A refused principal the list in force carries and whose grants there the part did not keep: leaving it out would lock + // it out of the bus (refused.go), so the list in force stands until its grants compose or are kept. + var unkept []string + for _, u := range dropped { + if slices.Contains(had, u) { + unkept = append(unkept, u) + } + } + if len(unkept) > 0 { + return "held: the list in force is kept, since " + strings.Join(unkept, ", ") + " would be left out: its grants are refused and the part kept none of the list in force" + } // More than one loaded user's hash changing at once, with no rotation by the vault naming each (ADR 0561 §7): a new // writer of hashes must not lock several holders out in one list, as nine seats were cut on 2026-10-27 at 09:21. changed, err := p.changedHashes(in.Hashes) diff --git a/modules/mesh-grants/internal/grants/userlist.go b/modules/mesh-grants/internal/grants/userlist.go index 601236c..b83a6a5 100644 --- a/modules/mesh-grants/internal/grants/userlist.go +++ b/modules/mesh-grants/internal/grants/userlist.go @@ -116,23 +116,18 @@ func CredentialsOf(shadow, loaded []User, hashes map[string]HeldHash) Credential return c } -// ComposeUsers is the shadow list: every user the composer derives from the records, with its grants. Pure. +// ComposeUsers is the shadow list: every user the composer derives from the records, with its grants; an error when any +// principal's grants are refused (ComposeEach composes past one). Pure. func ComposeUsers(r buscompose.Records) ([]User, error) { - principals, err := buscompose.Users(r) + users, refused, err := ComposeEach(r) if err != nil { return nil, err } - out := make([]User, 0, len(principals)) - for _, p := range principals { - perms, err := buscompose.PermissionsFor(p) - if err != nil { - return nil, fmt.Errorf("%s: %w", p.Username(), err) - } - out = append(out, User{User: p.Username(), Publish: set(perms.Publish), Subscribe: set(perms.Subscribe), - AllowResponses: perms.AllowResponses, PublishDeny: set(perms.PublishDeny)}) + if len(refused) > 0 { + first := sortedKeys(refused)[0] + return nil, fmt.Errorf("%s: %s", first, refused[first]) } - sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User }) - return out, nil + return users, nil } func set(in []string) []string {