From c1b28cc4fe086721389f3c625e04811df1f8758d Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 18:07:26 +0100 Subject: [PATCH] Keep each sealed list's permissions one key per user, so no value outgrows the bucket (issue 774) The sealed-grants value held every user of the last two lists in one value; at 61 users it was refused with "message size exceeds maximum" (JetStream 10054, 16:37 and 16:55 on 2026-10-30), so the permissions of the list in force were never kept and a grant judged against them waited. Quick fix: keep only the newest list in the one value (halves it; it still grows with the mesh). Thorough fix, chosen because the value grows with every placed module and the kind was met twice: one key per user and list, sealed-grants.., and an index sealed-lists naming the kept lists and their users, written only once every user's key is there; the third list's keys are taken away, and the old one-value key read as a fallback and removed. mesh-sdk's kvlimit guard (7b02f63) refuses an oversized value before the put through process.State.Put, which the part's own store (cmd/mesh-grants ownStore.Put) writes through once pinned. --- modules/mesh-grants/grants_test.go | 64 ++++++++ modules/mesh-grants/internal/grants/grants.go | 145 +++++++++++++++--- 2 files changed, 187 insertions(+), 22 deletions(-) diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index d10eed9..c08d497 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -67,6 +67,8 @@ type fakeBus struct { // consumerCreates each consumer the part asked to create, `.`. consumers map[string][]map[string]any consumerCreates []string + // maxValue, when set, refuses an own value larger than it as the bus does (JetStream 10054). + maxValue int } // MatchConsumer is the bus's update of a held durable consumer to the configuration given. @@ -260,6 +262,9 @@ func (s ownStore) Put(state, key string, value []byte) (uint64, error) { if b.refuseTo[state] { return 0, errors.New("Permissions Violation for Publish to $KV.mesh-grants_" + state) } + if b.maxValue > 0 && len(value) > b.maxValue { + return 0, errors.New("nats: API error: code=400 err_code=10054 description=message size exceeds maximum allowed") + } b.writes = append(b.writes, state) if b.own[state] == nil { b.own[state] = map[string][]byte{} @@ -2565,6 +2570,65 @@ func TestGrants(t *testing.T) { t.Errorf("changes the vault's rotation names were held: %+v", said) } }) + + // Sweep 2026-10-30 item 5: the permissions of the sealed lists were one value holding every user of two lists, which + // outgrew the bucket's largest value, so they were never kept. One key per user: no value grows with the list. + t.Run("sealed grants: one key per user, so no value grows with the list, and only the last two lists kept", func(t *testing.T) { + b := fixture() + for _, u := range shadowOf(t, b) { + b.put(grants.ReadAccounts, u.User, map[string]any{"user": u.User, "hash": "$2a$10$first", "generation": 1, "fingerprint": "sha256:1"}) + } + largest := 0 + for _, u := range shadowOf(t, b) { + raw, _ := json.Marshal(u) + largest = max(largest, len(raw)) + } + b.maxValue = largest + 256 // one user's permissions fit, every user's of a list do not + busKey, err := seal.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, + UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}}) + p := partOn(b) + p.Objects = &fakeObjects{} + var digests []string + for gen := 1; gen <= 3; gen++ { + b.put(grants.ReadAccounts, "a.mod-x", map[string]any{"user": "a.mod-x", "hash": fmt.Sprintf("$2a$10$gen%d", gen), "generation": gen, "fingerprint": fmt.Sprintf("sha256:%d", gen)}) + res := p.Compose() + for _, e := range res.Errors { + if strings.Contains(e, "10054") { + t.Fatalf("a sealed-grants write exceeded the bucket's largest value: %s", e) + } + } + g, _, _ := p.Sealed() + digests = append(digests, g.Digest) + } + var idx grants.SealedLists + raw, _, found, _ := b.ownGet(grants.StateUserList, grants.KeySealedLists) + if !found || json.Unmarshal(raw, &idx) != nil || len(idx.Lists) != 2 || idx.Lists[0].Digest != digests[2] || idx.Lists[1].Digest != digests[1] { + t.Fatalf("the index of kept lists = %s, want the last two of %v", raw, digests) + } + for i, d := range digests { + _, _, kept, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(d, "a.mod-x")) + if kept != (i > 0) { + t.Errorf("list %d's a.mod-x key kept = %v", i, kept) + } + } + for _, u := range idx.Lists[0].Users { + if _, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(digests[2], u)); !ok { + t.Errorf("the index names %s, whose key is not there", u) + } + } + // The loaded list's permissions are read back from the keys when the bus module's tools cannot say them. + b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK, + UserList: &delivery.UserListWritten{Source: delivery.GrantsUserListSource, Follows: delivery.FollowGrants, Revision: 8, + Digest: digests[2], Users: 6, Key: busKey.PublicKey}}) + b.refuseUsers = true + if res := p.Compose(); slices.ContainsFunc(res.Grants.Waits, func(w string) bool { return strings.Contains(w, "unknown") }) { + t.Errorf("the kept permissions of the loaded list were not read: %v", res.Grants.Waits) + } + }) } type fakeAsker struct { diff --git a/modules/mesh-grants/internal/grants/grants.go b/modules/mesh-grants/internal/grants/grants.go index 92df498..d0b582b 100644 --- a/modules/mesh-grants/internal/grants/grants.go +++ b/modules/mesh-grants/internal/grants/grants.go @@ -82,12 +82,26 @@ func GrantWaits(tookOver bool, accounts []string, composed, loaded map[string]Us return "waits for the bus to load a list carrying what it adds: " + strings.Join(adds, ", ") } -// KeySealedGrants is the key of the state user-list holding, per list the part sealed (the last two, by digest), each -// user's permissions as sealed, never a hash: what tells which grants the list the bus loaded carries, without the bus -// module's tools (issue 774). +// **The permissions of each list the part sealed, one key per user** (issue 774, sweep 2026-10-30 item 5): the last two +// lists, by digest, each user's permissions as sealed, never a hash: what tells which grants the list the bus loaded +// carries, without the bus module's tools. One value holding every user of two lists outgrew the bucket's largest value +// (JetStream 10054, "message size exceeds maximum", 16:37 and 16:55 on 2026-10-30), so a list's permissions were never +// kept and a grant judged against them waited. Each user's permissions are a key of their own, +// `sealed-grants..`, which grows with that user alone; the key `sealed-lists` names the lists kept and +// their users, and is written only once every user's key of its newest list is: it never names a list whose keys are not +// all there. + +// KeySealedGrants is the key that held every user of the last two lists in one value, before this release: read when the +// index names no list, and taken away once the index is written. const KeySealedGrants = "sealed-grants" -// SealedGrants is the value of KeySealedGrants. +// KeySealedLists is the key of the state user-list naming the sealed lists whose permissions are kept, newest first. +const KeySealedLists = "sealed-lists" + +// sealedKept is how many sealed lists keep their permissions: the one in force and the one before. +const sealedKept = 2 + +// SealedGrants is the value KeySealedGrants held. type SealedGrants struct { Lists []SealedList `json:"lists"` } @@ -98,7 +112,24 @@ type SealedList struct { Users []User `json:"users"` } -// keepSealedGrants records the permissions of the list sealed now, beside the one before. +// SealedLists is the value of KeySealedLists: the lists kept, newest first, each by its digest and its users' names. +type SealedLists struct { + Lists []SealedNames `json:"lists"` +} + +// SealedNames is one kept list: its digest and its users, sorted. +type SealedNames struct { + Digest string `json:"digest"` + Users []string `json:"users"` +} + +// SealedUserKey is the key of one user's permissions in the sealed list of digest. +func SealedUserKey(digest, user string) string { + return KeySealedGrants + "." + strings.TrimPrefix(digest, "sha256:") + "." + user +} + +// keepSealedGrants records the permissions of the list sealed now, one key per user, beside the list before; the lists +// older than those two lose their keys. func (p *Part) keepSealedGrants(res *Result, digest string, list []byte, composed []User) { in := listUsers(list) var users []User @@ -108,20 +139,98 @@ func (p *Part) keepSealedGrants(res *Result, digest string, list []byte, compose users = append(users, u) } } - var was SealedGrants - if raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants); err == nil && found { - _ = json.Unmarshal(raw, &was) + failed := false + names := make([]string, 0, len(users)) + for _, u := range users { + names = append(names, u.User) + data, _ := json.Marshal(u) + if _, err := p.Store.Put(StateUserList, SealedUserKey(digest, u.User), data); err != nil { + failed = true + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, SealedUserKey(digest, u.User), err)) + } } - next := SealedGrants{Lists: []SealedList{{Digest: digest, Users: users}}} + if failed { + // The index keeps naming the lists whose keys are all there; this list's permissions are not kept. + return + } + sort.Strings(names) + was := p.sealedLists() + next := SealedLists{Lists: []SealedNames{{Digest: digest, Users: names}}} + var gone []SealedNames for _, l := range was.Lists { - if l.Digest != digest && len(next.Lists) < 2 { + switch { + case l.Digest == digest: + case len(next.Lists) < sealedKept: next.Lists = append(next.Lists, l) + default: + gone = append(gone, l) } } data, _ := json.Marshal(next) - if _, err := p.Store.Put(StateUserList, KeySealedGrants, data); err != nil { - res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeySealedGrants, err)) + if _, err := p.Store.Put(StateUserList, KeySealedLists, data); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeySealedLists, err)) + return } + for _, l := range gone { + for _, user := range l.Users { + if err := p.Store.Delete(StateUserList, SealedUserKey(l.Digest, user)); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be taken away: %v", StateUserList, SealedUserKey(l.Digest, user), err)) + } + } + } + if _, _, found, err := p.Store.Get(StateUserList, KeySealedGrants); err == nil && found { + if err := p.Store.Delete(StateUserList, KeySealedGrants); err != nil { + res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be taken away: %v", StateUserList, KeySealedGrants, err)) + } + } +} + +// sealedLists is the index of kept lists; none when it cannot be read. +func (p *Part) sealedLists() SealedLists { + var out SealedLists + if raw, _, found, err := p.Store.Get(StateUserList, KeySealedLists); err == nil && found { + _ = json.Unmarshal(raw, &out) + } + return out +} + +// sealedGrantsOf is the permissions of each user of the sealed list of digest, as the part kept them, and whether it kept +// every one: from the index and each user's key, else from the one value written before this release. +func (p *Part) sealedGrantsOf(digest string) (map[string]User, bool) { + for _, l := range p.sealedLists().Lists { + if l.Digest != digest { + continue + } + out := map[string]User{} + for _, user := range l.Users { + raw, _, found, err := p.Store.Get(StateUserList, SealedUserKey(digest, user)) + if err != nil || !found { + return nil, false + } + var u User + if json.Unmarshal(raw, &u) != nil { + return nil, false + } + out[u.User] = u + } + return out, true + } + raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants) + if err == nil && found { + var sg SealedGrants + if json.Unmarshal(raw, &sg) == nil { + for _, sl := range sg.Lists { + if sl.Digest == digest { + out := map[string]User{} + for _, u := range sl.Users { + out[u.User] = u + } + return out, true + } + } + } + } + return nil, false } // loadedContents is the list the bus loaded, by user: as the bus module's nats_users read it, else as the part sealed the @@ -144,16 +253,8 @@ func (p *Part) loadedContents(in Inputs, read []User, readErr error) (map[string case l.Source != delivery.GrantsUserListSource || l.Follows != delivery.FollowGrants || l.Error != "": return nil, "the bus loaded no list of the grants part's (source " + l.Source + ", follows " + l.Follows + "), and nats_users: " + readErr.Error() } - raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants) - if err == nil && found { - var sg SealedGrants - if json.Unmarshal(raw, &sg) == nil { - for _, sl := range sg.Lists { - if sl.Digest == l.Digest { - return byUser(sl.Users), "" - } - } - } + if kept, ok := p.sealedGrantsOf(l.Digest); ok { + return kept, "" } return nil, "the part keeps no permissions of the loaded list " + short(l.Digest) + " (sealed before it kept them), and nats_users: " + readErr.Error() }