Keep each sealed list's permissions one key per user, so no value outgrows the bucket (issue 774)

The sealed-grants value held every user of the last two lists in one value; at 61 users it was refused with "message size exceeds maximum" (JetStream 10054, 16:37 and 16:55 on 2026-10-30), so the permissions of the list in force were never kept and a grant judged against them waited. Quick fix: keep only the newest list in the one value (halves it; it still grows with the mesh). Thorough fix, chosen because the value grows with every placed module and the kind was met twice: one key per user and list, sealed-grants.<digest hex>.<user>, and an index sealed-lists naming the kept lists and their users, written only once every user's key is there; the third list's keys are taken away, and the old one-value key read as a fallback and removed. mesh-sdk's kvlimit guard (7b02f63) refuses an oversized value before the put through process.State.Put, which the part's own store (cmd/mesh-grants ownStore.Put) writes through once pinned.
2026-10-30 18:07:26 +01:00
parent 33973d5338
commit c1b28cc4fe
2 changed files with 187 additions and 22 deletions
@@ -67,6 +67,8 @@ type fakeBus struct {
 	// consumerCreates each consumer the part asked to create, `<stream>.<name>`.
 	consumers       map[string][]map[string]any
 	consumerCreates []string
+	// maxValue, when set, refuses an own value larger than it as the bus does (JetStream 10054).
+	maxValue int
 }

 // MatchConsumer is the bus's update of a held durable consumer to the configuration given.
@@ -260,6 +262,9 @@ func (s ownStore) Put(state, key string, value []byte) (uint64, error) {
 	if b.refuseTo[state] {
 		return 0, errors.New("Permissions Violation for Publish to $KV.mesh-grants_" + state)
 	}
+	if b.maxValue > 0 && len(value) > b.maxValue {
+		return 0, errors.New("nats: API error: code=400 err_code=10054 description=message size exceeds maximum allowed")
+	}
 	b.writes = append(b.writes, state)
 	if b.own[state] == nil {
 		b.own[state] = map[string][]byte{}
@@ -2565,6 +2570,65 @@ func TestGrants(t *testing.T) {
 			t.Errorf("changes the vault's rotation names were held: %+v", said)
 		}
 	})
+
+	// Sweep 2026-10-30 item 5: the permissions of the sealed lists were one value holding every user of two lists, which
+	// outgrew the bucket's largest value, so they were never kept. One key per user: no value grows with the list.
+	t.Run("sealed grants: one key per user, so no value grows with the list, and only the last two lists kept", func(t *testing.T) {
+		b := fixture()
+		for _, u := range shadowOf(t, b) {
+			b.put(grants.ReadAccounts, u.User, map[string]any{"user": u.User, "hash": "$2a$10$first", "generation": 1, "fingerprint": "sha256:1"})
+		}
+		largest := 0
+		for _, u := range shadowOf(t, b) {
+			raw, _ := json.Marshal(u)
+			largest = max(largest, len(raw))
+		}
+		b.maxValue = largest + 256 // one user's permissions fit, every user's of a list do not
+		busKey, err := seal.GenerateKeyPair()
+		if err != nil {
+			t.Fatal(err)
+		}
+		b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
+			UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}})
+		p := partOn(b)
+		p.Objects = &fakeObjects{}
+		var digests []string
+		for gen := 1; gen <= 3; gen++ {
+			b.put(grants.ReadAccounts, "a.mod-x", map[string]any{"user": "a.mod-x", "hash": fmt.Sprintf("$2a$10$gen%d", gen), "generation": gen, "fingerprint": fmt.Sprintf("sha256:%d", gen)})
+			res := p.Compose()
+			for _, e := range res.Errors {
+				if strings.Contains(e, "10054") {
+					t.Fatalf("a sealed-grants write exceeded the bucket's largest value: %s", e)
+				}
+			}
+			g, _, _ := p.Sealed()
+			digests = append(digests, g.Digest)
+		}
+		var idx grants.SealedLists
+		raw, _, found, _ := b.ownGet(grants.StateUserList, grants.KeySealedLists)
+		if !found || json.Unmarshal(raw, &idx) != nil || len(idx.Lists) != 2 || idx.Lists[0].Digest != digests[2] || idx.Lists[1].Digest != digests[1] {
+			t.Fatalf("the index of kept lists = %s, want the last two of %v", raw, digests)
+		}
+		for i, d := range digests {
+			_, _, kept, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(d, "a.mod-x"))
+			if kept != (i > 0) {
+				t.Errorf("list %d's a.mod-x key kept = %v", i, kept)
+			}
+		}
+		for _, u := range idx.Lists[0].Users {
+			if _, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(digests[2], u)); !ok {
+				t.Errorf("the index names %s, whose key is not there", u)
+			}
+		}
+		// The loaded list's permissions are read back from the keys when the bus module's tools cannot say them.
+		b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
+			UserList: &delivery.UserListWritten{Source: delivery.GrantsUserListSource, Follows: delivery.FollowGrants, Revision: 8,
+				Digest: digests[2], Users: 6, Key: busKey.PublicKey}})
+		b.refuseUsers = true
+		if res := p.Compose(); slices.ContainsFunc(res.Grants.Waits, func(w string) bool { return strings.Contains(w, "unknown") }) {
+			t.Errorf("the kept permissions of the loaded list were not read: %v", res.Grants.Waits)
+		}
+	})
 }

 type fakeAsker struct {
@@ -82,12 +82,26 @@ func GrantWaits(tookOver bool, accounts []string, composed, loaded map[string]Us
 	return "waits for the bus to load a list carrying what it adds: " + strings.Join(adds, ", ")
 }

-// KeySealedGrants is the key of the state user-list holding, per list the part sealed (the last two, by digest), each
-// user's permissions as sealed, never a hash: what tells which grants the list the bus loaded carries, without the bus
-// module's tools (issue 774).
+// **The permissions of each list the part sealed, one key per user** (issue 774, sweep 2026-10-30 item 5): the last two
+// lists, by digest, each user's permissions as sealed, never a hash: what tells which grants the list the bus loaded
+// carries, without the bus module's tools. One value holding every user of two lists outgrew the bucket's largest value
+// (JetStream 10054, "message size exceeds maximum", 16:37 and 16:55 on 2026-10-30), so a list's permissions were never
+// kept and a grant judged against them waited. Each user's permissions are a key of their own,
+// `sealed-grants.<digest hex>.<user>`, which grows with that user alone; the key `sealed-lists` names the lists kept and
+// their users, and is written only once every user's key of its newest list is: it never names a list whose keys are not
+// all there.
+
+// KeySealedGrants is the key that held every user of the last two lists in one value, before this release: read when the
+// index names no list, and taken away once the index is written.
 const KeySealedGrants = "sealed-grants"

-// SealedGrants is the value of KeySealedGrants.
+// KeySealedLists is the key of the state user-list naming the sealed lists whose permissions are kept, newest first.
+const KeySealedLists = "sealed-lists"
+
+// sealedKept is how many sealed lists keep their permissions: the one in force and the one before.
+const sealedKept = 2
+
+// SealedGrants is the value KeySealedGrants held.
 type SealedGrants struct {
 	Lists []SealedList `json:"lists"`
 }
@@ -98,7 +112,24 @@ type SealedList struct {
 	Users  []User `json:"users"`
 }

-// keepSealedGrants records the permissions of the list sealed now, beside the one before.
+// SealedLists is the value of KeySealedLists: the lists kept, newest first, each by its digest and its users' names.
+type SealedLists struct {
+	Lists []SealedNames `json:"lists"`
+}
+
+// SealedNames is one kept list: its digest and its users, sorted.
+type SealedNames struct {
+	Digest string   `json:"digest"`
+	Users  []string `json:"users"`
+}
+
+// SealedUserKey is the key of one user's permissions in the sealed list of digest.
+func SealedUserKey(digest, user string) string {
+	return KeySealedGrants + "." + strings.TrimPrefix(digest, "sha256:") + "." + user
+}
+
+// keepSealedGrants records the permissions of the list sealed now, one key per user, beside the list before; the lists
+// older than those two lose their keys.
 func (p *Part) keepSealedGrants(res *Result, digest string, list []byte, composed []User) {
 	in := listUsers(list)
 	var users []User
@@ -108,20 +139,98 @@ func (p *Part) keepSealedGrants(res *Result, digest string, list []byte, compose
 			users = append(users, u)
 		}
 	}
-	var was SealedGrants
-	if raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants); err == nil && found {
-		_ = json.Unmarshal(raw, &was)
+	failed := false
+	names := make([]string, 0, len(users))
+	for _, u := range users {
+		names = append(names, u.User)
+		data, _ := json.Marshal(u)
+		if _, err := p.Store.Put(StateUserList, SealedUserKey(digest, u.User), data); err != nil {
+			failed = true
+			res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, SealedUserKey(digest, u.User), err))
+		}
 	}
-	next := SealedGrants{Lists: []SealedList{{Digest: digest, Users: users}}}
+	if failed {
+		// The index keeps naming the lists whose keys are all there; this list's permissions are not kept.
+		return
+	}
+	sort.Strings(names)
+	was := p.sealedLists()
+	next := SealedLists{Lists: []SealedNames{{Digest: digest, Users: names}}}
+	var gone []SealedNames
 	for _, l := range was.Lists {
-		if l.Digest != digest && len(next.Lists) < 2 {
+		switch {
+		case l.Digest == digest:
+		case len(next.Lists) < sealedKept:
 			next.Lists = append(next.Lists, l)
+		default:
+			gone = append(gone, l)
 		}
 	}
 	data, _ := json.Marshal(next)
-	if _, err := p.Store.Put(StateUserList, KeySealedGrants, data); err != nil {
-		res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeySealedGrants, err))
+	if _, err := p.Store.Put(StateUserList, KeySealedLists, data); err != nil {
+		res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeySealedLists, err))
+		return
 	}
+	for _, l := range gone {
+		for _, user := range l.Users {
+			if err := p.Store.Delete(StateUserList, SealedUserKey(l.Digest, user)); err != nil {
+				res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be taken away: %v", StateUserList, SealedUserKey(l.Digest, user), err))
+			}
+		}
+	}
+	if _, _, found, err := p.Store.Get(StateUserList, KeySealedGrants); err == nil && found {
+		if err := p.Store.Delete(StateUserList, KeySealedGrants); err != nil {
+			res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be taken away: %v", StateUserList, KeySealedGrants, err))
+		}
+	}
+}
+
+// sealedLists is the index of kept lists; none when it cannot be read.
+func (p *Part) sealedLists() SealedLists {
+	var out SealedLists
+	if raw, _, found, err := p.Store.Get(StateUserList, KeySealedLists); err == nil && found {
+		_ = json.Unmarshal(raw, &out)
+	}
+	return out
+}
+
+// sealedGrantsOf is the permissions of each user of the sealed list of digest, as the part kept them, and whether it kept
+// every one: from the index and each user's key, else from the one value written before this release.
+func (p *Part) sealedGrantsOf(digest string) (map[string]User, bool) {
+	for _, l := range p.sealedLists().Lists {
+		if l.Digest != digest {
+			continue
+		}
+		out := map[string]User{}
+		for _, user := range l.Users {
+			raw, _, found, err := p.Store.Get(StateUserList, SealedUserKey(digest, user))
+			if err != nil || !found {
+				return nil, false
+			}
+			var u User
+			if json.Unmarshal(raw, &u) != nil {
+				return nil, false
+			}
+			out[u.User] = u
+		}
+		return out, true
+	}
+	raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants)
+	if err == nil && found {
+		var sg SealedGrants
+		if json.Unmarshal(raw, &sg) == nil {
+			for _, sl := range sg.Lists {
+				if sl.Digest == digest {
+					out := map[string]User{}
+					for _, u := range sl.Users {
+						out[u.User] = u
+					}
+					return out, true
+				}
+			}
+		}
+	}
+	return nil, false
 }

 // loadedContents is the list the bus loaded, by user: as the bus module's nats_users read it, else as the part sealed the
@@ -144,16 +253,8 @@ func (p *Part) loadedContents(in Inputs, read []User, readErr error) (map[string
 	case l.Source != delivery.GrantsUserListSource || l.Follows != delivery.FollowGrants || l.Error != "":
 		return nil, "the bus loaded no list of the grants part's (source " + l.Source + ", follows " + l.Follows + "), and nats_users: " + readErr.Error()
 	}
-	raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants)
-	if err == nil && found {
-		var sg SealedGrants
-		if json.Unmarshal(raw, &sg) == nil {
-			for _, sl := range sg.Lists {
-				if sl.Digest == l.Digest {
-					return byUser(sl.Users), ""
-				}
-			}
-		}
+	if kept, ok := p.sealedGrantsOf(l.Digest); ok {
+		return kept, ""
 	}
 	return nil, "the part keeps no permissions of the loaded list " + short(l.Digest) + " (sealed before it kept them), and nats_users: " + readErr.Error()
 }