Grants purges, once at its start, the sealed keys a delete left behind (issue 1966)

The markers and the values of the lists taken away before the purge still fill
the state user-list. Before its first pass the process purges every
sealed-grants key whose last operation is a delete or purge marker, and every
user's key of a list the index of kept lists does not name; never a kept
list's key nor another key of the state, and nothing when the index cannot be
read. It logs the counts and the state's bytes before and after.

Claude-Session: https://claude.ai/code/session_01NcbgfFfQ4Z9UE5jK32PmE5

modules/mesh-grants/cmd/mesh-grants/main.go

@@ -207,6 +207,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 		}
 		return nil
 	}
+	// Once, before the first pass writes a list's keys: the sealed lists' keys a delete left in the state user-list are
+	// purged (issue 1966), said with the count and the bytes before and after.
+	if sw, err := f.Part.SweepSealed(); err != nil {
+		logger.Printf("the sealed lists' keys of %s were not swept: %v", grants.StateUserList, err)
+	} else {
+		for _, e := range sw.Errors {
+			logger.Printf("%s", e)
+		}
+		logger.Printf("swept the sealed lists' keys of %s: %d marked by a delete and %d of lists no longer kept purged, the kept lists %v untouched (no index: %v); %d bytes before, %d after",
+			grants.StateUserList, sw.Marked, sw.Orphaned, sw.Kept, sw.NoIndex, sw.BytesBefore, sw.BytesAfter)
+	}
 	logger.Printf("started on the account %s: publishing memberships, writing grants and the user list sealed to the bus node, which it follows, and creating the account consumers (the heard-stream ones too) and the declared buckets, logs and streams the bus lacks; a held stream and a held consumer only ever gain the subjects and filters a manifest adds", account.User)
 	f.Run(ctx)
 	return nil

modules/mesh-grants/grants_test.go

@@ -3682,6 +3682,79 @@ func TestGrants(t *testing.T) {
 		}
 	})
 
+	// Issue 1966, 2026-11-08 23:20: the state user-list held 5,968 keys, most of them markers of deletes whose values the
+	// bucket kept (94% of its cap). Once at the process's start, the marked sealed-grants keys and the keys of lists the
+	// index no longer names are purged; a kept list's keys and every other key of the state stay.
+	t.Run("sealed sweep: purges marked sealed-grants keys and lists no longer kept, never a kept list's or another key", func(t *testing.T) {
+		const d0, d1, d2, d9 = "sha256:d0", "sha256:d1", "sha256:d2", "sha256:d9"
+		seed := func(index bool) (*fakeBus, *fakePurging) {
+			b := fixture()
+			pg := &fakePurging{b: b, marked: map[string]bool{}}
+			if index {
+				raw, _ := json.Marshal(grants.SealedLists{Lists: []grants.SealedNames{{Digest: d2, Users: []string{"a.mod-x"}}, {Digest: d1, Users: []string{"a.mod-x"}}}})
+				_, _ = ownStore{b}.Put(grants.StateUserList, grants.KeySealedLists, raw)
+			}
+			for _, k := range []string{grants.SealedUserKey(d2, "a.mod-x"), grants.SealedUserKey(d1, "a.mod-x"), grants.SealedUserKey(d0, "a.mod-x"),
+				grants.SealedUserKey(d0, "b.mod-y"), "pass", grants.KeyWrittenCredentials} {
+				_, _ = ownStore{b}.Put(grants.StateUserList, k, []byte(`{"user":"a.mod-x","pad":"`+strings.Repeat("x", 4800)+`"}`))
+			}
+			// Markers: a deleted list's keys, the one value written before the index, a kept list's key and another key.
+			for _, k := range []string{grants.SealedUserKey(d9, "a.mod-x"), grants.SealedUserKey(d9, "b.mod-y"), grants.KeySealedGrants,
+				grants.SealedUserKey(d1, "b.mod-y"), grants.KeyHashHold} {
+				pg.marked[k] = true
+			}
+			return b, pg
+		}
+		rows := []struct {
+			name              string
+			index, refuse     bool
+			marked, orphaned  int
+			purged, untouched []string
+			err               string
+		}{
+			{name: "with the index", index: true, marked: 3, orphaned: 2,
+				purged: []string{grants.SealedUserKey(d9, "a.mod-x"), grants.SealedUserKey(d9, "b.mod-y"), grants.KeySealedGrants,
+					grants.SealedUserKey(d0, "a.mod-x"), grants.SealedUserKey(d0, "b.mod-y")}},
+			{name: "no index: the marked keys alone, no list kept", marked: 4,
+				purged: []string{grants.SealedUserKey(d9, "a.mod-x"), grants.SealedUserKey(d9, "b.mod-y"), grants.KeySealedGrants,
+					grants.SealedUserKey(d1, "b.mod-y")},
+				untouched: []string{grants.SealedUserKey(d0, "a.mod-x"), grants.SealedUserKey(d0, "b.mod-y")}},
+			{name: "an unreadable index: nothing", index: true, refuse: true, err: "index of kept lists cannot be read"},
+		}
+		for _, r := range rows {
+			b, pg := seed(r.index)
+			p := partOn(b)
+			p.Purging = pg
+			if r.refuse {
+				p.Store = refusingIndex{ownStore{b}}
+			}
+			sw, err := p.SweepSealed()
+			if r.err != "" {
+				if err == nil || !strings.Contains(err.Error(), r.err) || len(pg.purged) > 0 {
+					t.Errorf("%s: err %v, purged %v", r.name, err, pg.purged)
+				}
+				continue
+			}
+			if err != nil || sw.Marked != r.marked || sw.Orphaned != r.orphaned || sw.NoIndex == r.index || sw.BytesAfter >= sw.BytesBefore {
+				t.Errorf("%s: %+v, %v", r.name, sw, err)
+			}
+			if !slices.Equal(slices.Sorted(slices.Values(pg.purged)), slices.Sorted(slices.Values(r.purged))) {
+				t.Errorf("%s: purged %v, want %v", r.name, pg.purged, r.purged)
+			}
+			for _, k := range append([]string{grants.SealedUserKey(d2, "a.mod-x"), grants.SealedUserKey(d1, "a.mod-x"), "pass", grants.KeyWrittenCredentials}, r.untouched...) {
+				if _, _, ok, _ := b.ownGet(grants.StateUserList, k); !ok {
+					t.Errorf("%s: %s was taken away", r.name, k)
+				}
+			}
+			if (r.index && !pg.marked[grants.SealedUserKey(d1, "b.mod-y")]) || !pg.marked[grants.KeyHashHold] {
+				t.Errorf("%s: a kept list's or another key's marker was purged: %v", r.name, pg.purged)
+			}
+		}
+		if _, err := partOn(fixture()).SweepSealed(); err == nil {
+			t.Error("a part with no purger swept")
+		}
+	})
+
 	// Sweep 2026-10-30 item 5, a kind met twice that day (the engine's acts, then mesh-host_health): one principal the
 	// composer refuses as a second writer froze the whole list. Now it keeps its grants of the list in force, the rest is
 	// written, and the refusal is raised by name; one the list in force does not carry stays out, said too.
@@ -3991,6 +4064,16 @@ func (f *fakePurging) Bytes(state string) (uint64, error) {
 	return n, nil
 }
 
+// refusingIndex is the part's own state with the index of kept lists unreadable.
+type refusingIndex struct{ ownStore }
+
+func (r refusingIndex) Get(state, key string) ([]byte, uint64, bool, error) {
+	if key == grants.KeySealedLists {
+		return nil, 0, false, errors.New("nats: timeout")
+	}
+	return r.ownStore.Get(state, key)
+}
+
 // fakeObjects is the part's object bucket of sealed lists.
 type fakeObjects struct{ objects map[string][]byte }
 

modules/mesh-grants/internal/grants/purge.go

@@ -1,5 +1,12 @@
 package grants
 
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+)
+
 // Purger takes keys of the part's own state away whole (issue 1966, 2026-11-08): a key-value delete only adds a marker
 // and leaves every value the key kept in the bucket's stream, so the per-user keys of the sealed lists older than the
 // two kept filled the state user-list to 94% of its cap (5,968 keys, 13,228 values kept, 23:20 UTC). A purge takes the
@@ -21,3 +28,87 @@ func (p *Part) takeAway(state, key string) error {
 	}
 	return p.Store.Delete(state, key)
 }
+
+// sealedPrefix is the prefix of every user's key of the sealed list of digest.
+func sealedPrefix(digest string) string { return SealedUserKey(digest, "") }
+
+// SealedSweep is what SweepSealed took away of the state user-list.
+type SealedSweep struct {
+	// Marked are the sealed-grants keys whose last operation was a delete or purge marker, purged; Orphaned the live keys
+	// of a list the index of kept lists does not name, purged.
+	Marked   int `json:"marked"`
+	Orphaned int `json:"orphaned"`
+	// Kept is the digests of the kept lists, whose keys the sweep never touches.
+	Kept []string `json:"kept"`
+	// NoIndex says the state holds no index of kept lists: then only the marked keys are purged.
+	NoIndex     bool     `json:"no_index,omitempty"`
+	BytesBefore uint64   `json:"bytes_before"`
+	BytesAfter  uint64   `json:"bytes_after"`
+	Errors      []string `json:"errors,omitempty"`
+}
+
+// SweepSealed purges, once, the keys of the sealed lists that a delete left in the state user-list (issue 1966): every
+// sealed-grants key whose last operation is a delete or purge marker, and every user's key of a list the index of kept
+// lists does not name. Never a key of a kept list, and never another key of the state. Run at the process's start,
+// before its first pass, so no pass writes a list's keys beside it. Refused whole when the index cannot be read.
+func (p *Part) SweepSealed() (SealedSweep, error) {
+	var out SealedSweep
+	if p.Purging == nil {
+		return out, errors.New("the part holds no purger: nothing is taken away")
+	}
+	raw, _, found, err := p.Store.Get(StateUserList, KeySealedLists)
+	if err != nil {
+		return out, fmt.Errorf("the index of kept lists cannot be read, so which keys are kept is not known: %w", err)
+	}
+	var idx SealedLists
+	if found {
+		if err := json.Unmarshal(raw, &idx); err != nil {
+			return out, fmt.Errorf("the index of kept lists cannot be read, so which keys are kept is not known: %w", err)
+		}
+	}
+	out.NoIndex = !found
+	var kept []string
+	for _, l := range idx.Lists {
+		out.Kept = append(out.Kept, l.Digest)
+		kept = append(kept, sealedPrefix(l.Digest))
+	}
+	if out.BytesBefore, err = p.Purging.Bytes(StateUserList); err != nil {
+		return out, fmt.Errorf("the state %s cannot be read: %w", StateUserList, err)
+	}
+	isKept := func(key string) bool {
+		for _, pre := range kept {
+			if strings.HasPrefix(key, pre) {
+				return true
+			}
+		}
+		return false
+	}
+	purge := func(key string, n *int) {
+		if err := p.Purging.Purge(StateUserList, key); err != nil {
+			out.Errors = append(out.Errors, fmt.Sprintf("its own %s %s cannot be purged: %v", StateUserList, key, err))
+			return
+		}
+		*n++
+	}
+	// The one value written before the index (KeySealedGrants), once deleted, and every user's key.
+	for _, pattern := range []string{KeySealedGrants, KeySealedGrants + ".>"} {
+		ops, err := p.Purging.LastOps(StateUserList, pattern)
+		if err != nil {
+			return out, fmt.Errorf("the keys %s of %s cannot be listed: %w", pattern, StateUserList, err)
+		}
+		for _, key := range sortedKeys(ops) {
+			switch {
+			case key != KeySealedGrants && !strings.HasPrefix(key, KeySealedGrants+"."):
+			case key != KeySealedGrants && isKept(key):
+			case ops[key]:
+				purge(key, &out.Marked)
+			case key != KeySealedGrants && found:
+				purge(key, &out.Orphaned)
+			}
+		}
+	}
+	if out.BytesAfter, err = p.Purging.Bytes(StateUserList); err != nil {
+		out.Errors = append(out.Errors, fmt.Sprintf("the state %s cannot be read after the sweep: %v", StateUserList, err))
+	}
+	return out, nil
+}