From 126b40e6f585f8522928ee88864c24e6f4a11a2d Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 00:22:00 +0100 Subject: [PATCH] Grants purges, once at its start, the sealed keys a delete left behind (issue 1966) The markers and the values of the lists taken away before the purge still fill the state user-list. Before its first pass the process purges every sealed-grants key whose last operation is a delete or purge marker, and every user's key of a list the index of kept lists does not name; never a kept list's key nor another key of the state, and nothing when the index cannot be read. It logs the counts and the state's bytes before and after. Claude-Session: https://claude.ai/code/session_01NcbgfFfQ4Z9UE5jK32PmE5 --- modules/mesh-grants/cmd/mesh-grants/main.go | 11 +++ modules/mesh-grants/grants_test.go | 83 ++++++++++++++++++ modules/mesh-grants/internal/grants/purge.go | 91 ++++++++++++++++++++ 3 files changed, 185 insertions(+) diff --git a/modules/mesh-grants/cmd/mesh-grants/main.go b/modules/mesh-grants/cmd/mesh-grants/main.go index f5d37aa6..1867adc9 100644 --- a/modules/mesh-grants/cmd/mesh-grants/main.go +++ b/modules/mesh-grants/cmd/mesh-grants/main.go @@ -207,6 +207,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } return nil } + // Once, before the first pass writes a list's keys: the sealed lists' keys a delete left in the state user-list are + // purged (issue 1966), said with the count and the bytes before and after. + if sw, err := f.Part.SweepSealed(); err != nil { + logger.Printf("the sealed lists' keys of %s were not swept: %v", grants.StateUserList, err) + } else { + for _, e := range sw.Errors { + logger.Printf("%s", e) + } + logger.Printf("swept the sealed lists' keys of %s: %d marked by a delete and %d of lists no longer kept purged, the kept lists %v untouched (no index: %v); %d bytes before, %d after", + grants.StateUserList, sw.Marked, sw.Orphaned, sw.Kept, sw.NoIndex, sw.BytesBefore, sw.BytesAfter) + } logger.Printf("started on the account %s: publishing memberships, writing grants and the user list sealed to the bus node, which it follows, and creating the account consumers (the heard-stream ones too) and the declared buckets, logs and streams the bus lacks; a held stream and a held consumer only ever gain the subjects and filters a manifest adds", account.User) f.Run(ctx) return nil diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index 0c53614b..99d97ff2 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -3682,6 +3682,79 @@ func TestGrants(t *testing.T) { } }) + // Issue 1966, 2026-11-08 23:20: the state user-list held 5,968 keys, most of them markers of deletes whose values the + // bucket kept (94% of its cap). Once at the process's start, the marked sealed-grants keys and the keys of lists the + // index no longer names are purged; a kept list's keys and every other key of the state stay. + t.Run("sealed sweep: purges marked sealed-grants keys and lists no longer kept, never a kept list's or another key", func(t *testing.T) { + const d0, d1, d2, d9 = "sha256:d0", "sha256:d1", "sha256:d2", "sha256:d9" + seed := func(index bool) (*fakeBus, *fakePurging) { + b := fixture() + pg := &fakePurging{b: b, marked: map[string]bool{}} + if index { + raw, _ := json.Marshal(grants.SealedLists{Lists: []grants.SealedNames{{Digest: d2, Users: []string{"a.mod-x"}}, {Digest: d1, Users: []string{"a.mod-x"}}}}) + _, _ = ownStore{b}.Put(grants.StateUserList, grants.KeySealedLists, raw) + } + for _, k := range []string{grants.SealedUserKey(d2, "a.mod-x"), grants.SealedUserKey(d1, "a.mod-x"), grants.SealedUserKey(d0, "a.mod-x"), + grants.SealedUserKey(d0, "b.mod-y"), "pass", grants.KeyWrittenCredentials} { + _, _ = ownStore{b}.Put(grants.StateUserList, k, []byte(`{"user":"a.mod-x","pad":"`+strings.Repeat("x", 4800)+`"}`)) + } + // Markers: a deleted list's keys, the one value written before the index, a kept list's key and another key. + for _, k := range []string{grants.SealedUserKey(d9, "a.mod-x"), grants.SealedUserKey(d9, "b.mod-y"), grants.KeySealedGrants, + grants.SealedUserKey(d1, "b.mod-y"), grants.KeyHashHold} { + pg.marked[k] = true + } + return b, pg + } + rows := []struct { + name string + index, refuse bool + marked, orphaned int + purged, untouched []string + err string + }{ + {name: "with the index", index: true, marked: 3, orphaned: 2, + purged: []string{grants.SealedUserKey(d9, "a.mod-x"), grants.SealedUserKey(d9, "b.mod-y"), grants.KeySealedGrants, + grants.SealedUserKey(d0, "a.mod-x"), grants.SealedUserKey(d0, "b.mod-y")}}, + {name: "no index: the marked keys alone, no list kept", marked: 4, + purged: []string{grants.SealedUserKey(d9, "a.mod-x"), grants.SealedUserKey(d9, "b.mod-y"), grants.KeySealedGrants, + grants.SealedUserKey(d1, "b.mod-y")}, + untouched: []string{grants.SealedUserKey(d0, "a.mod-x"), grants.SealedUserKey(d0, "b.mod-y")}}, + {name: "an unreadable index: nothing", index: true, refuse: true, err: "index of kept lists cannot be read"}, + } + for _, r := range rows { + b, pg := seed(r.index) + p := partOn(b) + p.Purging = pg + if r.refuse { + p.Store = refusingIndex{ownStore{b}} + } + sw, err := p.SweepSealed() + if r.err != "" { + if err == nil || !strings.Contains(err.Error(), r.err) || len(pg.purged) > 0 { + t.Errorf("%s: err %v, purged %v", r.name, err, pg.purged) + } + continue + } + if err != nil || sw.Marked != r.marked || sw.Orphaned != r.orphaned || sw.NoIndex == r.index || sw.BytesAfter >= sw.BytesBefore { + t.Errorf("%s: %+v, %v", r.name, sw, err) + } + if !slices.Equal(slices.Sorted(slices.Values(pg.purged)), slices.Sorted(slices.Values(r.purged))) { + t.Errorf("%s: purged %v, want %v", r.name, pg.purged, r.purged) + } + for _, k := range append([]string{grants.SealedUserKey(d2, "a.mod-x"), grants.SealedUserKey(d1, "a.mod-x"), "pass", grants.KeyWrittenCredentials}, r.untouched...) { + if _, _, ok, _ := b.ownGet(grants.StateUserList, k); !ok { + t.Errorf("%s: %s was taken away", r.name, k) + } + } + if (r.index && !pg.marked[grants.SealedUserKey(d1, "b.mod-y")]) || !pg.marked[grants.KeyHashHold] { + t.Errorf("%s: a kept list's or another key's marker was purged: %v", r.name, pg.purged) + } + } + if _, err := partOn(fixture()).SweepSealed(); err == nil { + t.Error("a part with no purger swept") + } + }) + // Sweep 2026-10-30 item 5, a kind met twice that day (the engine's acts, then mesh-host_health): one principal the // composer refuses as a second writer froze the whole list. Now it keeps its grants of the list in force, the rest is // written, and the refusal is raised by name; one the list in force does not carry stays out, said too. @@ -3991,6 +4064,16 @@ func (f *fakePurging) Bytes(state string) (uint64, error) { return n, nil } +// refusingIndex is the part's own state with the index of kept lists unreadable. +type refusingIndex struct{ ownStore } + +func (r refusingIndex) Get(state, key string) ([]byte, uint64, bool, error) { + if key == grants.KeySealedLists { + return nil, 0, false, errors.New("nats: timeout") + } + return r.ownStore.Get(state, key) +} + // fakeObjects is the part's object bucket of sealed lists. type fakeObjects struct{ objects map[string][]byte } diff --git a/modules/mesh-grants/internal/grants/purge.go b/modules/mesh-grants/internal/grants/purge.go index 12f09d94..eaf0bd40 100644 --- a/modules/mesh-grants/internal/grants/purge.go +++ b/modules/mesh-grants/internal/grants/purge.go @@ -1,5 +1,12 @@ package grants +import ( + "encoding/json" + "errors" + "fmt" + "strings" +) + // Purger takes keys of the part's own state away whole (issue 1966, 2026-11-08): a key-value delete only adds a marker // and leaves every value the key kept in the bucket's stream, so the per-user keys of the sealed lists older than the // two kept filled the state user-list to 94% of its cap (5,968 keys, 13,228 values kept, 23:20 UTC). A purge takes the @@ -21,3 +28,87 @@ func (p *Part) takeAway(state, key string) error { } return p.Store.Delete(state, key) } + +// sealedPrefix is the prefix of every user's key of the sealed list of digest. +func sealedPrefix(digest string) string { return SealedUserKey(digest, "") } + +// SealedSweep is what SweepSealed took away of the state user-list. +type SealedSweep struct { + // Marked are the sealed-grants keys whose last operation was a delete or purge marker, purged; Orphaned the live keys + // of a list the index of kept lists does not name, purged. + Marked int `json:"marked"` + Orphaned int `json:"orphaned"` + // Kept is the digests of the kept lists, whose keys the sweep never touches. + Kept []string `json:"kept"` + // NoIndex says the state holds no index of kept lists: then only the marked keys are purged. + NoIndex bool `json:"no_index,omitempty"` + BytesBefore uint64 `json:"bytes_before"` + BytesAfter uint64 `json:"bytes_after"` + Errors []string `json:"errors,omitempty"` +} + +// SweepSealed purges, once, the keys of the sealed lists that a delete left in the state user-list (issue 1966): every +// sealed-grants key whose last operation is a delete or purge marker, and every user's key of a list the index of kept +// lists does not name. Never a key of a kept list, and never another key of the state. Run at the process's start, +// before its first pass, so no pass writes a list's keys beside it. Refused whole when the index cannot be read. +func (p *Part) SweepSealed() (SealedSweep, error) { + var out SealedSweep + if p.Purging == nil { + return out, errors.New("the part holds no purger: nothing is taken away") + } + raw, _, found, err := p.Store.Get(StateUserList, KeySealedLists) + if err != nil { + return out, fmt.Errorf("the index of kept lists cannot be read, so which keys are kept is not known: %w", err) + } + var idx SealedLists + if found { + if err := json.Unmarshal(raw, &idx); err != nil { + return out, fmt.Errorf("the index of kept lists cannot be read, so which keys are kept is not known: %w", err) + } + } + out.NoIndex = !found + var kept []string + for _, l := range idx.Lists { + out.Kept = append(out.Kept, l.Digest) + kept = append(kept, sealedPrefix(l.Digest)) + } + if out.BytesBefore, err = p.Purging.Bytes(StateUserList); err != nil { + return out, fmt.Errorf("the state %s cannot be read: %w", StateUserList, err) + } + isKept := func(key string) bool { + for _, pre := range kept { + if strings.HasPrefix(key, pre) { + return true + } + } + return false + } + purge := func(key string, n *int) { + if err := p.Purging.Purge(StateUserList, key); err != nil { + out.Errors = append(out.Errors, fmt.Sprintf("its own %s %s cannot be purged: %v", StateUserList, key, err)) + return + } + *n++ + } + // The one value written before the index (KeySealedGrants), once deleted, and every user's key. + for _, pattern := range []string{KeySealedGrants, KeySealedGrants + ".>"} { + ops, err := p.Purging.LastOps(StateUserList, pattern) + if err != nil { + return out, fmt.Errorf("the keys %s of %s cannot be listed: %w", pattern, StateUserList, err) + } + for _, key := range sortedKeys(ops) { + switch { + case key != KeySealedGrants && !strings.HasPrefix(key, KeySealedGrants+"."): + case key != KeySealedGrants && isKept(key): + case ops[key]: + purge(key, &out.Marked) + case key != KeySealedGrants && found: + purge(key, &out.Orphaned) + } + } + } + if out.BytesAfter, err = p.Purging.Bytes(StateUserList); err != nil { + out.Errors = append(out.Errors, fmt.Sprintf("the state %s cannot be read after the sweep: %v", StateUserList, err)) + } + return out, nil +}