The grants part writes its list whole however large, one refused principal no longer freezes it, and stale watchman keys close (issue 774) #117

Merged
jschoubben merged 10 commits from fix/sweep-2026-10-30 into main 2026-10-30 18:13:00 +00:00
5 changed files with 328 additions and 33 deletions
Showing only changes of commit 031cfb8277 - Show all commits
+105 -2
View File
@@ -1547,7 +1547,7 @@ func TestGrants(t *testing.T) {
fp := partOn(b)
fp.Objects = p.Objects
fp.Asker, fp.Say = fa, func(c asks.Condition) error {
if c.Key != grants.GrantMissingCondition { // its own row below
if c.Key != grants.GrantMissingCondition && c.Key != grants.RefusedCondition { // their own rows
fresh = append(fresh, c)
}
return nil
@@ -2445,7 +2445,7 @@ func TestGrants(t *testing.T) {
// the composed configuration only by hand (grants_consumer_match), never while the controller holds the lease.
var said []asks.Condition
p.Say = func(c asks.Condition) error {
if c.Key != grants.GrantMissingCondition { // its own row
if c.Key != grants.GrantMissingCondition && c.Key != grants.RefusedCondition { // their own rows
said = append(said, c)
}
return nil
@@ -2629,6 +2629,109 @@ func TestGrants(t *testing.T) {
t.Errorf("the kept permissions of the loaded list were not read: %v", res.Grants.Waits)
}
})
// Sweep 2026-10-30 item 5, a kind met twice that day (the engine's acts, then mesh-host_health): one principal the
// composer refuses as a second writer froze the whole list. Now it keeps its grants of the list in force, the rest is
// written, and the refusal is raised by name; one the list in force does not carry stays out, said too.
t.Run("a refused principal keeps its grants of the list in force; the rest of the list is written", func(t *testing.T) {
b := fixture()
const ctl = `{"module":"mesh-controller","version":"1","own-secrets":{"broker":"/var/lib/c/broker"}}`
refusedCtl := strings.Replace(ctl, `}}`, `},"state":[{"name":"declared-config"}]}`, 1)
b.manifestOf("mesh-controller", ctl)
b.assign("b", "mesh-controller")
hash := func(user, h string, gen int) {
b.put(grants.ReadAccounts, user, map[string]any{"user": user, "hash": h, "generation": gen, "fingerprint": fmt.Sprintf("sha256:%d", gen)})
}
for _, u := range shadowOf(t, b) {
hash(u.User, "$2a$10$first", 1)
}
hash("a.mesh-controller", "$2a$10$first", 1)
busKey, err := seal.GenerateKeyPair()
if err != nil {
t.Fatal(err)
}
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}})
p := partOn(b)
objs := &fakeObjects{}
p.Objects = objs
entryOf := func(g delivery.GrantsUserList, user string) string {
t.Helper()
list, err := delivery.OpenUserList(objs.objects[g.Object], g.Digest, func(bx seal.Box) ([]byte, error) {
out, err := seal.Open(bx, busKey.PrivateKey)
return []byte(out), err
})
if err != nil {
t.Fatalf("the sealed list does not open: %v", err)
}
at := strings.Index(string(list), fmt.Sprintf("{ user: %q,", user))
if at < 0 {
return ""
}
rest := string(list)[at:]
return rest[:strings.Index(rest, "} }\n")+4]
}
var said []asks.Condition
p.Say = func(c asks.Condition) error {
if c.Key == grants.RefusedCondition {
said = append(said, c)
}
return nil
}
sw, _ := json.Marshal(grants.Switch{V: 1, State: grants.TookOver, Acts: []grants.Act{}})
if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil {
t.Fatal(err)
}
p.Compose()
inForce, _, _ := p.Sealed()
if inForce.Digest == "" {
t.Fatal("no list written after take-over")
}
entryBefore := entryOf(inForce, "b.mesh-controller")
ctlKey := grants.SealedUserKey(inForce.Digest, "b.mesh-controller")
before, _, ok, _ := b.ownGet(grants.StateUserList, ctlKey)
if !ok {
t.Fatal("the list in force keeps no grants of b.mesh-controller")
}
// b.mesh-controller now declares the controller's mirror as its own state: refused as a second writer. a.mod-x's hash
// changes in the same pass, and a.mesh-controller, refused the same way, is placed new.
b.manifestOf("mesh-controller", refusedCtl)
b.assign("a", "mesh-controller")
hash("a.mod-x", "$2a$10$second", 2)
res := p.Compose()
now, _, _ := p.Sealed()
if now.Digest == inForce.Digest || slices.ContainsFunc(res.Errors, func(e string) bool { return strings.Contains(e, "composed whole") }) {
t.Fatalf("a refused principal froze the list: sealed %+v, errors %v", res.Sealed, res.Errors)
}
after, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(now.Digest, "b.mesh-controller"))
if !ok || string(after) != string(before) {
t.Errorf("b.mesh-controller did not keep its grants of the list in force: %s, was %s", after, before)
}
// The kept user is written as the composer writes one: the bus reads the same entry it loaded.
if got := entryOf(now, "b.mesh-controller"); entryBefore == "" || got != entryBefore {
t.Errorf("the kept entry is written otherwise than the list in force holds it:\n%s\nwas\n%s", got, entryBefore)
}
if entryOf(now, "a.mesh-controller") != "" {
t.Error("a refused principal the list in force did not carry is in the sealed list")
}
if _, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(now.Digest, "a.mesh-controller")); ok {
t.Error("a refused principal the list in force did not carry is in the list")
}
if len(res.PrincipalsRefused) != 2 || len(said) == 0 || said[len(said)-1].State != asks.StateRaised ||
!strings.Contains(said[len(said)-1].Summary, "b.mesh-controller (keeps its grants of the list in force)") ||
!strings.Contains(said[len(said)-1].Summary, "a.mesh-controller (left out of the list") {
t.Fatalf("the refusals were not said by name: %v, said %+v", res.PrincipalsRefused, said)
}
// Composing again changes nothing and says nothing new; once the manifest composes, the warning is cleared.
n := len(said)
if p.Compose(); len(said) != n {
t.Errorf("an unchanged refusal was said again: %+v", said[n:])
}
b.manifestOf("mesh-controller", ctl)
if res := p.Compose(); len(res.PrincipalsRefused) != 0 || said[len(said)-1].State != asks.StateCleared {
t.Errorf("the refusal was not cleared once it composes: %v, said %+v", res.PrincipalsRefused, said)
}
})
}
type fakeAsker struct {
+19 -7
View File
@@ -131,9 +131,12 @@ type Result struct {
// Grants is what the pass wrote of its grants (grants.go).
Grants *GrantsWritten `json:"grants,omitempty"`
// Withheld is what the pass wrote of the hearings it withholds (withheld.go).
Withheld *WithheldWritten `json:"withheld,omitempty"`
Errors []string `json:"errors,omitempty"`
PublishedOrAsserted string `json:"published_or_asserted"`
Withheld *WithheldWritten `json:"withheld,omitempty"`
// PrincipalsRefused is each principal whose composed grants the composer refused, `<user>: <why>`: it keeps its grants
// of the list in force, or stays out where it holds none (refused.go); the condition mesh-grants.principal-refused says it.
PrincipalsRefused []string `json:"principals_refused,omitempty"`
Errors []string `json:"errors,omitempty"`
PublishedOrAsserted string `json:"published_or_asserted"`
}
// NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships.
@@ -186,6 +189,9 @@ type Part struct {
// differSaid is the account consumers' differences last said as a condition, once differKnown (consumers.go).
differSaid string
differKnown bool
// refusedSaid is the refused principals last said as a condition, once refusedKnown (refused.go).
refusedSaid string
refusedKnown bool
// missingSaid is the modules without a grant last said as a condition, once missingKnown (grants.go).
missingSaid string
missingKnown bool
@@ -251,11 +257,17 @@ func (p *Part) Compose() Result {
if st.LeftOut == nil {
st.LeftOut = []Gap{}
}
users, err := ComposeUsers(listRecs)
users, refused, err := ComposeEach(listRecs)
if err != nil {
res.Errors = append(res.Errors, "the shadow list could not be composed: "+err.Error())
}
users = WithHashes(users, in.Hashes)
// A refused principal keeps its grants of the list in force, said once per change (refused.go).
kept, _ := p.keptOf(refused)
for _, user := range sortedKeys(refused) {
res.PrincipalsRefused = append(res.PrincipalsRefused, user+": "+refused[user])
}
p.sayPrincipalsRefused(refused, kept)
users = WithHashes(withKept(users, kept), in.Hashes)
st.Digest, st.Users = Digest(users), len(users)
loaded, loadErr := LoadedUsers(p.Call)
st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(listRecs)))
@@ -289,7 +301,7 @@ func (p *Part) Compose() Result {
if p.Store != nil {
p.writeWithheld(&res, in, recs)
}
list, n, leftOut, err := ComposeList(listRecs, in.Hashes)
list, n, leftOut, dropped, err := ComposeList(listRecs, in.Hashes, refused, kept)
// The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5); after take-over
// one that adds a permission only once the bus loaded a list that carries it (issue 774).
if p.Store != nil {
@@ -306,7 +318,7 @@ func (p *Part) Compose() Result {
bus = names(loaded)
lost = holdings.LastHolders(holdings.ByUsers(loaded), holdings.ByUsers(users))
}
p.writeSealed(&res, list, n, leftOut, in, bus, lost, users)
p.writeSealed(&res, list, n, leftOut, dropped, in, bus, lost, users)
}
ms := Memberships(recs)
res.Memberships = len(ms)
@@ -0,0 +1,157 @@
package grants
import (
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
// **One refused principal never freezes the list** (issue 774, sweep 2026-10-30; met twice that day: the engine's acts,
// then mesh-host_health). The composer refuses a principal's grants when they would make it a second writer of a state
// (buscompose.CheckWriters): that refusal used to end the whole composition, so the list in force stood for every principal
// while each change since piled up, and the next list that composed moved 52 users at once. Now each principal is composed
// on its own: one refused keeps the grants it holds in the list in force (the part's own record of that list, sealed
// grants), the rest of the list is composed and written as ever, and the refusal is raised as the warning
// mesh-grants.principal-refused, naming each principal and why, until it composes again. A refused principal the list in
// force does not carry stays out of the list: it never had those grants, and is refused them now.
// RefusedCondition is the key of the warning raised while a principal's grants are refused.
const RefusedCondition = "mesh-grants.principal-refused"
// ComposeEach is the shadow list composed principal by principal: every user whose grants compose, and each refused one
// by user with why. err is only for principals that cannot be derived at all (two users of one name). Pure.
func ComposeEach(r buscompose.Records) ([]User, map[string]string, error) {
principals, err := buscompose.Users(r)
if err != nil {
return nil, nil, err
}
out := make([]User, 0, len(principals))
refused := map[string]string{}
for _, p := range principals {
perms, err := buscompose.PermissionsFor(p)
if err != nil {
refused[p.Username()] = err.Error()
continue
}
out = append(out, User{User: p.Username(), Publish: set(perms.Publish), Subscribe: set(perms.Subscribe),
AllowResponses: perms.AllowResponses, PublishDeny: set(perms.PublishDeny)})
}
sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User })
return out, refused, nil
}
// keptOf is, for each refused user, its grants in the list in force as the part sealed them; the users it holds none of
// (not in the list in force, or that list's grants not kept) are the second answer, sorted.
func (p *Part) keptOf(refused map[string]string) (map[string]User, []string) {
if len(refused) == 0 {
return nil, nil
}
var inForce map[string]User
if g, _, had := p.Sealed(); had && g.Digest != "" && p.Store != nil {
inForce, _ = p.sealedGrantsOf(g.Digest)
}
kept := map[string]User{}
var none []string
for _, user := range sortedKeys(refused) {
if u, ok := inForce[user]; ok {
kept[user] = u
continue
}
none = append(none, user)
}
return kept, none
}
// withKept is users with each kept user in its place, sorted.
func withKept(users []User, kept map[string]User) []User {
out := slices.Clone(users)
for _, u := range kept {
out = append(out, u)
}
sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User })
return out
}
// keptEntry is a user of the accounts block as buscompose.ComposeAccounts writes one, from grants kept rather than
// composed: the same lines, so the bus reads it as it read the list in force.
func keptEntry(u User, hash string) string {
var b strings.Builder
fmt.Fprintf(&b, " { user: %q, password: %q, permissions: {\n", u.User, hash)
if len(u.PublishDeny) > 0 {
fmt.Fprintf(&b, " publish: { allow: [%s], deny: [%s] }\n", quotedList(u.Publish), quotedList(u.PublishDeny))
} else {
fmt.Fprintf(&b, " publish: { allow: [%s] }\n", quotedList(u.Publish))
}
fmt.Fprintf(&b, " subscribe: { allow: [%s] }\n", quotedList(u.Subscribe))
if u.AllowResponses {
fmt.Fprintf(&b, " allow_responses: { max: 1, ttl: \"%dm\" }\n", int(buscompose.ResponseTTL/time.Minute))
}
b.WriteString(" } }\n")
return b.String()
}
func quotedList(in []string) string {
q := make([]string, len(in))
for i, s := range in {
q[i] = fmt.Sprintf("%q", s)
}
return strings.Join(q, ", ")
}
// withEntries puts entries at the end of the accounts block's users, before it closes.
func withEntries(list string, entries []string) (string, error) {
if len(entries) == 0 {
return list, nil
}
start := strings.Index(list, " users = [\n")
if start < 0 {
return "", fmt.Errorf("the composed list holds no users block to keep a refused user's grants in")
}
end := strings.Index(list[start:], "\n ]\n")
if end < 0 {
return "", fmt.Errorf("the composed list's users block does not close")
}
at := start + end + 1
return list[:at] + strings.Join(entries, "") + list[at:], nil
}
// sayPrincipalsRefused raises RefusedCondition while a principal's grants are refused, naming each with why and what
// became of it, and clears it once none is; said once per set, again when it changes, and at the first pass after a start.
func (p *Part) sayPrincipalsRefused(refused map[string]string, kept map[string]User) {
if p.Say == nil {
return
}
var lines []string
for _, user := range sortedKeys(refused) {
became := "left out of the list: the list in force does not carry it"
if _, ok := kept[user]; ok {
became = "keeps its grants of the list in force"
}
lines = append(lines, user+" ("+became+"): "+refused[user])
}
why := strings.Join(lines, "; ")
p.holdMu.Lock()
defer p.holdMu.Unlock()
if p.refusedKnown && why == p.refusedSaid {
return
}
c := asks.Condition{Key: RefusedCondition, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}}
if why == "" {
c.State = asks.StateCleared
} else {
c.Severity, c.State = asks.SeverityWarning, asks.StateRaised
c.Summary = fmt.Sprintf("%d principal(s) refused their composed grants: %s", len(refused), why)
c.Headline, c.Needs = "A principal's bus grants are refused", "a fix of the manifest or the writers table"
c.Explanation = asks.NeedsYou + "a fix of what makes it a second writer. The rest of the user list is written as ever; " +
"each refused principal keeps the grants the list in force gives it, or stays out where it holds none: " + why
c.Resolved = "every principal's grants compose again"
}
if p.Say(c) == nil {
p.refusedSaid, p.refusedKnown = why, true
}
}
+40 -12
View File
@@ -31,24 +31,41 @@ type Objects interface {
DeleteObject(name string) error
}
// ComposeList is the bus's user list as the file holds it, with each user's hash from the vault, and the users left out
// for want of one. Pure.
func ComposeList(r buscompose.Records, hashes map[string]HeldHash) ([]byte, int, []string, error) {
// ComposeList is the bus's user list as the file holds it, with each user's hash from the vault; the users left out
// for want of one; and the refused principals (refused, by user) the list carries with their grants kept from the list
// in force (kept), or leaves out where kept holds none (dropped, sorted). Pure.
func ComposeList(r buscompose.Records, hashes map[string]HeldHash, refused map[string]string, kept map[string]User) (list []byte, users int, leftOut, dropped []string, err error) {
principals, err := buscompose.Users(r)
if err != nil {
return nil, 0, nil, err
return nil, 0, nil, nil, err
}
held := map[string]string{}
for user, h := range hashes {
held[user] = h.Hash
}
filled, missing := buscompose.WithPasswords(principals, held)
list, err := buscompose.ComposeAccounts(filled)
composing := slices.DeleteFunc(slices.Clone(principals), func(p buscompose.Principal) bool { _, no := refused[p.Username()]; return no })
filled, missing := buscompose.WithPasswords(composing, held)
text, err := buscompose.ComposeAccounts(filled)
if err != nil {
return nil, 0, nil, err
return nil, 0, nil, nil, err
}
var entries []string
for _, user := range sortedKeys(refused) {
u, ok := kept[user]
switch {
case !ok:
dropped = append(dropped, user)
case held[user] == "":
missing = append(missing, user)
default:
entries = append(entries, keptEntry(u, held[user]))
}
}
if text, err = withEntries(text, entries); err != nil {
return nil, 0, nil, nil, err
}
slices.Sort(missing)
return []byte(list), len(filled), missing, nil
return []byte(text), len(filled) + len(entries), missing, dropped, nil
}
// Sealed is the part's key `user-list.sealed` as written, and whether there is one.
@@ -74,7 +91,7 @@ func followOf(state string) string {
// writeSealed puts the list sealed to the bus node and writes the key, each only when what it says changed. It says in
// res what it wrote, or why it wrote nothing.
func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string, in Inputs, bus []string, lost []string, composed []User) {
func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut, dropped []string, in Inputs, bus []string, lost []string, composed []User) {
cur, _, had := p.Sealed()
next := cur
next.Follow = followOf(p.SwitchState())
@@ -93,10 +110,10 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string
why = "no node runs the bus's module, so the list is sealed to nobody"
case key == "":
why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)"
case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, bus, lost, digest) != "":
case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, digest) != "":
// After take-over the list in force is never replaced by one composed past a failed read, or one that drops a user
// of it for want of a hash: that user's module would be locked out of the bus (review of the switch).
why = p.holdsBack(in, leftOut, bus, lost, digest)
why = p.holdsBack(in, leftOut, dropped, bus, lost, digest)
held = why
p.keepHashHold(res, digest)
case cur.Digest != digest || cur.SealedTo != delivery.SealingKeyID(key) || cur.Node != in.BusNode:
@@ -356,7 +373,7 @@ func listUsers(list []byte) []string {
// loaded list not read holds it too, since that cannot be ruled out). The list in force is the one the part wrote, by the users
// it kept of it; where it kept none (a list written before this release), the users the bus loaded (bus, nil when
// they could not be read). Empty when it may.
func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string, digest string) string {
func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest string) string {
p.hashHold = nil
if f := in.ReadFailures(); len(f) > 0 {
return "held: the list in force is kept while a read of this pass failed (" + strings.Join(f, "; ") + ")"
@@ -387,6 +404,17 @@ func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string, digest string)
if len(unhashed) > 0 {
return "held: the list in force is kept, since " + strings.Join(unhashed, ", ") + " would be left out for want of a hash in the vault"
}
// A refused principal the list in force carries and whose grants there the part did not keep: leaving it out would lock
// it out of the bus (refused.go), so the list in force stands until its grants compose or are kept.
var unkept []string
for _, u := range dropped {
if slices.Contains(had, u) {
unkept = append(unkept, u)
}
}
if len(unkept) > 0 {
return "held: the list in force is kept, since " + strings.Join(unkept, ", ") + " would be left out: its grants are refused and the part kept none of the list in force"
}
// More than one loaded user's hash changing at once, with no rotation by the vault naming each (ADR 0561 §7): a new
// writer of hashes must not lock several holders out in one list, as nine seats were cut on 2026-10-27 at 09:21.
changed, err := p.changedHashes(in.Hashes)
@@ -116,23 +116,18 @@ func CredentialsOf(shadow, loaded []User, hashes map[string]HeldHash) Credential
return c
}
// ComposeUsers is the shadow list: every user the composer derives from the records, with its grants. Pure.
// ComposeUsers is the shadow list: every user the composer derives from the records, with its grants; an error when any
// principal's grants are refused (ComposeEach composes past one). Pure.
func ComposeUsers(r buscompose.Records) ([]User, error) {
principals, err := buscompose.Users(r)
users, refused, err := ComposeEach(r)
if err != nil {
return nil, err
}
out := make([]User, 0, len(principals))
for _, p := range principals {
perms, err := buscompose.PermissionsFor(p)
if err != nil {
return nil, fmt.Errorf("%s: %w", p.Username(), err)
}
out = append(out, User{User: p.Username(), Publish: set(perms.Publish), Subscribe: set(perms.Subscribe),
AllowResponses: perms.AllowResponses, PublishDeny: set(perms.PublishDeny)})
if len(refused) > 0 {
first := sortedKeys(refused)[0]
return nil, fmt.Errorf("%s: %s", first, refused[first])
}
sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User })
return out, nil
return users, nil
}
func set(in []string) []string {