Merge pull request 'The grants part writes its list whole however large, one refused principal no longer freezes it, and stale watchman keys close (issue 774)' (#117) from fix/sweep-2026-10-30 into main

Reviewed-on: #117
This commit was merged in pull request #117.
This commit is contained in:
2026-10-30 18:13:00 +00:00
33 changed files with 753 additions and 88 deletions
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -151,6 +151,31 @@ func IsUntrunked(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && v.Failed.Machine == "checker" && strings.HasPrefix(v.Failed.Words, UntrunkedWords) return v.State == delivery.VerdictFail && v.Failed.Machine == "checker" && strings.HasPrefix(v.Failed.Words, UntrunkedWords)
} }
// **A candidate outside the mesh is not the checker's** (sweep 2026-10-30 item 7): the forge carries repositories the mesh
// does not deliver (an operator's own), whose pull requests the indexer writes as candidates like any. Their trunk the
// gate never names, so each was judged "cannot be judged" and owed the status error on mesh/check, which the forge
// module wrote 119 times per head into a repository the mesh's forge account may not write (403). A candidate every pull
// request of which is of a repository the gate's source.main names no trunk head of, and touches no module, is outside
// the mesh: judged nothing, owed nothing on the forge, said once in the log and in checker_status (outside_the_mesh). A
// new repository of the mesh is not outside: its first pull request adds a module, and is said untrunked (issue 1422).
// OutsideWords say why a candidate is outside the mesh.
const OutsideWords = "outside the mesh: the gate's source.main names no trunk head of its repositories and it touches no module, so the checker judges nothing and owes no status on the forge"
// Outside says every pull request of c is of a repository trunk names no head of and touches no module. Never with
// trunk unknown (nil): before the gate's source is read nothing is outside.
func Outside(c delivery.Candidate, trunk map[string]string) bool {
if trunk == nil || len(c.Pulls) == 0 {
return false
}
for _, p := range c.Pulls {
if trunk[p.Repository] != "" || len(p.Modules) > 0 || len(p.Fps) > 0 {
return false
}
}
return true
}
// leftOf are the pull requests of a candidate whose repository has no trunk head known, with Trunk empty, in repository and // leftOf are the pull requests of a candidate whose repository has no trunk head known, with Trunk empty, in repository and
// number order; nil when every repository's is known. // number order; nil when every repository's is known.
func leftOf(c delivery.Candidate, trunk map[string]string) []Pull { func leftOf(c delivery.Candidate, trunk map[string]string) []Pull {
@@ -104,6 +104,8 @@ type Checker struct {
held map[string]string // by verdict key: the turn that holds its check off, as last said held map[string]string // by verdict key: the turn that holds its check off, as last said
live map[string]liveCheck // by verdict key: the check running in this process, ended when its heads go live map[string]liveCheck // by verdict key: the check running in this process, ended when its heads go
settling map[string]*window // by candidates key: the window its heads settle in settling map[string]*window // by candidates key: the window its heads settle in
// outsideSaid are the candidates outside the mesh said in the log, by group and heads.
outsideSaid map[string]bool
} }
// liveCheck is a check running in this process: the candidates key of its group and the end of its run. // liveCheck is a check running in this process: the candidates key of its group and the end of its run.
@@ -160,6 +162,7 @@ func (ch *Checker) init() {
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{} ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{} ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{}
ch.live, ch.settling = map[string]liveCheck{}, map[string]*window{} ch.live, ch.settling = map[string]liveCheck{}, map[string]*window{}
ch.outsideSaid = map[string]bool{}
}) })
} }
@@ -327,6 +330,10 @@ func (ch *Checker) heads() map[string]string {
func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidate) { func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidate) {
heads := ch.heads() heads := ch.heads()
if Outside(c, heads) {
ch.outside(ctx, c)
return
}
if left := leftOf(c, heads); len(left) > 0 { if left := leftOf(c, heads); len(left) > 0 {
// A pull request the checker cannot claim (its repository's trunk head is not known) leaves its whole group // A pull request the checker cannot claim (its repository's trunk head is not known) leaves its whole group
// unjudged, said in a failure verdict (issue 1422, issue 1733): the gate asks the Approve only on a verdict, and a // unjudged, said in a failure verdict (issue 1422, issue 1733): the gate asks the Approve only on a verdict, and a
@@ -464,6 +471,39 @@ func (ch *Checker) untrunked(ctx context.Context, c delivery.Candidate, left []P
} }
} }
// outside takes a candidate outside the mesh (case.go Outside): no verdict, and every status owed for its pull requests
// taken away, so the forge module stops writing one into a repository the mesh's forge account may not write. Said once
// per heads in the log.
func (ch *Checker) outside(ctx context.Context, c delivery.Candidate) {
var names []string
for _, p := range c.Pulls {
names = append(names, fmt.Sprintf("%s#%d@%s", p.Repository, p.Number, short12(p.Head.Head)))
key, err := delivery.CheckerOwedKey(p.Repository, p.Number, p.Head.Head)
if err != nil {
continue
}
_, _, found, err := ch.Owed.Get(ctx, key)
if err != nil || !found {
continue
}
d, ok := ch.Owed.(Deleter)
if !ok {
continue
}
if err := d.Delete(ctx, key); err != nil {
ch.logf("%s: the status owed of a pull request outside the mesh could not be taken away: %v", key, err)
}
}
said := c.Group + "@" + c.HeadsHash
ch.mu.Lock()
first := !ch.outsideSaid[said]
ch.outsideSaid[said] = true
ch.mu.Unlock()
if first {
ch.logf("candidates of %s: %s %s", c.Group, strings.Join(names, ", "), OutsideWords)
}
}
// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on // The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on
// the build seats' check stream; a carry, a verdict moved to a newer trunk or sibling commit by its package digests (ADR 0548 // the build seats' check stream; a carry, a verdict moved to a newer trunk or sibling commit by its package digests (ADR 0548
// §5), on the recheck stream, pulled after. // §5), on the recheck stream, pulled after.
@@ -48,6 +48,13 @@ func pullOf(repo string, number int, head string) delivery.CandidatePull {
return delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: number, Head: head}, Modules: []string{}} return delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: number, Head: head}, Modules: []string{}}
} }
// newRepoPullOf is a pull request of a new repository of the mesh: its trunk not named yet, its first module added.
func newRepoPullOf(repo string, number int, head string) delivery.CandidatePull {
p := pullOf(repo, number, head)
p.Modules = []string{"mod-u"}
return p
}
// judgeFunc is a Judge written as a function; it counts its calls. // judgeFunc is a Judge written as a function; it counts its calls.
type judgeFunc struct { type judgeFunc struct {
mu sync.Mutex mu sync.Mutex
@@ -249,7 +256,7 @@ func TestARepositoryWithNoTrunkCannotBeJudgedLoudlyUntilItsTrunkIsKnown(t *testi
var logged []string var logged []string
r.ch.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } r.ch.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
const unknown = "novox/unknown" const unknown = "novox/unknown"
cand := candidateOf(t, "feat/y", pullOf(unknown, 4, head1)) cand := candidateOf(t, "feat/y", newRepoPullOf(unknown, 4, head1))
key := keyOf(t, "feat/y", cand) key := keyOf(t, "feat/y", cand)
r.ch.Candidate(ctx, "feat/y", cand) r.ch.Candidate(ctx, "feat/y", cand)
r.ch.Wait() r.ch.Wait()
@@ -305,7 +312,7 @@ func TestARepositoryWithNoTrunkCannotBeJudgedLoudlyUntilItsTrunkIsKnown(t *testi
want string // the verdict's state, and for a failure the repository it names want string // the verdict's state, and for a failure the repository it names
}{ }{
{name: "a member the checker cannot claim", pulls: []delivery.CandidatePull{pullOf(repoA, 9, head2), pullOf("novox/other", 2, head2)}, want: "novox/other"}, {name: "a member the checker cannot claim", pulls: []delivery.CandidatePull{pullOf(repoA, 9, head2), pullOf("novox/other", 2, head2)}, want: "novox/other"},
{name: "a running verdict left for heads that cannot be judged", pulls: []delivery.CandidatePull{pullOf(unknown, 5, head2)}, running: true, want: unknown}, {name: "a running verdict left for heads that cannot be judged", pulls: []delivery.CandidatePull{newRepoPullOf(unknown, 5, head2)}, running: true, want: unknown},
{name: "a verdict key deleted", pulls: []delivery.CandidatePull{pullOf(repoA, 5, head2)}, deleted: true, judged: 2, want: delivery.VerdictPass}, {name: "a verdict key deleted", pulls: []delivery.CandidatePull{pullOf(repoA, 5, head2)}, deleted: true, judged: 2, want: delivery.VerdictPass},
} { } {
t.Run(row.name, func(t *testing.T) { t.Run(row.name, func(t *testing.T) {
@@ -566,3 +573,43 @@ func TestTheForgeBridgeIsToldOnceForEachChangeOfAStatus(t *testing.T) { // 1683
t.Errorf("told %v after a check that changed nothing", told) t.Errorf("told %v after a check that changed nothing", told)
} }
} }
// Sweep 2026-10-30 item 7: a pull request of a repository the mesh does not deliver (an operator's own, on the same
// forge) touches no module and its trunk the gate never names. It is outside the mesh: no verdict, no status owed (one
// owed before is taken away, so the forge module stops writing it into a repository the mesh may not write), said once.
func TestACandidateOutsideTheMeshIsJudgedNothingAndOwesNothing(t *testing.T) {
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
var logged []string
r.ch.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
const theirs = "someone/own-app"
okey, _ := delivery.CheckerOwedKey(theirs, 55, head1)
before, _ := delivery.EncodeCheckerOwed(delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "error", Text: "cannot be judged"})
if _, err := r.owed.Create(ctx, okey, before); err != nil {
t.Fatal(err)
}
cand := candidateOf(t, "feat/theirs", pullOf(theirs, 55, head1))
key := keyOf(t, "feat/theirs", cand)
r.ch.Candidate(ctx, "feat/theirs", cand)
r.ch.Wait()
r.ch.Reconsider(ctx)
r.ch.Wait()
if _, _, found, _ := r.verdicts.Get(ctx, key); found || j.count() != 0 {
t.Errorf("a candidate outside the mesh was judged (%d) or given a verdict (%v)", j.count(), found)
}
if s := owedState(t, r, theirs, 55, head1); s != "" {
t.Errorf("%s#55 still owes %q on the forge", theirs, s)
}
if n := countWith(logged, "outside the mesh"); n != 1 {
t.Errorf("said %d times in the log, want once: %q", n, logged)
}
ckey, _ := delivery.CandidatesKey("feat/theirs")
st, err := Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState(r.verdicts.vals).read(),
Trunks: func() (map[string]string, error) { return r.trunks.Heads(), nil }}.Status()
if err != nil {
t.Fatal(err)
}
if st.Outside["feat/theirs"] == "" || len(st.CannotBeJudged) != 0 || len(st.Unjudged) != 0 {
t.Errorf("status: outside %v, cannot be judged %v, unjudged %v", st.Outside, st.CannotBeJudged, st.Unjudged)
}
}
@@ -335,6 +335,8 @@ type Status struct {
// CannotBeJudged are the candidates no pull request of which can be judged, by group, with why: the gate's source.main // CannotBeJudged are the candidates no pull request of which can be judged, by group, with why: the gate's source.main
// names no trunk head of their repositories (issue 1422). They are judged once it does; until then their status is error. // names no trunk head of their repositories (issue 1422). They are judged once it does; until then their status is error.
CannotBeJudged map[string]string `json:"cannot_be_judged"` CannotBeJudged map[string]string `json:"cannot_be_judged"`
// Outside are the candidates outside the mesh, by group, with why (case.go Outside): judged nothing, owed nothing.
Outside map[string]string `json:"outside_the_mesh"`
// Lost are the candidates whose verdict is running while no step of theirs can still run and their newest step ended, by // Lost are the candidates whose verdict is running while no step of theirs can still run and their newest step ended, by
// group, with that step: nothing will judge them (the lost check of 2026-10-28); checker_ask_again asks one again. // group, with that step: nothing will judge them (the lost check of 2026-10-28); checker_ask_again asks one again.
Lost map[string]string `json:"lost"` Lost map[string]string `json:"lost"`
@@ -415,7 +417,7 @@ func (r Reader) Status() (Status, error) {
} }
st.Unjudged = []string{} st.Unjudged = []string{}
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}} st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
st.HeldByTurn, st.CannotBeJudged = map[string]string{}, map[string]string{} st.HeldByTurn, st.CannotBeJudged, st.Outside = map[string]string{}, map[string]string{}, map[string]string{}
turns := r.turns(&st) turns := r.turns(&st)
var heads map[string]string var heads map[string]string
if r.Trunks == nil { if r.Trunks == nil {
@@ -466,6 +468,10 @@ func (r Reader) Status() (Status, error) {
continue continue
} }
c, raw := row.c, row.raw c, raw := row.c, row.raw
if Outside(c, heads) {
st.Outside[c.Group] = OutsideWords
continue
}
if heads != nil && len(leftOf(c, heads)) > 0 { if heads != nil && len(leftOf(c, heads)) > 0 {
// Nothing of it can be judged now, whatever its verdict says (issue 1422): never counted as waiting. // Nothing of it can be judged now, whatever its verdict says (issue 1422): never counted as waiting.
var repos []string var repos []string
@@ -17,3 +17,8 @@ type KV interface {
// Keys are every key of the bucket. // Keys are every key of the bucket.
Keys(ctx context.Context) ([]string, error) Keys(ctx context.Context) ([]string, error)
} }
// Deleter is a KV that can take a key away; a KV that cannot leaves it.
type Deleter interface {
Delete(ctx context.Context, key string) error
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-teller
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-view
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -244,6 +244,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
process.NotWatched(logger.Printf, delivery.AssignerWantsBucket, err, "moves are not watched until the watchman starts again") process.NotWatched(logger.Printf, delivery.AssignerWantsBucket, err, "moves are not watched until the watchman starts again")
haveSeen.Stop() haveSeen.Stop()
haveSeen = nil haveSeen = nil
} else {
w.ReadWhole(delivery.AssignerWantsBucket)
} }
} }
// The build seats' steps: a step key `<node>.<ask id>` written is the ask taken (issue 1887 A8b). A seat whose build module // The build seats' steps: a step key `<node>.<ask id>` written is the ask taken (issue 1887 A8b). A seat whose build module
@@ -263,6 +265,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
continue continue
} }
taken[lang] = sw taken[lang] = sw
if bucket, err := delivery.AskStream(lang); err == nil {
w.ReadWhole(bucket)
}
} }
// The events first, then the one request of the open connections, so no connection falls between them. // The events first, then the one request of the open connections, so no connection falls between them.
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-watchman
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -40,6 +40,7 @@ func (w *Watchman) Step(language, key string, value []byte, op Op) {
w.moved(bucket, id) w.moved(bucket, id)
return return
} }
w.stands(bucket, id)
step, err := delivery.DecodeStep(value) step, err := delivery.DecodeStep(value)
if err != nil { if err != nil {
w.say("the step %s cannot be read, and is left: %v", key, err) w.say("the step %s cannot be read, and is left: %v", key, err)
@@ -80,6 +81,7 @@ func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) {
w.moved(delivery.AssignerWantsBucket, key) w.moved(delivery.AssignerWantsBucket, key)
return return
} }
w.stands(delivery.AssignerWantsBucket, key)
want, err := delivery.DecodeWant(value) want, err := delivery.DecodeWant(value)
if err != nil { if err != nil {
w.say("the want %s cannot be read, and is left: %v", key, err) w.say("the want %s cannot be read, and is left: %v", key, err)
@@ -91,6 +91,11 @@ type Watchman struct {
tunnels map[string]string tunnels map[string]string
// stepBound is how long a build step may run once a node took the ask; a test shortens it. // stepBound is how long a build step may run once a node took the ask; a test shortens it.
stepBound time.Duration stepBound time.Duration
// standing is, until the start ends (Reconcile), each mirrored key read standing, by the bucket a stuck key names it
// under; readWhole the buckets whose keys were all read at this start (ReadWhole). reconciled is the start's end.
standing map[string]map[string]bool
readWhole map[string]bool
reconciled bool
} }
// New is a watchman writing to out, reading the time from now. // New is a watchman writing to out, reading the time from now.
@@ -100,7 +105,8 @@ func New(out Out, now func() time.Time) *Watchman {
findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, away: map[string]bool{}, findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, away: map[string]bool{},
haveState: map[string]delivery.Have{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, haveState: map[string]delivery.Have{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{},
control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{},
health: map[string]delivery.NodeHealth{}, tunnels: map[string]string{}} health: map[string]delivery.NodeHealth{}, tunnels: map[string]string{},
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}}
} }
// Logf sets where a write that failed is said; it is silent until set. // Logf sets where a write that failed is said; it is silent until set.
@@ -257,7 +263,8 @@ func (w *Watchman) SeedPulse(key string) {
} }
// Reconcile ends the start: a pulse that stood and that no mirrored key has made again is a move that happened while // Reconcile ends the start: a pulse that stood and that no mirrored key has made again is a move that happened while
// the watchman was away, so it is deleted and writes no stuck key. // the watchman was away, so it is deleted and writes no stuck key; and a stuck key whose watched key is gone is closed
// (closeGone).
func (w *Watchman) Reconcile() { func (w *Watchman) Reconcile() {
w.mu.Lock() w.mu.Lock()
defer w.mu.Unlock() defer w.mu.Unlock()
@@ -266,9 +273,49 @@ func (w *Watchman) Reconcile() {
w.deletePulse(pk) w.deletePulse(pk)
} }
} }
w.closeGone()
w.reconcileFindings() w.reconcileFindings()
} }
// **A stuck key whose watched key is gone is closed** (issue 774, sweep 2026-10-30): a key that ages out of its bucket
// (a stream's maximum age, 168 h for the build seats' asks) leaves no marker a watch hears, and one deleted while the
// watchman was down is not heard either, so its stuck key stood unmoved for ever: 26 stuck-key conditions of asks from
// 2026-10-21 were open on 10-30. At the start's end, every stuck key not moved whose bucket was read whole at this start
// (ReadWhole) and whose key was not read standing is written moved, which clears its finding, and said. A bucket that
// could not be read closes nothing: its keys are not known to be gone.
// ReadWhole says every key of bucket (as a stuck key names it: an ask stream for a build seat's steps, the assigner's
// wants) was read at this start; called once its watch holds what stands.
func (w *Watchman) ReadWhole(bucket string) {
w.mu.Lock()
defer w.mu.Unlock()
w.readWhole[bucket] = true
}
// stands records a mirrored key read standing during the start.
func (w *Watchman) stands(bucket, key string) {
if w.reconciled {
return
}
if w.standing[bucket] == nil {
w.standing[bucket] = map[string]bool{}
}
w.standing[bucket][key] = true
}
func (w *Watchman) closeGone() {
for sk, s := range w.stuck {
if s.Moved || !w.readWhole[s.Bucket] || w.standing[s.Bucket][s.Key] {
continue
}
s.Moved = true
w.say("the stuck key %s is closed: its key %s is no longer in %s (it aged out of the bucket or was deleted while nobody heard), so nothing waits for it since %s",
sk, s.Key, s.Bucket, s.Since.UTC().Format(time.RFC3339))
w.putStuck(sk, s)
}
w.standing, w.readWhole, w.reconciled = map[string]map[string]bool{}, map[string]bool{}, true
}
// reconcileFindings ends the start for the findings: one read at the start that what the watchman read since did not // reconcileFindings ends the start for the findings: one read at the start that what the watchman read since did not
// observe again is deleted, unless a signal still standing keeps it (a provider's failing word, an advisory's quiet hour, a // observe again is deleted, unless a signal still standing keeps it (a provider's failing word, an advisory's quiet hour, a
// machine's silent tools, a silent self-check): those end by their own events or expiry. // machine's silent tools, a silent self-check): those end by their own events or expiry.
@@ -656,3 +656,43 @@ func TestAWantNotAppliedByItsBoundIsStuckAndAKeyDeletedEndsIt(t *testing.T) {
t.Errorf("a deleted want left its pulse: %v", o2.pulses) t.Errorf("a deleted want left its pulse: %v", o2.pulses)
} }
} }
// Sweep 2026-10-30: a watched key that aged out of its bucket or was deleted while nobody heard leaves its stuck key; at the
// start's end the watchman closes it (moved, its finding cleared), but only for a bucket it read whole at that start.
func TestAStuckKeyWhoseKeyIsGoneIsClosedAtTheStart(t *testing.T) {
wantKey := "want.n1.mod-a"
for _, row := range []struct {
name string
bucket string
key string
standing func(w *Watchman) // what the start reads standing
readWhole bool
closed bool
}{
{"an ask's step aged out of the steps bucket", checksKey, "askold1", func(*Watchman) {}, true, true},
{"an ask's step still running", checksKey, "askold1", func(w *Watchman) { w.Step("go", "n1.askold1", runningStep("askold1", t0), Put) }, true, false},
{"the steps bucket could not be read", checksKey, "askold1", func(*Watchman) {}, false, false},
{"a want deleted while nobody heard", delivery.AssignerWantsBucket, wantKey, func(*Watchman) {}, true, true},
{"a want still standing", delivery.AssignerWantsBucket, wantKey, func(w *Watchman) { w.Want(wantKey, wantValue(t, fpNew, false), Put, t0) }, true, false},
} {
t.Run(row.name, func(t *testing.T) {
w, o, c := setup()
c.t = t0.Add(9 * 24 * time.Hour)
sk := stuckKeyOf(t, row.bucket, row.key)
v, _ := delivery.EncodeStuck(delivery.Stuck{Bucket: row.bucket, Key: row.key, Waited: "a move", Since: t0})
w.SeedStuck(sk, v)
if _, ok := o.findings[FindingKey(ScopeDelivery, sk, KindStuck)]; !ok {
t.Fatal("the seeded stuck key raised no finding")
}
row.standing(w)
if row.readWhole {
w.ReadWhole(row.bucket)
}
w.Reconcile()
_, open := o.findings[FindingKey(ScopeDelivery, sk, KindStuck)]
if got := o.stuck[sk].Moved; got != row.closed || open == row.closed {
t.Errorf("moved %v, finding open %v; want closed %v", got, open, row.closed)
}
})
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-grants
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030154046-af66ccb8335a git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
) )
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030154046-af66ccb8335a h1:8A43iLD5YlqwyeKmGvAgTvf2b5W/FDi9VDkqvPIh5MU= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030154046-af66ccb8335a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+171 -2
View File
@@ -67,6 +67,8 @@ type fakeBus struct {
// consumerCreates each consumer the part asked to create, `<stream>.<name>`. // consumerCreates each consumer the part asked to create, `<stream>.<name>`.
consumers map[string][]map[string]any consumers map[string][]map[string]any
consumerCreates []string consumerCreates []string
// maxValue, when set, refuses an own value larger than it as the bus does (JetStream 10054).
maxValue int
} }
// MatchConsumer is the bus's update of a held durable consumer to the configuration given. // MatchConsumer is the bus's update of a held durable consumer to the configuration given.
@@ -260,6 +262,9 @@ func (s ownStore) Put(state, key string, value []byte) (uint64, error) {
if b.refuseTo[state] { if b.refuseTo[state] {
return 0, errors.New("Permissions Violation for Publish to $KV.mesh-grants_" + state) return 0, errors.New("Permissions Violation for Publish to $KV.mesh-grants_" + state)
} }
if b.maxValue > 0 && len(value) > b.maxValue {
return 0, errors.New("nats: API error: code=400 err_code=10054 description=message size exceeds maximum allowed")
}
b.writes = append(b.writes, state) b.writes = append(b.writes, state)
if b.own[state] == nil { if b.own[state] == nil {
b.own[state] = map[string][]byte{} b.own[state] = map[string][]byte{}
@@ -1542,7 +1547,7 @@ func TestGrants(t *testing.T) {
fp := partOn(b) fp := partOn(b)
fp.Objects = p.Objects fp.Objects = p.Objects
fp.Asker, fp.Say = fa, func(c asks.Condition) error { fp.Asker, fp.Say = fa, func(c asks.Condition) error {
if c.Key != grants.GrantMissingCondition { // its own row below if c.Key != grants.GrantMissingCondition && c.Key != grants.RefusedCondition { // their own rows
fresh = append(fresh, c) fresh = append(fresh, c)
} }
return nil return nil
@@ -2188,6 +2193,8 @@ func TestGrants(t *testing.T) {
return nil return nil
} }
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"}) b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"})
// The private network the mesh provides is placed with no fingerprint by construction: never a missing grant.
b.assign("b", "mesh-wireguard")
fp := "src2:d4afe7a1abab5ccdbef6d8629b2245006a2aec60290f430f7b4756e2930ed60c" fp := "src2:d4afe7a1abab5ccdbef6d8629b2245006a2aec60290f430f7b4756e2930ed60c"
b.put(grants.ReadModules, "mod-x", delivery.Module{V: delivery.Version, FP2: fp, Manifest: delivery.ManifestDigest([]byte(xManifest)), b.put(grants.ReadModules, "mod-x", delivery.Module{V: delivery.Version, FP2: fp, Manifest: delivery.ManifestDigest([]byte(xManifest)),
Repository: "novox/Link2Pay", Commit: "5d20e8bfbc953c51e8ebe3c671fe883df9c36e15"}) Repository: "novox/Link2Pay", Commit: "5d20e8bfbc953c51e8ebe3c671fe883df9c36e15"})
@@ -2440,7 +2447,7 @@ func TestGrants(t *testing.T) {
// the composed configuration only by hand (grants_consumer_match), never while the controller holds the lease. // the composed configuration only by hand (grants_consumer_match), never while the controller holds the lease.
var said []asks.Condition var said []asks.Condition
p.Say = func(c asks.Condition) error { p.Say = func(c asks.Condition) error {
if c.Key != grants.GrantMissingCondition { // its own row if c.Key != grants.GrantMissingCondition && c.Key != grants.RefusedCondition { // their own rows
said = append(said, c) said = append(said, c)
} }
return nil return nil
@@ -2565,6 +2572,168 @@ func TestGrants(t *testing.T) {
t.Errorf("changes the vault's rotation names were held: %+v", said) t.Errorf("changes the vault's rotation names were held: %+v", said)
} }
}) })
// Sweep 2026-10-30 item 5: the permissions of the sealed lists were one value holding every user of two lists, which
// outgrew the bucket's largest value, so they were never kept. One key per user: no value grows with the list.
t.Run("sealed grants: one key per user, so no value grows with the list, and only the last two lists kept", func(t *testing.T) {
b := fixture()
for _, u := range shadowOf(t, b) {
b.put(grants.ReadAccounts, u.User, map[string]any{"user": u.User, "hash": "$2a$10$first", "generation": 1, "fingerprint": "sha256:1"})
}
largest := 0
for _, u := range shadowOf(t, b) {
raw, _ := json.Marshal(u)
largest = max(largest, len(raw))
}
b.maxValue = largest + 256 // one user's permissions fit, every user's of a list do not
busKey, err := seal.GenerateKeyPair()
if err != nil {
t.Fatal(err)
}
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}})
p := partOn(b)
p.Objects = &fakeObjects{}
var digests []string
for gen := 1; gen <= 3; gen++ {
b.put(grants.ReadAccounts, "a.mod-x", map[string]any{"user": "a.mod-x", "hash": fmt.Sprintf("$2a$10$gen%d", gen), "generation": gen, "fingerprint": fmt.Sprintf("sha256:%d", gen)})
res := p.Compose()
for _, e := range res.Errors {
if strings.Contains(e, "10054") {
t.Fatalf("a sealed-grants write exceeded the bucket's largest value: %s", e)
}
}
g, _, _ := p.Sealed()
digests = append(digests, g.Digest)
}
var idx grants.SealedLists
raw, _, found, _ := b.ownGet(grants.StateUserList, grants.KeySealedLists)
if !found || json.Unmarshal(raw, &idx) != nil || len(idx.Lists) != 2 || idx.Lists[0].Digest != digests[2] || idx.Lists[1].Digest != digests[1] {
t.Fatalf("the index of kept lists = %s, want the last two of %v", raw, digests)
}
for i, d := range digests {
_, _, kept, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(d, "a.mod-x"))
if kept != (i > 0) {
t.Errorf("list %d's a.mod-x key kept = %v", i, kept)
}
}
for _, u := range idx.Lists[0].Users {
if _, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(digests[2], u)); !ok {
t.Errorf("the index names %s, whose key is not there", u)
}
}
// The loaded list's permissions are read back from the keys when the bus module's tools cannot say them.
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: delivery.GrantsUserListSource, Follows: delivery.FollowGrants, Revision: 8,
Digest: digests[2], Users: 6, Key: busKey.PublicKey}})
b.refuseUsers = true
if res := p.Compose(); slices.ContainsFunc(res.Grants.Waits, func(w string) bool { return strings.Contains(w, "unknown") }) {
t.Errorf("the kept permissions of the loaded list were not read: %v", res.Grants.Waits)
}
})
// Sweep 2026-10-30 item 5, a kind met twice that day (the engine's acts, then mesh-host_health): one principal the
// composer refuses as a second writer froze the whole list. Now it keeps its grants of the list in force, the rest is
// written, and the refusal is raised by name; one the list in force does not carry stays out, said too.
t.Run("a refused principal keeps its grants of the list in force; the rest of the list is written", func(t *testing.T) {
b := fixture()
const ctl = `{"module":"mesh-controller","version":"1","own-secrets":{"broker":"/var/lib/c/broker"}}`
refusedCtl := strings.Replace(ctl, `}}`, `},"state":[{"name":"declared-config"}]}`, 1)
b.manifestOf("mesh-controller", ctl)
b.assign("b", "mesh-controller")
hash := func(user, h string, gen int) {
b.put(grants.ReadAccounts, user, map[string]any{"user": user, "hash": h, "generation": gen, "fingerprint": fmt.Sprintf("sha256:%d", gen)})
}
for _, u := range shadowOf(t, b) {
hash(u.User, "$2a$10$first", 1)
}
hash("a.mesh-controller", "$2a$10$first", 1)
busKey, err := seal.GenerateKeyPair()
if err != nil {
t.Fatal(err)
}
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}})
p := partOn(b)
objs := &fakeObjects{}
p.Objects = objs
entryOf := func(g delivery.GrantsUserList, user string) string {
t.Helper()
list, err := delivery.OpenUserList(objs.objects[g.Object], g.Digest, func(bx seal.Box) ([]byte, error) {
out, err := seal.Open(bx, busKey.PrivateKey)
return []byte(out), err
})
if err != nil {
t.Fatalf("the sealed list does not open: %v", err)
}
at := strings.Index(string(list), fmt.Sprintf("{ user: %q,", user))
if at < 0 {
return ""
}
rest := string(list)[at:]
return rest[:strings.Index(rest, "} }\n")+4]
}
var said []asks.Condition
p.Say = func(c asks.Condition) error {
if c.Key == grants.RefusedCondition {
said = append(said, c)
}
return nil
}
sw, _ := json.Marshal(grants.Switch{V: 1, State: grants.TookOver, Acts: []grants.Act{}})
if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil {
t.Fatal(err)
}
p.Compose()
inForce, _, _ := p.Sealed()
if inForce.Digest == "" {
t.Fatal("no list written after take-over")
}
entryBefore := entryOf(inForce, "b.mesh-controller")
ctlKey := grants.SealedUserKey(inForce.Digest, "b.mesh-controller")
before, _, ok, _ := b.ownGet(grants.StateUserList, ctlKey)
if !ok {
t.Fatal("the list in force keeps no grants of b.mesh-controller")
}
// b.mesh-controller now declares the controller's mirror as its own state: refused as a second writer. a.mod-x's hash
// changes in the same pass, and a.mesh-controller, refused the same way, is placed new.
b.manifestOf("mesh-controller", refusedCtl)
b.assign("a", "mesh-controller")
hash("a.mod-x", "$2a$10$second", 2)
res := p.Compose()
now, _, _ := p.Sealed()
if now.Digest == inForce.Digest || slices.ContainsFunc(res.Errors, func(e string) bool { return strings.Contains(e, "composed whole") }) {
t.Fatalf("a refused principal froze the list: sealed %+v, errors %v", res.Sealed, res.Errors)
}
after, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(now.Digest, "b.mesh-controller"))
if !ok || string(after) != string(before) {
t.Errorf("b.mesh-controller did not keep its grants of the list in force: %s, was %s", after, before)
}
// The kept user is written as the composer writes one: the bus reads the same entry it loaded.
if got := entryOf(now, "b.mesh-controller"); entryBefore == "" || got != entryBefore {
t.Errorf("the kept entry is written otherwise than the list in force holds it:\n%s\nwas\n%s", got, entryBefore)
}
if entryOf(now, "a.mesh-controller") != "" {
t.Error("a refused principal the list in force did not carry is in the sealed list")
}
if _, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(now.Digest, "a.mesh-controller")); ok {
t.Error("a refused principal the list in force did not carry is in the list")
}
if len(res.PrincipalsRefused) != 2 || len(said) == 0 || said[len(said)-1].State != asks.StateRaised ||
!strings.Contains(said[len(said)-1].Summary, "b.mesh-controller (keeps its grants of the list in force)") ||
!strings.Contains(said[len(said)-1].Summary, "a.mesh-controller (left out of the list") {
t.Fatalf("the refusals were not said by name: %v, said %+v", res.PrincipalsRefused, said)
}
// Composing again changes nothing and says nothing new; once the manifest composes, the warning is cleared.
n := len(said)
if p.Compose(); len(said) != n {
t.Errorf("an unchanged refusal was said again: %+v", said[n:])
}
b.manifestOf("mesh-controller", ctl)
if res := p.Compose(); len(res.PrincipalsRefused) != 0 || said[len(said)-1].State != asks.StateCleared {
t.Errorf("the refusal was not cleared once it composes: %v, said %+v", res.PrincipalsRefused, said)
}
})
} }
type fakeAsker struct { type fakeAsker struct {
+130 -23
View File
@@ -82,12 +82,26 @@ func GrantWaits(tookOver bool, accounts []string, composed, loaded map[string]Us
return "waits for the bus to load a list carrying what it adds: " + strings.Join(adds, ", ") return "waits for the bus to load a list carrying what it adds: " + strings.Join(adds, ", ")
} }
// KeySealedGrants is the key of the state user-list holding, per list the part sealed (the last two, by digest), each // **The permissions of each list the part sealed, one key per user** (issue 774, sweep 2026-10-30 item 5): the last two
// user's permissions as sealed, never a hash: what tells which grants the list the bus loaded carries, without the bus // lists, by digest, each user's permissions as sealed, never a hash: what tells which grants the list the bus loaded
// module's tools (issue 774). // carries, without the bus module's tools. One value holding every user of two lists outgrew the bucket's largest value
// (JetStream 10054, "message size exceeds maximum", 16:37 and 16:55 on 2026-10-30), so a list's permissions were never
// kept and a grant judged against them waited. Each user's permissions are a key of their own,
// `sealed-grants.<digest hex>.<user>`, which grows with that user alone; the key `sealed-lists` names the lists kept and
// their users, and is written only once every user's key of its newest list is: it never names a list whose keys are not
// all there.
// KeySealedGrants is the key that held every user of the last two lists in one value, before this release: read when the
// index names no list, and taken away once the index is written.
const KeySealedGrants = "sealed-grants" const KeySealedGrants = "sealed-grants"
// SealedGrants is the value of KeySealedGrants. // KeySealedLists is the key of the state user-list naming the sealed lists whose permissions are kept, newest first.
const KeySealedLists = "sealed-lists"
// sealedKept is how many sealed lists keep their permissions: the one in force and the one before.
const sealedKept = 2
// SealedGrants is the value KeySealedGrants held.
type SealedGrants struct { type SealedGrants struct {
Lists []SealedList `json:"lists"` Lists []SealedList `json:"lists"`
} }
@@ -98,7 +112,24 @@ type SealedList struct {
Users []User `json:"users"` Users []User `json:"users"`
} }
// keepSealedGrants records the permissions of the list sealed now, beside the one before. // SealedLists is the value of KeySealedLists: the lists kept, newest first, each by its digest and its users' names.
type SealedLists struct {
Lists []SealedNames `json:"lists"`
}
// SealedNames is one kept list: its digest and its users, sorted.
type SealedNames struct {
Digest string `json:"digest"`
Users []string `json:"users"`
}
// SealedUserKey is the key of one user's permissions in the sealed list of digest.
func SealedUserKey(digest, user string) string {
return KeySealedGrants + "." + strings.TrimPrefix(digest, "sha256:") + "." + user
}
// keepSealedGrants records the permissions of the list sealed now, one key per user, beside the list before; the lists
// older than those two lose their keys.
func (p *Part) keepSealedGrants(res *Result, digest string, list []byte, composed []User) { func (p *Part) keepSealedGrants(res *Result, digest string, list []byte, composed []User) {
in := listUsers(list) in := listUsers(list)
var users []User var users []User
@@ -108,20 +139,98 @@ func (p *Part) keepSealedGrants(res *Result, digest string, list []byte, compose
users = append(users, u) users = append(users, u)
} }
} }
var was SealedGrants failed := false
if raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants); err == nil && found { names := make([]string, 0, len(users))
_ = json.Unmarshal(raw, &was) for _, u := range users {
names = append(names, u.User)
data, _ := json.Marshal(u)
if _, err := p.Store.Put(StateUserList, SealedUserKey(digest, u.User), data); err != nil {
failed = true
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, SealedUserKey(digest, u.User), err))
}
} }
next := SealedGrants{Lists: []SealedList{{Digest: digest, Users: users}}} if failed {
// The index keeps naming the lists whose keys are all there; this list's permissions are not kept.
return
}
sort.Strings(names)
was := p.sealedLists()
next := SealedLists{Lists: []SealedNames{{Digest: digest, Users: names}}}
var gone []SealedNames
for _, l := range was.Lists { for _, l := range was.Lists {
if l.Digest != digest && len(next.Lists) < 2 { switch {
case l.Digest == digest:
case len(next.Lists) < sealedKept:
next.Lists = append(next.Lists, l) next.Lists = append(next.Lists, l)
default:
gone = append(gone, l)
} }
} }
data, _ := json.Marshal(next) data, _ := json.Marshal(next)
if _, err := p.Store.Put(StateUserList, KeySealedGrants, data); err != nil { if _, err := p.Store.Put(StateUserList, KeySealedLists, data); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeySealedGrants, err)) res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeySealedLists, err))
return
} }
for _, l := range gone {
for _, user := range l.Users {
if err := p.Store.Delete(StateUserList, SealedUserKey(l.Digest, user)); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be taken away: %v", StateUserList, SealedUserKey(l.Digest, user), err))
}
}
}
if _, _, found, err := p.Store.Get(StateUserList, KeySealedGrants); err == nil && found {
if err := p.Store.Delete(StateUserList, KeySealedGrants); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be taken away: %v", StateUserList, KeySealedGrants, err))
}
}
}
// sealedLists is the index of kept lists; none when it cannot be read.
func (p *Part) sealedLists() SealedLists {
var out SealedLists
if raw, _, found, err := p.Store.Get(StateUserList, KeySealedLists); err == nil && found {
_ = json.Unmarshal(raw, &out)
}
return out
}
// sealedGrantsOf is the permissions of each user of the sealed list of digest, as the part kept them, and whether it kept
// every one: from the index and each user's key, else from the one value written before this release.
func (p *Part) sealedGrantsOf(digest string) (map[string]User, bool) {
for _, l := range p.sealedLists().Lists {
if l.Digest != digest {
continue
}
out := map[string]User{}
for _, user := range l.Users {
raw, _, found, err := p.Store.Get(StateUserList, SealedUserKey(digest, user))
if err != nil || !found {
return nil, false
}
var u User
if json.Unmarshal(raw, &u) != nil {
return nil, false
}
out[u.User] = u
}
return out, true
}
raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants)
if err == nil && found {
var sg SealedGrants
if json.Unmarshal(raw, &sg) == nil {
for _, sl := range sg.Lists {
if sl.Digest == digest {
out := map[string]User{}
for _, u := range sl.Users {
out[u.User] = u
}
return out, true
}
}
}
}
return nil, false
} }
// loadedContents is the list the bus loaded, by user: as the bus module's nats_users read it, else as the part sealed the // loadedContents is the list the bus loaded, by user: as the bus module's nats_users read it, else as the part sealed the
@@ -144,16 +253,8 @@ func (p *Part) loadedContents(in Inputs, read []User, readErr error) (map[string
case l.Source != delivery.GrantsUserListSource || l.Follows != delivery.FollowGrants || l.Error != "": case l.Source != delivery.GrantsUserListSource || l.Follows != delivery.FollowGrants || l.Error != "":
return nil, "the bus loaded no list of the grants part's (source " + l.Source + ", follows " + l.Follows + "), and nats_users: " + readErr.Error() return nil, "the bus loaded no list of the grants part's (source " + l.Source + ", follows " + l.Follows + "), and nats_users: " + readErr.Error()
} }
raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants) if kept, ok := p.sealedGrantsOf(l.Digest); ok {
if err == nil && found { return kept, ""
var sg SealedGrants
if json.Unmarshal(raw, &sg) == nil {
for _, sl := range sg.Lists {
if sl.Digest == l.Digest {
return byUser(sl.Users), ""
}
}
}
} }
return nil, "the part keeps no permissions of the loaded list " + short(l.Digest) + " (sealed before it kept them), and nats_users: " + readErr.Error() return nil, "the part keeps no permissions of the loaded list " + short(l.Digest) + " (sealed before it kept them), and nats_users: " + readErr.Error()
} }
@@ -162,7 +263,7 @@ func (p *Part) loadedContents(in Inputs, read []User, readErr error) (map[string
const GrantMissingCondition = "mesh-grants.grant-missing" const GrantMissingCondition = "mesh-grants.grant-missing"
// MissingGrants is each module assigned or placed on a machine that want leaves without a grant, `<module>: <why>`, // MissingGrants is each module assigned or placed on a machine that want leaves without a grant, `<module>: <why>`,
// sorted: its manifest refused (skip), no fingerprint of the new form at the indexer's trunk, or no account with a hash // sorted, but a module the mesh provides itself, which is never wanted by fingerprint: its manifest refused (skip), no fingerprint of the new form at the indexer's trunk, or no account with a hash
// in the vault (drop). **The fingerprint is the indexer's, never a commit's** (issue 1126: link2pay, snake, f1-circuits // in the vault (drop). **The fingerprint is the indexer's, never a commit's** (issue 1126: link2pay, snake, f1-circuits
// and txt-game, whose source commit has not moved since a build of the old form, have a done key of the new form at it, // and txt-game, whose source commit has not moved since a build of the old form, have a done key of the new form at it,
// and that is the fingerprint granted). Pure. // and that is the fingerprint granted). Pure.
@@ -175,6 +276,12 @@ func MissingGrants(in Inputs, want map[string]delivery.ControllerGrant, drop []s
} }
var out []string var out []string
for _, module := range sortedKeys(placed) { for _, module := range sortedKeys(placed) {
// A module the mesh provides itself (the private network) is built by no repository, so the indexer holds no
// fingerprint of it and the assigner wants none: the controller composes it with the node's own. It needs no grant,
// and naming it missing raised a warning nothing could clear (sweep 2026-10-30 item 6).
if delivery.ProvidedByMesh(module) {
continue
}
fp, has := in.Fingerprints[module] fp, has := in.Fingerprints[module]
key, err := delivery.PartGrantKey(module, fp) key, err := delivery.PartGrantKey(module, fp)
if has && err == nil { if has && err == nil {
+19 -7
View File
@@ -131,9 +131,12 @@ type Result struct {
// Grants is what the pass wrote of its grants (grants.go). // Grants is what the pass wrote of its grants (grants.go).
Grants *GrantsWritten `json:"grants,omitempty"` Grants *GrantsWritten `json:"grants,omitempty"`
// Withheld is what the pass wrote of the hearings it withholds (withheld.go). // Withheld is what the pass wrote of the hearings it withholds (withheld.go).
Withheld *WithheldWritten `json:"withheld,omitempty"` Withheld *WithheldWritten `json:"withheld,omitempty"`
Errors []string `json:"errors,omitempty"` // PrincipalsRefused is each principal whose composed grants the composer refused, `<user>: <why>`: it keeps its grants
PublishedOrAsserted string `json:"published_or_asserted"` // of the list in force, or stays out where it holds none (refused.go); the condition mesh-grants.principal-refused says it.
PrincipalsRefused []string `json:"principals_refused,omitempty"`
Errors []string `json:"errors,omitempty"`
PublishedOrAsserted string `json:"published_or_asserted"`
} }
// NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships. // NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships.
@@ -186,6 +189,9 @@ type Part struct {
// differSaid is the account consumers' differences last said as a condition, once differKnown (consumers.go). // differSaid is the account consumers' differences last said as a condition, once differKnown (consumers.go).
differSaid string differSaid string
differKnown bool differKnown bool
// refusedSaid is the refused principals last said as a condition, once refusedKnown (refused.go).
refusedSaid string
refusedKnown bool
// missingSaid is the modules without a grant last said as a condition, once missingKnown (grants.go). // missingSaid is the modules without a grant last said as a condition, once missingKnown (grants.go).
missingSaid string missingSaid string
missingKnown bool missingKnown bool
@@ -251,11 +257,17 @@ func (p *Part) Compose() Result {
if st.LeftOut == nil { if st.LeftOut == nil {
st.LeftOut = []Gap{} st.LeftOut = []Gap{}
} }
users, err := ComposeUsers(listRecs) users, refused, err := ComposeEach(listRecs)
if err != nil { if err != nil {
res.Errors = append(res.Errors, "the shadow list could not be composed: "+err.Error()) res.Errors = append(res.Errors, "the shadow list could not be composed: "+err.Error())
} }
users = WithHashes(users, in.Hashes) // A refused principal keeps its grants of the list in force, said once per change (refused.go).
kept, _ := p.keptOf(refused)
for _, user := range sortedKeys(refused) {
res.PrincipalsRefused = append(res.PrincipalsRefused, user+": "+refused[user])
}
p.sayPrincipalsRefused(refused, kept)
users = WithHashes(withKept(users, kept), in.Hashes)
st.Digest, st.Users = Digest(users), len(users) st.Digest, st.Users = Digest(users), len(users)
loaded, loadErr := LoadedUsers(p.Call) loaded, loadErr := LoadedUsers(p.Call)
st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(listRecs))) st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(listRecs)))
@@ -289,7 +301,7 @@ func (p *Part) Compose() Result {
if p.Store != nil { if p.Store != nil {
p.writeWithheld(&res, in, recs) p.writeWithheld(&res, in, recs)
} }
list, n, leftOut, err := ComposeList(listRecs, in.Hashes) list, n, leftOut, dropped, err := ComposeList(listRecs, in.Hashes, refused, kept)
// The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5); after take-over // The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5); after take-over
// one that adds a permission only once the bus loaded a list that carries it (issue 774). // one that adds a permission only once the bus loaded a list that carries it (issue 774).
if p.Store != nil { if p.Store != nil {
@@ -306,7 +318,7 @@ func (p *Part) Compose() Result {
bus = names(loaded) bus = names(loaded)
lost = holdings.LastHolders(holdings.ByUsers(loaded), holdings.ByUsers(users)) lost = holdings.LastHolders(holdings.ByUsers(loaded), holdings.ByUsers(users))
} }
p.writeSealed(&res, list, n, leftOut, in, bus, lost, users) p.writeSealed(&res, list, n, leftOut, dropped, in, bus, lost, users)
} }
ms := Memberships(recs) ms := Memberships(recs)
res.Memberships = len(ms) res.Memberships = len(ms)
@@ -0,0 +1,137 @@
package grants
import (
"fmt"
"slices"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
// **One refused principal never freezes the list** (issue 774, sweep 2026-10-30; met twice that day: the engine's acts,
// then mesh-host_health). The composer refuses a principal's grants when they would make it a second writer of a state
// (buscompose.CheckWriters): that refusal used to end the whole composition, so the list in force stood for every principal
// while each change since piled up, and the next list that composed moved 52 users at once. Now each principal is composed
// on its own: one refused keeps the grants it holds in the list in force (the part's own record of that list, sealed
// grants), the rest of the list is composed and written as ever, and the refusal is raised as the warning
// mesh-grants.principal-refused, naming each principal and why, until it composes again. A refused principal the list in
// force does not carry stays out of the list: it never had those grants, and is refused them now.
// RefusedCondition is the key of the warning raised while a principal's grants are refused.
const RefusedCondition = "mesh-grants.principal-refused"
// ComposeEach is the shadow list composed principal by principal: every user whose grants compose, and each refused one
// by user with why. err is only for principals that cannot be derived at all (two users of one name). Pure.
func ComposeEach(r buscompose.Records) ([]User, map[string]string, error) {
principals, err := buscompose.Users(r)
if err != nil {
return nil, nil, err
}
out := make([]User, 0, len(principals))
refused := map[string]string{}
for _, p := range principals {
perms, err := buscompose.PermissionsFor(p)
if err != nil {
refused[p.Username()] = err.Error()
continue
}
out = append(out, User{User: p.Username(), Publish: set(perms.Publish), Subscribe: set(perms.Subscribe),
AllowResponses: perms.AllowResponses, PublishDeny: set(perms.PublishDeny)})
}
sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User })
return out, refused, nil
}
// keptOf is, for each refused user, its grants in the list in force as the part sealed them; the users it holds none of
// (not in the list in force, or that list's grants not kept) are the second answer, sorted.
func (p *Part) keptOf(refused map[string]string) (map[string]User, []string) {
if len(refused) == 0 {
return nil, nil
}
var inForce map[string]User
if g, _, had := p.Sealed(); had && g.Digest != "" && p.Store != nil {
inForce, _ = p.sealedGrantsOf(g.Digest)
}
kept := map[string]User{}
var none []string
for _, user := range sortedKeys(refused) {
if u, ok := inForce[user]; ok {
kept[user] = u
continue
}
none = append(none, user)
}
return kept, none
}
// withKept is users with each kept user in its place, sorted.
func withKept(users []User, kept map[string]User) []User {
out := slices.Clone(users)
for _, u := range kept {
out = append(out, u)
}
sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User })
return out
}
// keptEntry is a user of the accounts block from grants kept rather than composed, in ComposeAccounts' own lines
// (buscompose.AccountsEntry), so the bus reads it as it read the list in force.
func keptEntry(u User, hash string) string {
return buscompose.AccountsEntry(u.User, hash, buscompose.Permissions{Publish: u.Publish, PublishDeny: u.PublishDeny,
Subscribe: u.Subscribe, AllowResponses: u.AllowResponses})
}
// withEntries puts entries at the end of the accounts block's users, before it closes.
func withEntries(list string, entries []string) (string, error) {
if len(entries) == 0 {
return list, nil
}
start := strings.Index(list, " users = [\n")
if start < 0 {
return "", fmt.Errorf("the composed list holds no users block to keep a refused user's grants in")
}
end := strings.Index(list[start:], "\n ]\n")
if end < 0 {
return "", fmt.Errorf("the composed list's users block does not close")
}
at := start + end + 1
return list[:at] + strings.Join(entries, "") + list[at:], nil
}
// sayPrincipalsRefused raises RefusedCondition while a principal's grants are refused, naming each with why and what
// became of it, and clears it once none is; said once per set, again when it changes, and at the first pass after a start.
func (p *Part) sayPrincipalsRefused(refused map[string]string, kept map[string]User) {
if p.Say == nil {
return
}
var lines []string
for _, user := range sortedKeys(refused) {
became := "left out of the list: the list in force does not carry it"
if _, ok := kept[user]; ok {
became = "keeps its grants of the list in force"
}
lines = append(lines, user+" ("+became+"): "+refused[user])
}
why := strings.Join(lines, "; ")
p.holdMu.Lock()
defer p.holdMu.Unlock()
if p.refusedKnown && why == p.refusedSaid {
return
}
c := asks.Condition{Key: RefusedCondition, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}}
if why == "" {
c.State = asks.StateCleared
} else {
c.Severity, c.State = asks.SeverityWarning, asks.StateRaised
c.Summary = fmt.Sprintf("%d principal(s) refused their composed grants: %s", len(refused), why)
c.Headline, c.Needs = "A principal's bus grants are refused", "a fix of the manifest or the writers table"
c.Explanation = asks.NeedsYou + "a fix of what makes it a second writer. The rest of the user list is written as ever; " +
"each refused principal keeps the grants the list in force gives it, or stays out where it holds none: " + why
c.Resolved = "every principal's grants compose again"
}
if p.Say(c) == nil {
p.refusedSaid, p.refusedKnown = why, true
}
}
+40 -12
View File
@@ -31,24 +31,41 @@ type Objects interface {
DeleteObject(name string) error DeleteObject(name string) error
} }
// ComposeList is the bus's user list as the file holds it, with each user's hash from the vault, and the users left out // ComposeList is the bus's user list as the file holds it, with each user's hash from the vault; the users left out
// for want of one. Pure. // for want of one; and the refused principals (refused, by user) the list carries with their grants kept from the list
func ComposeList(r buscompose.Records, hashes map[string]HeldHash) ([]byte, int, []string, error) { // in force (kept), or leaves out where kept holds none (dropped, sorted). Pure.
func ComposeList(r buscompose.Records, hashes map[string]HeldHash, refused map[string]string, kept map[string]User) (list []byte, users int, leftOut, dropped []string, err error) {
principals, err := buscompose.Users(r) principals, err := buscompose.Users(r)
if err != nil { if err != nil {
return nil, 0, nil, err return nil, 0, nil, nil, err
} }
held := map[string]string{} held := map[string]string{}
for user, h := range hashes { for user, h := range hashes {
held[user] = h.Hash held[user] = h.Hash
} }
filled, missing := buscompose.WithPasswords(principals, held) composing := slices.DeleteFunc(slices.Clone(principals), func(p buscompose.Principal) bool { _, no := refused[p.Username()]; return no })
list, err := buscompose.ComposeAccounts(filled) filled, missing := buscompose.WithPasswords(composing, held)
text, err := buscompose.ComposeAccounts(filled)
if err != nil { if err != nil {
return nil, 0, nil, err return nil, 0, nil, nil, err
}
var entries []string
for _, user := range sortedKeys(refused) {
u, ok := kept[user]
switch {
case !ok:
dropped = append(dropped, user)
case held[user] == "":
missing = append(missing, user)
default:
entries = append(entries, keptEntry(u, held[user]))
}
}
if text, err = withEntries(text, entries); err != nil {
return nil, 0, nil, nil, err
} }
slices.Sort(missing) slices.Sort(missing)
return []byte(list), len(filled), missing, nil return []byte(text), len(filled) + len(entries), missing, dropped, nil
} }
// Sealed is the part's key `user-list.sealed` as written, and whether there is one. // Sealed is the part's key `user-list.sealed` as written, and whether there is one.
@@ -74,7 +91,7 @@ func followOf(state string) string {
// writeSealed puts the list sealed to the bus node and writes the key, each only when what it says changed. It says in // writeSealed puts the list sealed to the bus node and writes the key, each only when what it says changed. It says in
// res what it wrote, or why it wrote nothing. // res what it wrote, or why it wrote nothing.
func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string, in Inputs, bus []string, lost []string, composed []User) { func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut, dropped []string, in Inputs, bus []string, lost []string, composed []User) {
cur, _, had := p.Sealed() cur, _, had := p.Sealed()
next := cur next := cur
next.Follow = followOf(p.SwitchState()) next.Follow = followOf(p.SwitchState())
@@ -93,10 +110,10 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string
why = "no node runs the bus's module, so the list is sealed to nobody" why = "no node runs the bus's module, so the list is sealed to nobody"
case key == "": case key == "":
why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)" why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)"
case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, bus, lost, digest) != "": case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, digest) != "":
// After take-over the list in force is never replaced by one composed past a failed read, or one that drops a user // After take-over the list in force is never replaced by one composed past a failed read, or one that drops a user
// of it for want of a hash: that user's module would be locked out of the bus (review of the switch). // of it for want of a hash: that user's module would be locked out of the bus (review of the switch).
why = p.holdsBack(in, leftOut, bus, lost, digest) why = p.holdsBack(in, leftOut, dropped, bus, lost, digest)
held = why held = why
p.keepHashHold(res, digest) p.keepHashHold(res, digest)
case cur.Digest != digest || cur.SealedTo != delivery.SealingKeyID(key) || cur.Node != in.BusNode: case cur.Digest != digest || cur.SealedTo != delivery.SealingKeyID(key) || cur.Node != in.BusNode:
@@ -356,7 +373,7 @@ func listUsers(list []byte) []string {
// loaded list not read holds it too, since that cannot be ruled out). The list in force is the one the part wrote, by the users // loaded list not read holds it too, since that cannot be ruled out). The list in force is the one the part wrote, by the users
// it kept of it; where it kept none (a list written before this release), the users the bus loaded (bus, nil when // it kept of it; where it kept none (a list written before this release), the users the bus loaded (bus, nil when
// they could not be read). Empty when it may. // they could not be read). Empty when it may.
func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string, digest string) string { func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest string) string {
p.hashHold = nil p.hashHold = nil
if f := in.ReadFailures(); len(f) > 0 { if f := in.ReadFailures(); len(f) > 0 {
return "held: the list in force is kept while a read of this pass failed (" + strings.Join(f, "; ") + ")" return "held: the list in force is kept while a read of this pass failed (" + strings.Join(f, "; ") + ")"
@@ -387,6 +404,17 @@ func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string, digest string)
if len(unhashed) > 0 { if len(unhashed) > 0 {
return "held: the list in force is kept, since " + strings.Join(unhashed, ", ") + " would be left out for want of a hash in the vault" return "held: the list in force is kept, since " + strings.Join(unhashed, ", ") + " would be left out for want of a hash in the vault"
} }
// A refused principal the list in force carries and whose grants there the part did not keep: leaving it out would lock
// it out of the bus (refused.go), so the list in force stands until its grants compose or are kept.
var unkept []string
for _, u := range dropped {
if slices.Contains(had, u) {
unkept = append(unkept, u)
}
}
if len(unkept) > 0 {
return "held: the list in force is kept, since " + strings.Join(unkept, ", ") + " would be left out: its grants are refused and the part kept none of the list in force"
}
// More than one loaded user's hash changing at once, with no rotation by the vault naming each (ADR 0561 §7): a new // More than one loaded user's hash changing at once, with no rotation by the vault naming each (ADR 0561 §7): a new
// writer of hashes must not lock several holders out in one list, as nine seats were cut on 2026-10-27 at 09:21. // writer of hashes must not lock several holders out in one list, as nine seats were cut on 2026-10-27 at 09:21.
changed, err := p.changedHashes(in.Hashes) changed, err := p.changedHashes(in.Hashes)
@@ -116,23 +116,18 @@ func CredentialsOf(shadow, loaded []User, hashes map[string]HeldHash) Credential
return c return c
} }
// ComposeUsers is the shadow list: every user the composer derives from the records, with its grants. Pure. // ComposeUsers is the shadow list: every user the composer derives from the records, with its grants; an error when any
// principal's grants are refused (ComposeEach composes past one). Pure.
func ComposeUsers(r buscompose.Records) ([]User, error) { func ComposeUsers(r buscompose.Records) ([]User, error) {
principals, err := buscompose.Users(r) users, refused, err := ComposeEach(r)
if err != nil { if err != nil {
return nil, err return nil, err
} }
out := make([]User, 0, len(principals)) if len(refused) > 0 {
for _, p := range principals { first := sortedKeys(refused)[0]
perms, err := buscompose.PermissionsFor(p) return nil, fmt.Errorf("%s: %s", first, refused[first])
if err != nil {
return nil, fmt.Errorf("%s: %w", p.Username(), err)
}
out = append(out, User{User: p.Username(), Publish: set(perms.Publish), Subscribe: set(perms.Subscribe),
AllowResponses: perms.AllowResponses, PublishDeny: set(perms.PublishDeny)})
} }
sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User }) return users, nil
return out, nil
} }
func set(in []string) []string { func set(in []string) []string {