The grants part writes its list whole however large, one refused principal no longer freezes it, and stale watchman keys close (issue 774) #117

Merged
jschoubben merged 10 commits from fix/sweep-2026-10-30 into main 2026-10-30 18:13:00 +00:00
33 changed files with 753 additions and 88 deletions
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -151,6 +151,31 @@ func IsUntrunked(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && v.Failed.Machine == "checker" && strings.HasPrefix(v.Failed.Words, UntrunkedWords)
}
// **A candidate outside the mesh is not the checker's** (sweep 2026-10-30 item 7): the forge carries repositories the mesh
// does not deliver (an operator's own), whose pull requests the indexer writes as candidates like any. Their trunk the
// gate never names, so each was judged "cannot be judged" and owed the status error on mesh/check, which the forge
// module wrote 119 times per head into a repository the mesh's forge account may not write (403). A candidate every pull
// request of which is of a repository the gate's source.main names no trunk head of, and touches no module, is outside
// the mesh: judged nothing, owed nothing on the forge, said once in the log and in checker_status (outside_the_mesh). A
// new repository of the mesh is not outside: its first pull request adds a module, and is said untrunked (issue 1422).
// OutsideWords say why a candidate is outside the mesh.
const OutsideWords = "outside the mesh: the gate's source.main names no trunk head of its repositories and it touches no module, so the checker judges nothing and owes no status on the forge"
// Outside says every pull request of c is of a repository trunk names no head of and touches no module. Never with
// trunk unknown (nil): before the gate's source is read nothing is outside.
func Outside(c delivery.Candidate, trunk map[string]string) bool {
if trunk == nil || len(c.Pulls) == 0 {
return false
}
for _, p := range c.Pulls {
if trunk[p.Repository] != "" || len(p.Modules) > 0 || len(p.Fps) > 0 {
return false
}
}
return true
}
// leftOf are the pull requests of a candidate whose repository has no trunk head known, with Trunk empty, in repository and
// number order; nil when every repository's is known.
func leftOf(c delivery.Candidate, trunk map[string]string) []Pull {
@@ -104,6 +104,8 @@ type Checker struct {
held map[string]string // by verdict key: the turn that holds its check off, as last said
live map[string]liveCheck // by verdict key: the check running in this process, ended when its heads go
settling map[string]*window // by candidates key: the window its heads settle in
// outsideSaid are the candidates outside the mesh said in the log, by group and heads.
outsideSaid map[string]bool
}
// liveCheck is a check running in this process: the candidates key of its group and the end of its run.
@@ -160,6 +162,7 @@ func (ch *Checker) init() {
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{}
ch.live, ch.settling = map[string]liveCheck{}, map[string]*window{}
ch.outsideSaid = map[string]bool{}
})
}
@@ -327,6 +330,10 @@ func (ch *Checker) heads() map[string]string {
func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidate) {
heads := ch.heads()
if Outside(c, heads) {
ch.outside(ctx, c)
return
}
if left := leftOf(c, heads); len(left) > 0 {
// A pull request the checker cannot claim (its repository's trunk head is not known) leaves its whole group
// unjudged, said in a failure verdict (issue 1422, issue 1733): the gate asks the Approve only on a verdict, and a
@@ -464,6 +471,39 @@ func (ch *Checker) untrunked(ctx context.Context, c delivery.Candidate, left []P
}
}
// outside takes a candidate outside the mesh (case.go Outside): no verdict, and every status owed for its pull requests
// taken away, so the forge module stops writing one into a repository the mesh's forge account may not write. Said once
// per heads in the log.
func (ch *Checker) outside(ctx context.Context, c delivery.Candidate) {
var names []string
for _, p := range c.Pulls {
names = append(names, fmt.Sprintf("%s#%d@%s", p.Repository, p.Number, short12(p.Head.Head)))
key, err := delivery.CheckerOwedKey(p.Repository, p.Number, p.Head.Head)
if err != nil {
continue
}
_, _, found, err := ch.Owed.Get(ctx, key)
if err != nil || !found {
continue
}
d, ok := ch.Owed.(Deleter)
if !ok {
continue
}
if err := d.Delete(ctx, key); err != nil {
ch.logf("%s: the status owed of a pull request outside the mesh could not be taken away: %v", key, err)
}
}
said := c.Group + "@" + c.HeadsHash
ch.mu.Lock()
first := !ch.outsideSaid[said]
ch.outsideSaid[said] = true
ch.mu.Unlock()
if first {
ch.logf("candidates of %s: %s %s", c.Group, strings.Join(names, ", "), OutsideWords)
}
}
// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on
// the build seats' check stream; a carry, a verdict moved to a newer trunk or sibling commit by its package digests (ADR 0548
// §5), on the recheck stream, pulled after.
@@ -48,6 +48,13 @@ func pullOf(repo string, number int, head string) delivery.CandidatePull {
return delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: number, Head: head}, Modules: []string{}}
}
// newRepoPullOf is a pull request of a new repository of the mesh: its trunk not named yet, its first module added.
func newRepoPullOf(repo string, number int, head string) delivery.CandidatePull {
p := pullOf(repo, number, head)
p.Modules = []string{"mod-u"}
return p
}
// judgeFunc is a Judge written as a function; it counts its calls.
type judgeFunc struct {
mu sync.Mutex
@@ -249,7 +256,7 @@ func TestARepositoryWithNoTrunkCannotBeJudgedLoudlyUntilItsTrunkIsKnown(t *testi
var logged []string
r.ch.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
const unknown = "novox/unknown"
cand := candidateOf(t, "feat/y", pullOf(unknown, 4, head1))
cand := candidateOf(t, "feat/y", newRepoPullOf(unknown, 4, head1))
key := keyOf(t, "feat/y", cand)
r.ch.Candidate(ctx, "feat/y", cand)
r.ch.Wait()
@@ -305,7 +312,7 @@ func TestARepositoryWithNoTrunkCannotBeJudgedLoudlyUntilItsTrunkIsKnown(t *testi
want string // the verdict's state, and for a failure the repository it names
}{
{name: "a member the checker cannot claim", pulls: []delivery.CandidatePull{pullOf(repoA, 9, head2), pullOf("novox/other", 2, head2)}, want: "novox/other"},
{name: "a running verdict left for heads that cannot be judged", pulls: []delivery.CandidatePull{pullOf(unknown, 5, head2)}, running: true, want: unknown},
{name: "a running verdict left for heads that cannot be judged", pulls: []delivery.CandidatePull{newRepoPullOf(unknown, 5, head2)}, running: true, want: unknown},
{name: "a verdict key deleted", pulls: []delivery.CandidatePull{pullOf(repoA, 5, head2)}, deleted: true, judged: 2, want: delivery.VerdictPass},
} {
t.Run(row.name, func(t *testing.T) {
@@ -566,3 +573,43 @@ func TestTheForgeBridgeIsToldOnceForEachChangeOfAStatus(t *testing.T) { // 1683
t.Errorf("told %v after a check that changed nothing", told)
}
}
// Sweep 2026-10-30 item 7: a pull request of a repository the mesh does not deliver (an operator's own, on the same
// forge) touches no module and its trunk the gate never names. It is outside the mesh: no verdict, no status owed (one
// owed before is taken away, so the forge module stops writing it into a repository the mesh may not write), said once.
func TestACandidateOutsideTheMeshIsJudgedNothingAndOwesNothing(t *testing.T) {
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
var logged []string
r.ch.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
const theirs = "someone/own-app"
okey, _ := delivery.CheckerOwedKey(theirs, 55, head1)
before, _ := delivery.EncodeCheckerOwed(delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "error", Text: "cannot be judged"})
if _, err := r.owed.Create(ctx, okey, before); err != nil {
t.Fatal(err)
}
cand := candidateOf(t, "feat/theirs", pullOf(theirs, 55, head1))
key := keyOf(t, "feat/theirs", cand)
r.ch.Candidate(ctx, "feat/theirs", cand)
r.ch.Wait()
r.ch.Reconsider(ctx)
r.ch.Wait()
if _, _, found, _ := r.verdicts.Get(ctx, key); found || j.count() != 0 {
t.Errorf("a candidate outside the mesh was judged (%d) or given a verdict (%v)", j.count(), found)
}
if s := owedState(t, r, theirs, 55, head1); s != "" {
t.Errorf("%s#55 still owes %q on the forge", theirs, s)
}
if n := countWith(logged, "outside the mesh"); n != 1 {
t.Errorf("said %d times in the log, want once: %q", n, logged)
}
ckey, _ := delivery.CandidatesKey("feat/theirs")
st, err := Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState(r.verdicts.vals).read(),
Trunks: func() (map[string]string, error) { return r.trunks.Heads(), nil }}.Status()
if err != nil {
t.Fatal(err)
}
if st.Outside["feat/theirs"] == "" || len(st.CannotBeJudged) != 0 || len(st.Unjudged) != 0 {
t.Errorf("status: outside %v, cannot be judged %v, unjudged %v", st.Outside, st.CannotBeJudged, st.Unjudged)
}
}
@@ -335,6 +335,8 @@ type Status struct {
// CannotBeJudged are the candidates no pull request of which can be judged, by group, with why: the gate's source.main
// names no trunk head of their repositories (issue 1422). They are judged once it does; until then their status is error.
CannotBeJudged map[string]string `json:"cannot_be_judged"`
// Outside are the candidates outside the mesh, by group, with why (case.go Outside): judged nothing, owed nothing.
Outside map[string]string `json:"outside_the_mesh"`
// Lost are the candidates whose verdict is running while no step of theirs can still run and their newest step ended, by
// group, with that step: nothing will judge them (the lost check of 2026-10-28); checker_ask_again asks one again.
Lost map[string]string `json:"lost"`
@@ -415,7 +417,7 @@ func (r Reader) Status() (Status, error) {
}
st.Unjudged = []string{}
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
st.HeldByTurn, st.CannotBeJudged = map[string]string{}, map[string]string{}
st.HeldByTurn, st.CannotBeJudged, st.Outside = map[string]string{}, map[string]string{}, map[string]string{}
turns := r.turns(&st)
var heads map[string]string
if r.Trunks == nil {
@@ -466,6 +468,10 @@ func (r Reader) Status() (Status, error) {
continue
}
c, raw := row.c, row.raw
if Outside(c, heads) {
st.Outside[c.Group] = OutsideWords
continue
}
if heads != nil && len(leftOf(c, heads)) > 0 {
// Nothing of it can be judged now, whatever its verdict says (issue 1422): never counted as waiting.
var repos []string
@@ -17,3 +17,8 @@ type KV interface {
// Keys are every key of the bucket.
Keys(ctx context.Context) ([]string, error)
}
// Deleter is a KV that can take a key away; a KV that cannot leaves it.
type Deleter interface {
Delete(ctx context.Context, key string) error
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-teller
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-view
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -244,6 +244,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
process.NotWatched(logger.Printf, delivery.AssignerWantsBucket, err, "moves are not watched until the watchman starts again")
haveSeen.Stop()
haveSeen = nil
} else {
w.ReadWhole(delivery.AssignerWantsBucket)
}
}
// The build seats' steps: a step key `<node>.<ask id>` written is the ask taken (issue 1887 A8b). A seat whose build module
@@ -263,6 +265,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
continue
}
taken[lang] = sw
if bucket, err := delivery.AskStream(lang); err == nil {
w.ReadWhole(bucket)
}
}
// The events first, then the one request of the open connections, so no connection falls between them.
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-watchman
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op h1:1BOWQJweNyvZMlpAHXGLiZQn9S+QXGcz3xh94lC0w6E=
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -40,6 +40,7 @@ func (w *Watchman) Step(language, key string, value []byte, op Op) {
w.moved(bucket, id)
return
}
w.stands(bucket, id)
step, err := delivery.DecodeStep(value)
if err != nil {
w.say("the step %s cannot be read, and is left: %v", key, err)
@@ -80,6 +81,7 @@ func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) {
w.moved(delivery.AssignerWantsBucket, key)
return
}
w.stands(delivery.AssignerWantsBucket, key)
want, err := delivery.DecodeWant(value)
if err != nil {
w.say("the want %s cannot be read, and is left: %v", key, err)
@@ -91,6 +91,11 @@ type Watchman struct {
tunnels map[string]string
// stepBound is how long a build step may run once a node took the ask; a test shortens it.
stepBound time.Duration
// standing is, until the start ends (Reconcile), each mirrored key read standing, by the bucket a stuck key names it
// under; readWhole the buckets whose keys were all read at this start (ReadWhole). reconciled is the start's end.
standing map[string]map[string]bool
readWhole map[string]bool
reconciled bool
}
// New is a watchman writing to out, reading the time from now.
@@ -100,7 +105,8 @@ func New(out Out, now func() time.Time) *Watchman {
findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, away: map[string]bool{},
haveState: map[string]delivery.Have{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{},
control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{},
health: map[string]delivery.NodeHealth{}, tunnels: map[string]string{}}
health: map[string]delivery.NodeHealth{}, tunnels: map[string]string{},
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}}
}
// Logf sets where a write that failed is said; it is silent until set.
@@ -257,7 +263,8 @@ func (w *Watchman) SeedPulse(key string) {
}
// Reconcile ends the start: a pulse that stood and that no mirrored key has made again is a move that happened while
// the watchman was away, so it is deleted and writes no stuck key.
// the watchman was away, so it is deleted and writes no stuck key; and a stuck key whose watched key is gone is closed
// (closeGone).
func (w *Watchman) Reconcile() {
w.mu.Lock()
defer w.mu.Unlock()
@@ -266,9 +273,49 @@ func (w *Watchman) Reconcile() {
w.deletePulse(pk)
}
}
w.closeGone()
w.reconcileFindings()
}
// **A stuck key whose watched key is gone is closed** (issue 774, sweep 2026-10-30): a key that ages out of its bucket
// (a stream's maximum age, 168 h for the build seats' asks) leaves no marker a watch hears, and one deleted while the
// watchman was down is not heard either, so its stuck key stood unmoved for ever: 26 stuck-key conditions of asks from
// 2026-10-21 were open on 10-30. At the start's end, every stuck key not moved whose bucket was read whole at this start
// (ReadWhole) and whose key was not read standing is written moved, which clears its finding, and said. A bucket that
// could not be read closes nothing: its keys are not known to be gone.
// ReadWhole says every key of bucket (as a stuck key names it: an ask stream for a build seat's steps, the assigner's
// wants) was read at this start; called once its watch holds what stands.
func (w *Watchman) ReadWhole(bucket string) {
w.mu.Lock()
defer w.mu.Unlock()
w.readWhole[bucket] = true
}
// stands records a mirrored key read standing during the start.
func (w *Watchman) stands(bucket, key string) {
if w.reconciled {
return
}
if w.standing[bucket] == nil {
w.standing[bucket] = map[string]bool{}
}
w.standing[bucket][key] = true
}
func (w *Watchman) closeGone() {
for sk, s := range w.stuck {
if s.Moved || !w.readWhole[s.Bucket] || w.standing[s.Bucket][s.Key] {
continue
}
s.Moved = true
w.say("the stuck key %s is closed: its key %s is no longer in %s (it aged out of the bucket or was deleted while nobody heard), so nothing waits for it since %s",
sk, s.Key, s.Bucket, s.Since.UTC().Format(time.RFC3339))
w.putStuck(sk, s)
}
w.standing, w.readWhole, w.reconciled = map[string]map[string]bool{}, map[string]bool{}, true
}
// reconcileFindings ends the start for the findings: one read at the start that what the watchman read since did not
// observe again is deleted, unless a signal still standing keeps it (a provider's failing word, an advisory's quiet hour, a
// machine's silent tools, a silent self-check): those end by their own events or expiry.
@@ -656,3 +656,43 @@ func TestAWantNotAppliedByItsBoundIsStuckAndAKeyDeletedEndsIt(t *testing.T) {
t.Errorf("a deleted want left its pulse: %v", o2.pulses)
}
}
// Sweep 2026-10-30: a watched key that aged out of its bucket or was deleted while nobody heard leaves its stuck key; at the
// start's end the watchman closes it (moved, its finding cleared), but only for a bucket it read whole at that start.
func TestAStuckKeyWhoseKeyIsGoneIsClosedAtTheStart(t *testing.T) {
wantKey := "want.n1.mod-a"
for _, row := range []struct {
name string
bucket string
key string
standing func(w *Watchman) // what the start reads standing
readWhole bool
closed bool
}{
{"an ask's step aged out of the steps bucket", checksKey, "askold1", func(*Watchman) {}, true, true},
{"an ask's step still running", checksKey, "askold1", func(w *Watchman) { w.Step("go", "n1.askold1", runningStep("askold1", t0), Put) }, true, false},
{"the steps bucket could not be read", checksKey, "askold1", func(*Watchman) {}, false, false},
{"a want deleted while nobody heard", delivery.AssignerWantsBucket, wantKey, func(*Watchman) {}, true, true},
{"a want still standing", delivery.AssignerWantsBucket, wantKey, func(w *Watchman) { w.Want(wantKey, wantValue(t, fpNew, false), Put, t0) }, true, false},
} {
t.Run(row.name, func(t *testing.T) {
w, o, c := setup()
c.t = t0.Add(9 * 24 * time.Hour)
sk := stuckKeyOf(t, row.bucket, row.key)
v, _ := delivery.EncodeStuck(delivery.Stuck{Bucket: row.bucket, Key: row.key, Waited: "a move", Since: t0})
w.SeedStuck(sk, v)
if _, ok := o.findings[FindingKey(ScopeDelivery, sk, KindStuck)]; !ok {
t.Fatal("the seeded stuck key raised no finding")
}
row.standing(w)
if row.readWhole {
w.ReadWhole(row.bucket)
}
w.Reconcile()
_, open := o.findings[FindingKey(ScopeDelivery, sk, KindStuck)]
if got := o.stuck[sk].Moved; got != row.closed || open == row.closed {
t.Errorf("moved %v, finding open %v; want closed %v", got, open, row.closed)
}
})
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-grants
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030154046-af66ccb8335a
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030154046-af66ccb8335a h1:8A43iLD5YlqwyeKmGvAgTvf2b5W/FDi9VDkqvPIh5MU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030154046-af66ccb8335a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24 h1:s0XXZtR7CFPBbnaDygWnE4/Q827iEFt0YfEzujk4Q9g=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030174137-a37181261c24/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+171 -2
View File
@@ -67,6 +67,8 @@ type fakeBus struct {
// consumerCreates each consumer the part asked to create, `<stream>.<name>`.
consumers map[string][]map[string]any
consumerCreates []string
// maxValue, when set, refuses an own value larger than it as the bus does (JetStream 10054).
maxValue int
}
// MatchConsumer is the bus's update of a held durable consumer to the configuration given.
@@ -260,6 +262,9 @@ func (s ownStore) Put(state, key string, value []byte) (uint64, error) {
if b.refuseTo[state] {
return 0, errors.New("Permissions Violation for Publish to $KV.mesh-grants_" + state)
}
if b.maxValue > 0 && len(value) > b.maxValue {
return 0, errors.New("nats: API error: code=400 err_code=10054 description=message size exceeds maximum allowed")
}
b.writes = append(b.writes, state)
if b.own[state] == nil {
b.own[state] = map[string][]byte{}
@@ -1542,7 +1547,7 @@ func TestGrants(t *testing.T) {
fp := partOn(b)
fp.Objects = p.Objects
fp.Asker, fp.Say = fa, func(c asks.Condition) error {
if c.Key != grants.GrantMissingCondition { // its own row below
if c.Key != grants.GrantMissingCondition && c.Key != grants.RefusedCondition { // their own rows
fresh = append(fresh, c)
}
return nil
@@ -2188,6 +2193,8 @@ func TestGrants(t *testing.T) {
return nil
}
b.put(grants.ReadLease, grants.LeaseKey, map[string]any{"handed-to": "the operator's hand-over"})
// The private network the mesh provides is placed with no fingerprint by construction: never a missing grant.
b.assign("b", "mesh-wireguard")
fp := "src2:d4afe7a1abab5ccdbef6d8629b2245006a2aec60290f430f7b4756e2930ed60c"
b.put(grants.ReadModules, "mod-x", delivery.Module{V: delivery.Version, FP2: fp, Manifest: delivery.ManifestDigest([]byte(xManifest)),
Repository: "novox/Link2Pay", Commit: "5d20e8bfbc953c51e8ebe3c671fe883df9c36e15"})
@@ -2440,7 +2447,7 @@ func TestGrants(t *testing.T) {
// the composed configuration only by hand (grants_consumer_match), never while the controller holds the lease.
var said []asks.Condition
p.Say = func(c asks.Condition) error {
if c.Key != grants.GrantMissingCondition { // its own row
if c.Key != grants.GrantMissingCondition && c.Key != grants.RefusedCondition { // their own rows
said = append(said, c)
}
return nil
@@ -2565,6 +2572,168 @@ func TestGrants(t *testing.T) {
t.Errorf("changes the vault's rotation names were held: %+v", said)
}
})
// Sweep 2026-10-30 item 5: the permissions of the sealed lists were one value holding every user of two lists, which
// outgrew the bucket's largest value, so they were never kept. One key per user: no value grows with the list.
t.Run("sealed grants: one key per user, so no value grows with the list, and only the last two lists kept", func(t *testing.T) {
b := fixture()
for _, u := range shadowOf(t, b) {
b.put(grants.ReadAccounts, u.User, map[string]any{"user": u.User, "hash": "$2a$10$first", "generation": 1, "fingerprint": "sha256:1"})
}
largest := 0
for _, u := range shadowOf(t, b) {
raw, _ := json.Marshal(u)
largest = max(largest, len(raw))
}
b.maxValue = largest + 256 // one user's permissions fit, every user's of a list do not
busKey, err := seal.GenerateKeyPair()
if err != nil {
t.Fatal(err)
}
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}})
p := partOn(b)
p.Objects = &fakeObjects{}
var digests []string
for gen := 1; gen <= 3; gen++ {
b.put(grants.ReadAccounts, "a.mod-x", map[string]any{"user": "a.mod-x", "hash": fmt.Sprintf("$2a$10$gen%d", gen), "generation": gen, "fingerprint": fmt.Sprintf("sha256:%d", gen)})
res := p.Compose()
for _, e := range res.Errors {
if strings.Contains(e, "10054") {
t.Fatalf("a sealed-grants write exceeded the bucket's largest value: %s", e)
}
}
g, _, _ := p.Sealed()
digests = append(digests, g.Digest)
}
var idx grants.SealedLists
raw, _, found, _ := b.ownGet(grants.StateUserList, grants.KeySealedLists)
if !found || json.Unmarshal(raw, &idx) != nil || len(idx.Lists) != 2 || idx.Lists[0].Digest != digests[2] || idx.Lists[1].Digest != digests[1] {
t.Fatalf("the index of kept lists = %s, want the last two of %v", raw, digests)
}
for i, d := range digests {
_, _, kept, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(d, "a.mod-x"))
if kept != (i > 0) {
t.Errorf("list %d's a.mod-x key kept = %v", i, kept)
}
}
for _, u := range idx.Lists[0].Users {
if _, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(digests[2], u)); !ok {
t.Errorf("the index names %s, whose key is not there", u)
}
}
// The loaded list's permissions are read back from the keys when the bus module's tools cannot say them.
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: delivery.GrantsUserListSource, Follows: delivery.FollowGrants, Revision: 8,
Digest: digests[2], Users: 6, Key: busKey.PublicKey}})
b.refuseUsers = true
if res := p.Compose(); slices.ContainsFunc(res.Grants.Waits, func(w string) bool { return strings.Contains(w, "unknown") }) {
t.Errorf("the kept permissions of the loaded list were not read: %v", res.Grants.Waits)
}
})
// Sweep 2026-10-30 item 5, a kind met twice that day (the engine's acts, then mesh-host_health): one principal the
// composer refuses as a second writer froze the whole list. Now it keeps its grants of the list in force, the rest is
// written, and the refusal is raised by name; one the list in force does not carry stays out, said too.
t.Run("a refused principal keeps its grants of the list in force; the rest of the list is written", func(t *testing.T) {
b := fixture()
const ctl = `{"module":"mesh-controller","version":"1","own-secrets":{"broker":"/var/lib/c/broker"}}`
refusedCtl := strings.Replace(ctl, `}}`, `},"state":[{"name":"declared-config"}]}`, 1)
b.manifestOf("mesh-controller", ctl)
b.assign("b", "mesh-controller")
hash := func(user, h string, gen int) {
b.put(grants.ReadAccounts, user, map[string]any{"user": user, "hash": h, "generation": gen, "fingerprint": fmt.Sprintf("sha256:%d", gen)})
}
for _, u := range shadowOf(t, b) {
hash(u.User, "$2a$10$first", 1)
}
hash("a.mesh-controller", "$2a$10$first", 1)
busKey, err := seal.GenerateKeyPair()
if err != nil {
t.Fatal(err)
}
b.put(grants.ReadHave, "a.nats", delivery.Have{V: delivery.Version, Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: "a.nats.src2", Revision: 7, Digest: "sha256:aa", Users: 6, Key: busKey.PublicKey}})
p := partOn(b)
objs := &fakeObjects{}
p.Objects = objs
entryOf := func(g delivery.GrantsUserList, user string) string {
t.Helper()
list, err := delivery.OpenUserList(objs.objects[g.Object], g.Digest, func(bx seal.Box) ([]byte, error) {
out, err := seal.Open(bx, busKey.PrivateKey)
return []byte(out), err
})
if err != nil {
t.Fatalf("the sealed list does not open: %v", err)
}
at := strings.Index(string(list), fmt.Sprintf("{ user: %q,", user))
if at < 0 {
return ""
}
rest := string(list)[at:]
return rest[:strings.Index(rest, "} }\n")+4]
}
var said []asks.Condition
p.Say = func(c asks.Condition) error {
if c.Key == grants.RefusedCondition {
said = append(said, c)
}
return nil
}
sw, _ := json.Marshal(grants.Switch{V: 1, State: grants.TookOver, Acts: []grants.Act{}})
if _, err := (ownStore{b}).Put(grants.StateTakeover, grants.KeySwitch, sw); err != nil {
t.Fatal(err)
}
p.Compose()
inForce, _, _ := p.Sealed()
if inForce.Digest == "" {
t.Fatal("no list written after take-over")
}
entryBefore := entryOf(inForce, "b.mesh-controller")
ctlKey := grants.SealedUserKey(inForce.Digest, "b.mesh-controller")
before, _, ok, _ := b.ownGet(grants.StateUserList, ctlKey)
if !ok {
t.Fatal("the list in force keeps no grants of b.mesh-controller")
}
// b.mesh-controller now declares the controller's mirror as its own state: refused as a second writer. a.mod-x's hash
// changes in the same pass, and a.mesh-controller, refused the same way, is placed new.
b.manifestOf("mesh-controller", refusedCtl)
b.assign("a", "mesh-controller")
hash("a.mod-x", "$2a$10$second", 2)
res := p.Compose()
now, _, _ := p.Sealed()
if now.Digest == inForce.Digest || slices.ContainsFunc(res.Errors, func(e string) bool { return strings.Contains(e, "composed whole") }) {
t.Fatalf("a refused principal froze the list: sealed %+v, errors %v", res.Sealed, res.Errors)
}
after, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(now.Digest, "b.mesh-controller"))
if !ok || string(after) != string(before) {
t.Errorf("b.mesh-controller did not keep its grants of the list in force: %s, was %s", after, before)
}
// The kept user is written as the composer writes one: the bus reads the same entry it loaded.
if got := entryOf(now, "b.mesh-controller"); entryBefore == "" || got != entryBefore {
t.Errorf("the kept entry is written otherwise than the list in force holds it:\n%s\nwas\n%s", got, entryBefore)
}
if entryOf(now, "a.mesh-controller") != "" {
t.Error("a refused principal the list in force did not carry is in the sealed list")
}
if _, _, ok, _ := b.ownGet(grants.StateUserList, grants.SealedUserKey(now.Digest, "a.mesh-controller")); ok {
t.Error("a refused principal the list in force did not carry is in the list")
}
if len(res.PrincipalsRefused) != 2 || len(said) == 0 || said[len(said)-1].State != asks.StateRaised ||
!strings.Contains(said[len(said)-1].Summary, "b.mesh-controller (keeps its grants of the list in force)") ||
!strings.Contains(said[len(said)-1].Summary, "a.mesh-controller (left out of the list") {
t.Fatalf("the refusals were not said by name: %v, said %+v", res.PrincipalsRefused, said)
}
// Composing again changes nothing and says nothing new; once the manifest composes, the warning is cleared.
n := len(said)
if p.Compose(); len(said) != n {
t.Errorf("an unchanged refusal was said again: %+v", said[n:])
}
b.manifestOf("mesh-controller", ctl)
if res := p.Compose(); len(res.PrincipalsRefused) != 0 || said[len(said)-1].State != asks.StateCleared {
t.Errorf("the refusal was not cleared once it composes: %v, said %+v", res.PrincipalsRefused, said)
}
})
}
type fakeAsker struct {
+130 -23
View File
@@ -82,12 +82,26 @@ func GrantWaits(tookOver bool, accounts []string, composed, loaded map[string]Us
return "waits for the bus to load a list carrying what it adds: " + strings.Join(adds, ", ")
}
// KeySealedGrants is the key of the state user-list holding, per list the part sealed (the last two, by digest), each
// user's permissions as sealed, never a hash: what tells which grants the list the bus loaded carries, without the bus
// module's tools (issue 774).
// **The permissions of each list the part sealed, one key per user** (issue 774, sweep 2026-10-30 item 5): the last two
// lists, by digest, each user's permissions as sealed, never a hash: what tells which grants the list the bus loaded
// carries, without the bus module's tools. One value holding every user of two lists outgrew the bucket's largest value
// (JetStream 10054, "message size exceeds maximum", 16:37 and 16:55 on 2026-10-30), so a list's permissions were never
// kept and a grant judged against them waited. Each user's permissions are a key of their own,
// `sealed-grants.<digest hex>.<user>`, which grows with that user alone; the key `sealed-lists` names the lists kept and
// their users, and is written only once every user's key of its newest list is: it never names a list whose keys are not
// all there.
// KeySealedGrants is the key that held every user of the last two lists in one value, before this release: read when the
// index names no list, and taken away once the index is written.
const KeySealedGrants = "sealed-grants"
// SealedGrants is the value of KeySealedGrants.
// KeySealedLists is the key of the state user-list naming the sealed lists whose permissions are kept, newest first.
const KeySealedLists = "sealed-lists"
// sealedKept is how many sealed lists keep their permissions: the one in force and the one before.
const sealedKept = 2
// SealedGrants is the value KeySealedGrants held.
type SealedGrants struct {
Lists []SealedList `json:"lists"`
}
@@ -98,7 +112,24 @@ type SealedList struct {
Users []User `json:"users"`
}
// keepSealedGrants records the permissions of the list sealed now, beside the one before.
// SealedLists is the value of KeySealedLists: the lists kept, newest first, each by its digest and its users' names.
type SealedLists struct {
Lists []SealedNames `json:"lists"`
}
// SealedNames is one kept list: its digest and its users, sorted.
type SealedNames struct {
Digest string `json:"digest"`
Users []string `json:"users"`
}
// SealedUserKey is the key of one user's permissions in the sealed list of digest.
func SealedUserKey(digest, user string) string {
return KeySealedGrants + "." + strings.TrimPrefix(digest, "sha256:") + "." + user
}
// keepSealedGrants records the permissions of the list sealed now, one key per user, beside the list before; the lists
// older than those two lose their keys.
func (p *Part) keepSealedGrants(res *Result, digest string, list []byte, composed []User) {
in := listUsers(list)
var users []User
@@ -108,20 +139,98 @@ func (p *Part) keepSealedGrants(res *Result, digest string, list []byte, compose
users = append(users, u)
}
}
var was SealedGrants
if raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants); err == nil && found {
_ = json.Unmarshal(raw, &was)
failed := false
names := make([]string, 0, len(users))
for _, u := range users {
names = append(names, u.User)
data, _ := json.Marshal(u)
if _, err := p.Store.Put(StateUserList, SealedUserKey(digest, u.User), data); err != nil {
failed = true
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, SealedUserKey(digest, u.User), err))
}
}
next := SealedGrants{Lists: []SealedList{{Digest: digest, Users: users}}}
if failed {
// The index keeps naming the lists whose keys are all there; this list's permissions are not kept.
return
}
sort.Strings(names)
was := p.sealedLists()
next := SealedLists{Lists: []SealedNames{{Digest: digest, Users: names}}}
var gone []SealedNames
for _, l := range was.Lists {
if l.Digest != digest && len(next.Lists) < 2 {
switch {
case l.Digest == digest:
case len(next.Lists) < sealedKept:
next.Lists = append(next.Lists, l)
default:
gone = append(gone, l)
}
}
data, _ := json.Marshal(next)
if _, err := p.Store.Put(StateUserList, KeySealedGrants, data); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeySealedGrants, err))
if _, err := p.Store.Put(StateUserList, KeySealedLists, data); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateUserList, KeySealedLists, err))
return
}
for _, l := range gone {
for _, user := range l.Users {
if err := p.Store.Delete(StateUserList, SealedUserKey(l.Digest, user)); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be taken away: %v", StateUserList, SealedUserKey(l.Digest, user), err))
}
}
}
if _, _, found, err := p.Store.Get(StateUserList, KeySealedGrants); err == nil && found {
if err := p.Store.Delete(StateUserList, KeySealedGrants); err != nil {
res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be taken away: %v", StateUserList, KeySealedGrants, err))
}
}
}
// sealedLists is the index of kept lists; none when it cannot be read.
func (p *Part) sealedLists() SealedLists {
var out SealedLists
if raw, _, found, err := p.Store.Get(StateUserList, KeySealedLists); err == nil && found {
_ = json.Unmarshal(raw, &out)
}
return out
}
// sealedGrantsOf is the permissions of each user of the sealed list of digest, as the part kept them, and whether it kept
// every one: from the index and each user's key, else from the one value written before this release.
func (p *Part) sealedGrantsOf(digest string) (map[string]User, bool) {
for _, l := range p.sealedLists().Lists {
if l.Digest != digest {
continue
}
out := map[string]User{}
for _, user := range l.Users {
raw, _, found, err := p.Store.Get(StateUserList, SealedUserKey(digest, user))
if err != nil || !found {
return nil, false
}
var u User
if json.Unmarshal(raw, &u) != nil {
return nil, false
}
out[u.User] = u
}
return out, true
}
raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants)
if err == nil && found {
var sg SealedGrants
if json.Unmarshal(raw, &sg) == nil {
for _, sl := range sg.Lists {
if sl.Digest == digest {
out := map[string]User{}
for _, u := range sl.Users {
out[u.User] = u
}
return out, true
}
}
}
}
return nil, false
}
// loadedContents is the list the bus loaded, by user: as the bus module's nats_users read it, else as the part sealed the
@@ -144,16 +253,8 @@ func (p *Part) loadedContents(in Inputs, read []User, readErr error) (map[string
case l.Source != delivery.GrantsUserListSource || l.Follows != delivery.FollowGrants || l.Error != "":
return nil, "the bus loaded no list of the grants part's (source " + l.Source + ", follows " + l.Follows + "), and nats_users: " + readErr.Error()
}
raw, _, found, err := p.Store.Get(StateUserList, KeySealedGrants)
if err == nil && found {
var sg SealedGrants
if json.Unmarshal(raw, &sg) == nil {
for _, sl := range sg.Lists {
if sl.Digest == l.Digest {
return byUser(sl.Users), ""
}
}
}
if kept, ok := p.sealedGrantsOf(l.Digest); ok {
return kept, ""
}
return nil, "the part keeps no permissions of the loaded list " + short(l.Digest) + " (sealed before it kept them), and nats_users: " + readErr.Error()
}
@@ -162,7 +263,7 @@ func (p *Part) loadedContents(in Inputs, read []User, readErr error) (map[string
const GrantMissingCondition = "mesh-grants.grant-missing"
// MissingGrants is each module assigned or placed on a machine that want leaves without a grant, `<module>: <why>`,
// sorted: its manifest refused (skip), no fingerprint of the new form at the indexer's trunk, or no account with a hash
// sorted, but a module the mesh provides itself, which is never wanted by fingerprint: its manifest refused (skip), no fingerprint of the new form at the indexer's trunk, or no account with a hash
// in the vault (drop). **The fingerprint is the indexer's, never a commit's** (issue 1126: link2pay, snake, f1-circuits
// and txt-game, whose source commit has not moved since a build of the old form, have a done key of the new form at it,
// and that is the fingerprint granted). Pure.
@@ -175,6 +276,12 @@ func MissingGrants(in Inputs, want map[string]delivery.ControllerGrant, drop []s
}
var out []string
for _, module := range sortedKeys(placed) {
// A module the mesh provides itself (the private network) is built by no repository, so the indexer holds no
// fingerprint of it and the assigner wants none: the controller composes it with the node's own. It needs no grant,
// and naming it missing raised a warning nothing could clear (sweep 2026-10-30 item 6).
if delivery.ProvidedByMesh(module) {
continue
}
fp, has := in.Fingerprints[module]
key, err := delivery.PartGrantKey(module, fp)
if has && err == nil {
+19 -7
View File
@@ -131,9 +131,12 @@ type Result struct {
// Grants is what the pass wrote of its grants (grants.go).
Grants *GrantsWritten `json:"grants,omitempty"`
// Withheld is what the pass wrote of the hearings it withholds (withheld.go).
Withheld *WithheldWritten `json:"withheld,omitempty"`
Errors []string `json:"errors,omitempty"`
PublishedOrAsserted string `json:"published_or_asserted"`
Withheld *WithheldWritten `json:"withheld,omitempty"`
// PrincipalsRefused is each principal whose composed grants the composer refused, `<user>: <why>`: it keeps its grants
// of the list in force, or stays out where it holds none (refused.go); the condition mesh-grants.principal-refused says it.
PrincipalsRefused []string `json:"principals_refused,omitempty"`
Errors []string `json:"errors,omitempty"`
PublishedOrAsserted string `json:"published_or_asserted"`
}
// NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships.
@@ -186,6 +189,9 @@ type Part struct {
// differSaid is the account consumers' differences last said as a condition, once differKnown (consumers.go).
differSaid string
differKnown bool
// refusedSaid is the refused principals last said as a condition, once refusedKnown (refused.go).
refusedSaid string
refusedKnown bool
// missingSaid is the modules without a grant last said as a condition, once missingKnown (grants.go).
missingSaid string
missingKnown bool
@@ -251,11 +257,17 @@ func (p *Part) Compose() Result {
if st.LeftOut == nil {
st.LeftOut = []Gap{}
}
users, err := ComposeUsers(listRecs)
users, refused, err := ComposeEach(listRecs)
if err != nil {
res.Errors = append(res.Errors, "the shadow list could not be composed: "+err.Error())
}
users = WithHashes(users, in.Hashes)
// A refused principal keeps its grants of the list in force, said once per change (refused.go).
kept, _ := p.keptOf(refused)
for _, user := range sortedKeys(refused) {
res.PrincipalsRefused = append(res.PrincipalsRefused, user+": "+refused[user])
}
p.sayPrincipalsRefused(refused, kept)
users = WithHashes(withKept(users, kept), in.Hashes)
st.Digest, st.Users = Digest(users), len(users)
loaded, loadErr := LoadedUsers(p.Call)
st.Compare = Compare(users, loaded, Placed(in)).Excusing(ExcusedBy(MultiHeld(listRecs)))
@@ -289,7 +301,7 @@ func (p *Part) Compose() Result {
if p.Store != nil {
p.writeWithheld(&res, in, recs)
}
list, n, leftOut, err := ComposeList(listRecs, in.Hashes)
list, n, leftOut, dropped, err := ComposeList(listRecs, in.Hashes, refused, kept)
// The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5); after take-over
// one that adds a permission only once the bus loaded a list that carries it (issue 774).
if p.Store != nil {
@@ -306,7 +318,7 @@ func (p *Part) Compose() Result {
bus = names(loaded)
lost = holdings.LastHolders(holdings.ByUsers(loaded), holdings.ByUsers(users))
}
p.writeSealed(&res, list, n, leftOut, in, bus, lost, users)
p.writeSealed(&res, list, n, leftOut, dropped, in, bus, lost, users)
}
ms := Memberships(recs)
res.Memberships = len(ms)
@@ -0,0 +1,137 @@
package grants
import (
"fmt"
"slices"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
// **One refused principal never freezes the list** (issue 774, sweep 2026-10-30; met twice that day: the engine's acts,
// then mesh-host_health). The composer refuses a principal's grants when they would make it a second writer of a state
// (buscompose.CheckWriters): that refusal used to end the whole composition, so the list in force stood for every principal
// while each change since piled up, and the next list that composed moved 52 users at once. Now each principal is composed
// on its own: one refused keeps the grants it holds in the list in force (the part's own record of that list, sealed
// grants), the rest of the list is composed and written as ever, and the refusal is raised as the warning
// mesh-grants.principal-refused, naming each principal and why, until it composes again. A refused principal the list in
// force does not carry stays out of the list: it never had those grants, and is refused them now.
// RefusedCondition is the key of the warning raised while a principal's grants are refused.
const RefusedCondition = "mesh-grants.principal-refused"
// ComposeEach is the shadow list composed principal by principal: every user whose grants compose, and each refused one
// by user with why. err is only for principals that cannot be derived at all (two users of one name). Pure.
func ComposeEach(r buscompose.Records) ([]User, map[string]string, error) {
principals, err := buscompose.Users(r)
if err != nil {
return nil, nil, err
}
out := make([]User, 0, len(principals))
refused := map[string]string{}
for _, p := range principals {
perms, err := buscompose.PermissionsFor(p)
if err != nil {
refused[p.Username()] = err.Error()
continue
}
out = append(out, User{User: p.Username(), Publish: set(perms.Publish), Subscribe: set(perms.Subscribe),
AllowResponses: perms.AllowResponses, PublishDeny: set(perms.PublishDeny)})
}
sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User })
return out, refused, nil
}
// keptOf is, for each refused user, its grants in the list in force as the part sealed them; the users it holds none of
// (not in the list in force, or that list's grants not kept) are the second answer, sorted.
func (p *Part) keptOf(refused map[string]string) (map[string]User, []string) {
if len(refused) == 0 {
return nil, nil
}
var inForce map[string]User
if g, _, had := p.Sealed(); had && g.Digest != "" && p.Store != nil {
inForce, _ = p.sealedGrantsOf(g.Digest)
}
kept := map[string]User{}
var none []string
for _, user := range sortedKeys(refused) {
if u, ok := inForce[user]; ok {
kept[user] = u
continue
}
none = append(none, user)
}
return kept, none
}
// withKept is users with each kept user in its place, sorted.
func withKept(users []User, kept map[string]User) []User {
out := slices.Clone(users)
for _, u := range kept {
out = append(out, u)
}
sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User })
return out
}
// keptEntry is a user of the accounts block from grants kept rather than composed, in ComposeAccounts' own lines
// (buscompose.AccountsEntry), so the bus reads it as it read the list in force.
func keptEntry(u User, hash string) string {
return buscompose.AccountsEntry(u.User, hash, buscompose.Permissions{Publish: u.Publish, PublishDeny: u.PublishDeny,
Subscribe: u.Subscribe, AllowResponses: u.AllowResponses})
}
// withEntries puts entries at the end of the accounts block's users, before it closes.
func withEntries(list string, entries []string) (string, error) {
if len(entries) == 0 {
return list, nil
}
start := strings.Index(list, " users = [\n")
if start < 0 {
return "", fmt.Errorf("the composed list holds no users block to keep a refused user's grants in")
}
end := strings.Index(list[start:], "\n ]\n")
if end < 0 {
return "", fmt.Errorf("the composed list's users block does not close")
}
at := start + end + 1
return list[:at] + strings.Join(entries, "") + list[at:], nil
}
// sayPrincipalsRefused raises RefusedCondition while a principal's grants are refused, naming each with why and what
// became of it, and clears it once none is; said once per set, again when it changes, and at the first pass after a start.
func (p *Part) sayPrincipalsRefused(refused map[string]string, kept map[string]User) {
if p.Say == nil {
return
}
var lines []string
for _, user := range sortedKeys(refused) {
became := "left out of the list: the list in force does not carry it"
if _, ok := kept[user]; ok {
became = "keeps its grants of the list in force"
}
lines = append(lines, user+" ("+became+"): "+refused[user])
}
why := strings.Join(lines, "; ")
p.holdMu.Lock()
defer p.holdMu.Unlock()
if p.refusedKnown && why == p.refusedSaid {
return
}
c := asks.Condition{Key: RefusedCondition, Subject: asks.Scope{Scope: "module", ID: "mesh-grants"}}
if why == "" {
c.State = asks.StateCleared
} else {
c.Severity, c.State = asks.SeverityWarning, asks.StateRaised
c.Summary = fmt.Sprintf("%d principal(s) refused their composed grants: %s", len(refused), why)
c.Headline, c.Needs = "A principal's bus grants are refused", "a fix of the manifest or the writers table"
c.Explanation = asks.NeedsYou + "a fix of what makes it a second writer. The rest of the user list is written as ever; " +
"each refused principal keeps the grants the list in force gives it, or stays out where it holds none: " + why
c.Resolved = "every principal's grants compose again"
}
if p.Say(c) == nil {
p.refusedSaid, p.refusedKnown = why, true
}
}
+40 -12
View File
@@ -31,24 +31,41 @@ type Objects interface {
DeleteObject(name string) error
}
// ComposeList is the bus's user list as the file holds it, with each user's hash from the vault, and the users left out
// for want of one. Pure.
func ComposeList(r buscompose.Records, hashes map[string]HeldHash) ([]byte, int, []string, error) {
// ComposeList is the bus's user list as the file holds it, with each user's hash from the vault; the users left out
// for want of one; and the refused principals (refused, by user) the list carries with their grants kept from the list
// in force (kept), or leaves out where kept holds none (dropped, sorted). Pure.
func ComposeList(r buscompose.Records, hashes map[string]HeldHash, refused map[string]string, kept map[string]User) (list []byte, users int, leftOut, dropped []string, err error) {
principals, err := buscompose.Users(r)
if err != nil {
return nil, 0, nil, err
return nil, 0, nil, nil, err
}
held := map[string]string{}
for user, h := range hashes {
held[user] = h.Hash
}
filled, missing := buscompose.WithPasswords(principals, held)
list, err := buscompose.ComposeAccounts(filled)
composing := slices.DeleteFunc(slices.Clone(principals), func(p buscompose.Principal) bool { _, no := refused[p.Username()]; return no })
filled, missing := buscompose.WithPasswords(composing, held)
text, err := buscompose.ComposeAccounts(filled)
if err != nil {
return nil, 0, nil, err
return nil, 0, nil, nil, err
}
var entries []string
for _, user := range sortedKeys(refused) {
u, ok := kept[user]
switch {
case !ok:
dropped = append(dropped, user)
case held[user] == "":
missing = append(missing, user)
default:
entries = append(entries, keptEntry(u, held[user]))
}
}
if text, err = withEntries(text, entries); err != nil {
return nil, 0, nil, nil, err
}
slices.Sort(missing)
return []byte(list), len(filled), missing, nil
return []byte(text), len(filled) + len(entries), missing, dropped, nil
}
// Sealed is the part's key `user-list.sealed` as written, and whether there is one.
@@ -74,7 +91,7 @@ func followOf(state string) string {
// writeSealed puts the list sealed to the bus node and writes the key, each only when what it says changed. It says in
// res what it wrote, or why it wrote nothing.
func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string, in Inputs, bus []string, lost []string, composed []User) {
func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut, dropped []string, in Inputs, bus []string, lost []string, composed []User) {
cur, _, had := p.Sealed()
next := cur
next.Follow = followOf(p.SwitchState())
@@ -93,10 +110,10 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut []string
why = "no node runs the bus's module, so the list is sealed to nobody"
case key == "":
why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)"
case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, bus, lost, digest) != "":
case next.Follow == delivery.FollowGrants && cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, digest) != "":
// After take-over the list in force is never replaced by one composed past a failed read, or one that drops a user
// of it for want of a hash: that user's module would be locked out of the bus (review of the switch).
why = p.holdsBack(in, leftOut, bus, lost, digest)
why = p.holdsBack(in, leftOut, dropped, bus, lost, digest)
held = why
p.keepHashHold(res, digest)
case cur.Digest != digest || cur.SealedTo != delivery.SealingKeyID(key) || cur.Node != in.BusNode:
@@ -356,7 +373,7 @@ func listUsers(list []byte) []string {
// loaded list not read holds it too, since that cannot be ruled out). The list in force is the one the part wrote, by the users
// it kept of it; where it kept none (a list written before this release), the users the bus loaded (bus, nil when
// they could not be read). Empty when it may.
func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string, digest string) string {
func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest string) string {
p.hashHold = nil
if f := in.ReadFailures(); len(f) > 0 {
return "held: the list in force is kept while a read of this pass failed (" + strings.Join(f, "; ") + ")"
@@ -387,6 +404,17 @@ func (p *Part) holdsBack(in Inputs, leftOut, bus, lost []string, digest string)
if len(unhashed) > 0 {
return "held: the list in force is kept, since " + strings.Join(unhashed, ", ") + " would be left out for want of a hash in the vault"
}
// A refused principal the list in force carries and whose grants there the part did not keep: leaving it out would lock
// it out of the bus (refused.go), so the list in force stands until its grants compose or are kept.
var unkept []string
for _, u := range dropped {
if slices.Contains(had, u) {
unkept = append(unkept, u)
}
}
if len(unkept) > 0 {
return "held: the list in force is kept, since " + strings.Join(unkept, ", ") + " would be left out: its grants are refused and the part kept none of the list in force"
}
// More than one loaded user's hash changing at once, with no rotation by the vault naming each (ADR 0561 §7): a new
// writer of hashes must not lock several holders out in one list, as nine seats were cut on 2026-10-27 at 09:21.
changed, err := p.changedHashes(in.Hashes)
@@ -116,23 +116,18 @@ func CredentialsOf(shadow, loaded []User, hashes map[string]HeldHash) Credential
return c
}
// ComposeUsers is the shadow list: every user the composer derives from the records, with its grants. Pure.
// ComposeUsers is the shadow list: every user the composer derives from the records, with its grants; an error when any
// principal's grants are refused (ComposeEach composes past one). Pure.
func ComposeUsers(r buscompose.Records) ([]User, error) {
principals, err := buscompose.Users(r)
users, refused, err := ComposeEach(r)
if err != nil {
return nil, err
}
out := make([]User, 0, len(principals))
for _, p := range principals {
perms, err := buscompose.PermissionsFor(p)
if err != nil {
return nil, fmt.Errorf("%s: %w", p.Username(), err)
}
out = append(out, User{User: p.Username(), Publish: set(perms.Publish), Subscribe: set(perms.Subscribe),
AllowResponses: perms.AllowResponses, PublishDeny: set(perms.PublishDeny)})
if len(refused) > 0 {
first := sortedKeys(refused)[0]
return nil, fmt.Errorf("%s: %s", first, refused[first])
}
sort.Slice(out, func(i, j int) bool { return out[i].User < out[j].User })
return out, nil
return users, nil
}
func set(in []string) []string {