novox/mesh-delivery
10
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mesh-grants: place by the node-engines' haves while the assignments view does not answer

Browse Source 
Bootstrap only (issue 1966, notes 63 and 64; the operator chose it 2026-11-07 22:19,
bending ADR 0567 §9 for the bootstrap). An assigner that serves no first views answers
no assignments, so the grants part placed nothing and wrote no grant, membership or
bucket for anything, not even for the assigner that would serve the view.

When the assignments read fails and the haves read, each module is placed on the node
whose have says it runs (any phase but removed), the bus node found among them, and what
runs composed from each have's fingerprint instead of the assigner's wants. Said as a gap
naming the fallback, and kept as Inputs.ByHaves: such a pass retires no account and deletes
no withheld hearing (ReadFailures), and seals no list that drops a user of the list in
force or of the loaded list (holdsBack). Once the view answers, the fallback is unused.

Quick fix; the thorough one stays the module-graph edge (the assigner's views before the
grants part that reads them) in the delivery plan.

Claude-Session: https://claude.ai/code/session_01NcbgfFfQ4Z9UE5jK32PmE5
This commit is contained in:
jschoubben
2026-11-07 22:37:56 +01:00
parent b73315ffe0
commit 9756dd6f1f
3 changed files with 212 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-grants/grants_test.go
+88 -7
View File
@@ -633,8 +633,9 @@ func TestGrants(t *testing.T) {
key, _ := builds.BuilderBuildKey("mod-x", fp)
b.put(grants.ReadBuilds, key, map[string]any{"v": keys.Version, "state": "done", "manifest": digest})
}, []string{"a.mod-x", "a.nats", "b.mod-y", "node.a", "node.b"}, "people"},
{"assignments refused is a gap; the machines come from the haves", func(b *fakeBus) { b.refused[grants.ReadAssignments] = true },
[]string{"node.a", "node.b"}, grants.ReadAssignments},
// Issue 1966: refused, each module is placed where its have says it runs (the bootstrap fallback); a has nats.
{"assignments refused is a gap; the machines and the modules come from the haves", func(b *fakeBus) { b.refused[grants.ReadAssignments] = true },
[]string{"a.nats", "node.a", "node.b"}, grants.ReadAssignments},
} {
t.Run(row.name, func(t *testing.T) {
b := fixture()
@@ -1310,10 +1311,17 @@ func TestGrants(t *testing.T) {
t.Errorf("read: assigned %v, placed %v, gaps %v", read.Assigned, read.Placed, read.Gaps)
}
b.refused[grants.ReadAssignments] = true
// The assignments alone places (issue 774, 774/S29): not read, nothing is placed, and it is a failed read.
// The assignments alone places (issue 774, 774/S29), but for the bootstrap (issue 1966): not read, a module is placed
// where its have says it runs, said as a gap that is no failed read; with the haves refused too nothing is placed,
// and it is a failed read.
if read := grants.ReadInputs(b, &grants.Manifests{}); !slices.ContainsFunc(read.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadAssignments && !g.Failed }) ||
!slices.Equal(read.Assigned["a"], []string{"nats"}) || len(read.Assigned) != 1 || read.ByHaves == "" || read.BusNode != "a" {
t.Errorf("assignments not on the bus is not placed by the haves: %v, %v", read.Gaps, read.Assigned)
}
b.refused[grants.ReadHave] = true
if read := grants.ReadInputs(b, &grants.Manifests{}); !slices.ContainsFunc(read.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadAssignments && g.Failed }) ||
len(read.Assigned) != 0 {
t.Errorf("assignments not on the bus is no failed read, or something is placed: %v, %v", read.Gaps, read.Assigned)
len(read.Assigned) != 0 || read.ByHaves != "" {
t.Errorf("assignments and haves not on the bus is no failed read, or something is placed: %v, %v", read.Gaps, read.Assigned)
}
// Issue 1945, the user list of 2026-10-27 09:21: the assignments' holds are the holders on record, not every holder.
// A mesh seat no assignments holds keeps the module claiming it as its holder, wherever assignments puts it.
@@ -2343,6 +2351,76 @@ func TestGrants(t *testing.T) {
}
})
// Issue 1966 (the registry delivery's second bootstrap loop): an assigner that serves no first views answers no
// assignments, so nothing was placed and no grant written, not even for the assigner that would serve them. While the
// view does not answer, each module is placed where its have says it runs: the grants are written at the indexer's
// trunk fingerprints, the memberships published, the buckets a placed module declares created and the list sealed; but
// no account is retired and no list sealed that drops a user of the list in force. Once the view answers, unused.
t.Run("bootstrap: the assignments view unanswered places by the haves, losing no login (issue 1966)", func(t *testing.T) {
b := fixture()
b.refused[grants.ReadAssignments] = true
b.manifestOf("mesh-delivery-assigner", `{"module":"mesh-delivery-assigner","version":"1","own-secrets":{"broker":"/var/lib/a/broker"},"state":[{"name":"assignments"}]}`)
b.manifestOf("mesh-delivery-builder", `{"module":"mesh-delivery-builder","version":"1","own-secrets":{"broker":"/var/lib/b/broker"},"state":[{"name":"builds"}]}`)
runs := nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
b.put(grants.ReadHave, "a.mesh-delivery-assigner", runs)
b.put(grants.ReadHave, "b.mesh-delivery-builder", runs)
removed := runs
removed.Phase = nodeengineclient.PhaseRemoved
b.put(grants.ReadHave, "b.mod-y", removed)
busKey, err := seal.GenerateKeyPair()
if err != nil {
t.Fatal(err)
}
b.put(grants.ReadHave, "a.nats", nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK,
UserList: &nodeengine.UserListWritten{Source: grantsstate.GrantsUserListSource, Revision: 8, Wanted: "sha256:aa", Digest: "sha256:aa", Users: 5,
Key: busKey.PublicKey, Follows: grantsstate.FollowGrants}})
hash := func(user string) {
b.put(grants.ReadAccounts, user, map[string]any{"user": user, "hash": "$2a$10$x", "generation": 1})
}
b.loaded = shadowOf(t, b)
for _, u := range b.loaded {
hash(u.User)
}
// The account of a user nothing composes now: a pass placed by the haves never retires it.
b.own[grants.StateAccounts] = map[string][]byte{"c.old": []byte(`{}`)}
p := partOn(b)
p.Create, p.Objects = b, &fakeObjects{}
p.Bus = grants.BusAddress{URL: "nats://a.mesh:4222", Fingerprint: "ab"}
res := p.Compose()
for _, m := range []string{"mesh-delivery-assigner", "mesh-delivery-builder"} {
key, _ := grantsstate.PartGrantKey(m, "src2:"+strings.Repeat("ab", 32))
if _, ok := b.own[grants.StateGrants][key]; !ok {
t.Fatalf("no grant of %s placed by its have: %+v, gaps %+v", m, res.Grants, res.UserList.Gaps)
}
}
if !slices.Contains(b.creates, "KV_mesh-delivery-assigner_assignments") || !slices.Contains(b.creates, "KV_mesh-delivery-builder_builds") {
t.Errorf("the placed modules' buckets were not created: %v (%v)", b.creates, res.NotAsserted)
}
if !slices.Contains(res.Published, "a.mesh-delivery-assigner") || !slices.Contains(res.Published, "b.mesh-delivery-builder") ||
slices.Contains(res.Published, "b.mod-y") {
t.Errorf("memberships published %v (a removed have places nothing)", res.Published)
}
if sealed, _, _ := p.Sealed(); sealed.Digest == "" || sealed.Node != "a" {
t.Errorf("no list sealed to the bus node the haves name: %+v, %+v", sealed, res.Sealed)
}
if b.own[grants.StateAccounts]["c.old"] == nil || res.Accounts == nil ||
!slices.Contains(res.Accounts.Kept, "c.old: a read of this pass failed, so no account is retired") {
t.Errorf("a pass placed by the haves retired an account: %+v", res.Accounts)
}
// A user of the loaded list the haves do not compose: the list composed now is held, never sealed without it.
b.loaded = append(b.loaded, grants.User{User: "c.old"})
b.put(grants.ReadHave, "b.mod-y", runs)
hash("b.mod-y")
if res := p.Compose(); res.Sealed == nil || !strings.Contains(res.Sealed.Why, "would drop c.old of the list in force") {
t.Errorf("a list dropping a loaded user was not held: %+v", res.Sealed)
}
// The view answers: the assignments alone places again.
delete(b.refused, grants.ReadAssignments)
if in := grants.ReadInputs(b, &grants.Manifests{}); in.ByHaves != "" || slices.Contains(in.Assigned["a"], "mesh-delivery-assigner") {
t.Errorf("an answering assignments view still placed by the haves: %q, %v", in.ByHaves, in.Assigned)
}
})
// ADR 0590 clause 5's test for this repository (issue 1126): no module.json here names mesh-controller, the lease
// read and the builder's consume of artifacts-gone included. Read as text, as mesh-catalog's controllerdeps check reads
// its own: mesh-sdk names no module, so it holds no such check (issue 774, the operator 2026-11-03).
@@ -2653,8 +2731,11 @@ func TestGrants(t *testing.T) {
if !raised(grants.FirstViewUnanswered+".mesh-delivery-indexer.verdicts", occ.StateRaised) {
t.Fatalf("the refused raise was not said again at the pass: %+v", said)
}
if !slices.ContainsFunc(before.UserList.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadAssignments && g.Failed }) {
t.Fatalf("assignments answered empty at revision 0 was not a failed read: %+v", before.UserList.Gaps)
// Not answered above revision 0, the modules are placed by the node-engines' haves (issue 1966), said as its gap.
if !slices.ContainsFunc(before.UserList.Gaps, func(g grants.Gap) bool {
return g.Input == grants.ReadAssignments && strings.Contains(g.Why, "placed by the node-engines' haves")
}) {
t.Fatalf("assignments answered empty at revision 0 was not placed by the haves: %+v", before.UserList.Gaps)
}
if !raised(grants.FirstViewEmpty+".mesh-delivery-assigner.assignments", occ.StateRaised) {
t.Fatalf("no first-view-empty: %+v", said)
modules/mesh-grants/internal/grants/inputs.go
+105 -4
View File
@@ -167,6 +167,10 @@ type Inputs struct {
// Refused is every manifest the indexer holds that the part cannot read or mesh-sdk's reader refuses, assigned or
// not, each with why: its module counts as placed nowhere, so it is said loudly, never passed over (issue 1945).
Refused []Gap
// ByHaves says why this pass placed the modules where the node-engines' haves say they run, the assignments view not
// answering (bootstrap only, issue 1966); empty when the assignments placed them. Such a pass retires nothing
// (ReadFailures) and seals no list dropping a user of the list in force (holdsBack).
ByHaves string
// Held is, by module, the manifest a held module's running build was judged on (its have's fingerprint, the builder's
// build of it): what the seats it defines, the streams others hear from it and its reads are composed from while its
// trunk manifest is refused or waits, so no other principal loses a grant to it (ADR 0601 clause 3).
@@ -321,9 +325,8 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
// **The seat's assignments alone places modules** (novox/hq ADR 0549 §5, ADR 0567 §9; issue 774): the controller's
// assignments are read no more, and the seats held once for the mesh are held where it says. Not read is a failed
// read: nothing is placed, so the list composed past it is not sealed.
if keys, err := r.Keys(ReadAssignments); err != nil {
in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "refused or absent: nothing is placed (" + err.Error() + ")", Failed: true})
} else {
keys, assignErr := r.Keys(ReadAssignments)
if assignErr == nil {
// Each node's keys in either form, assembled by the seat's one reader (meshmoduleassignerclient.AssignmentsOf; research 072
// row 37): its node key and a key per module, or its legacy key `assignments.<node>`.
values := map[string][]byte{}
@@ -367,6 +370,23 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
if err != nil {
in.Gaps = append(in.Gaps, Gap{Input: ReadHave, Why: "refused or absent: " + err.Error(), Failed: true})
}
switch {
case assignErr == nil:
case err != nil:
in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "refused or absent: nothing is placed (" + assignErr.Error() + ")", Failed: true})
default:
// **Bootstrap only** (issue 1966, the operator 2026-11-07 22:19, bending ADR 0567 §9 for it): while the assignments
// view does not answer (an assigner that serves no first views yet), each module is placed where a node-engine's have
// says it runs, so the grants of the assigner that serves the view can be written. Said as a gap naming the
// fallback, and kept as ByHaves: no account is retired and no hearing deleted in such a pass (ReadFailures), and the
// list is sealed only when it keeps every user of the list in force (holdsBack). Once the view answers, unused.
in.ByHaves = "the assignments view did not answer (" + assignErr.Error() + "), so each module is placed where a node-engine's have says it runs"
in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "refused or absent: placed by the node-engines' haves instead, until it answers (" + assignErr.Error() + ")"})
placeByHaves(r, &in, haveKeys)
for node := range in.Assigned {
nodes[node] = true
}
}
for _, k := range haveKeys {
if node, _, isNode, err := nodeengineclient.ParseHaveKey(k); err == nil && isNode {
nodes[node] = true
@@ -488,7 +508,16 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
}
// What each machine is wanted to run: the want's fingerprint, the builder's build of it, the manifest it built.
var fromTrunk []string
for key, raw := range each(r, ReadWants, &in.Gaps) {
wants := map[string][]byte{}
if in.ByHaves == "" {
wants = each(r, ReadWants, &in.Gaps)
} else {
// Placed by the haves, what runs is what each have's fingerprint names, not a want: the assigner's wants are its
// view too, and not read in such a pass.
in.Gaps = append(in.Gaps, Gap{Input: ReadWants, Why: "not read: the modules are placed by the node-engines' haves this pass, so what runs is composed from each have's fingerprint"})
fromTrunk = runningByHaves(r, &in, cache)
}
for key, raw := range wants {
node, module, err := meshwantsclient.ParseWantKey(key)
if err != nil || !slices.Contains(in.Assigned[node], module) {
continue
@@ -671,12 +700,84 @@ func (in Inputs) ReadFailed() []string {
// keeps, the hearings it withholds).
func (in Inputs) ReadFailures() []string {
out := in.ReadFailed()
if in.ByHaves != "" {
out = append(out, ReadAssignments+": "+in.ByHaves)
}
for _, g := range in.Refused {
out = append(out, g.Input+": "+g.Why)
}
return out
}
// placeByHaves places each module on the node whose node-engine's have says it runs (any phase but removed), and finds the
// bus node among them: the bootstrap placement while the assignments view does not answer (issue 1966). A have that does
// not read places nothing, said as a gap.
func placeByHaves(r Reader, in *Inputs, haveKeys []string) {
for _, k := range haveKeys {
node, module, isNode, err := nodeengineclient.ParseHaveKey(k)
if err != nil || isNode {
continue
}
raw, found, err := r.Get(ReadHave, k)
if err != nil || !found {
continue
}
h, err := nodeengineclient.DecodeHave(raw)
if err != nil {
in.Gaps = append(in.Gaps, Gap{Input: ReadHave + " " + k, Why: "does not read, so it places nothing: " + err.Error()})
continue
}
if h.Phase == nodeengineclient.PhaseRemoved || slices.Contains(in.Assigned[node], module) {
continue
}
in.Assigned[node] = append(in.Assigned[node], module)
if module == BusModule && (in.BusNode == "" || node < in.BusNode) {
in.BusNode = node
}
}
}
// runningByHaves is, in a pass placed by the haves, the manifest each placed module runs: its have's fingerprint, the
// builder's build of it, the manifest it built; the placed modules with none, `<node>.<module>`, to compose from the trunk.
func runningByHaves(r Reader, in *Inputs, cache *Manifests) []string {
var fromTrunk []string
for node, modules := range in.Assigned {
for _, module := range modules {
raw, found, err := r.Get(ReadHave, node+"."+module)
if err != nil || !found {
fromTrunk = append(fromTrunk, node+"."+module)
continue
}
h, err := nodeengineclient.DecodeHave(raw)
if err != nil || h.Fp == "" {
fromTrunk = append(fromTrunk, node+"."+module)
continue
}
bk, err := builds.BuilderBuildKey(module, h.Fp)
if err != nil {
fromTrunk = append(fromTrunk, node+"."+module)
continue
}
braw, found, err := r.Get(ReadBuilds, bk)
if err != nil || !found {
fromTrunk = append(fromTrunk, node+"."+module)
continue
}
b, err := builds.DecodeBuilderBuild(braw)
if err != nil || b.Manifest == "" {
fromTrunk = append(fromTrunk, node+"."+module)
continue
}
if m, err := cache.get(r, b.Manifest); err == nil {
in.Running[node+"."+module] = m
} else {
fromTrunk = append(fromTrunk, node+"."+module)
}
}
}
return fromTrunk
}
// heldManifests are the manifests the running builds of the held modules (in.Refused) were judged on, by module: the have
// of the module on a node names the fingerprint that runs, and the builder's build of it the manifest. A module that runs
// nowhere, or whose build or manifest cannot be read, has none, and that is said as a gap.
modules/mesh-grants/internal/grants/sealed.go
+19 -3
View File
@@ -104,10 +104,10 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut, dropped
why = "no node runs the bus's module, so the list is sealed to nobody"
case key == "":
why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)"
case cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, digest) != "":
case cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, listUsers(list), digest) != "":
// The list in force is never replaced by one composed past a failed read, or one that drops a user of it for want
// of a hash: that user's module would be locked out of the bus (review of the switch).
why = p.holdsBack(in, leftOut, dropped, bus, lost, digest)
why = p.holdsBack(in, leftOut, dropped, bus, lost, listUsers(list), digest)
held = why
p.keepHashHold(res, digest)
case cur.Digest != digest || cur.SealedTo != grantsstate.SealingKeyID(key) || cur.Node != in.BusNode:
@@ -298,7 +298,11 @@ func listUsers(list []byte) []string {
// loaded list not read holds it too, since that cannot be ruled out). The list in force is the one the part wrote, by the users
// it kept of it; where it kept none (a list written before this release), the users the bus loaded (bus, nil when
// they could not be read). Empty when it may.
func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest string) string {
//
// In a pass placed by the node-engines' haves (in.ByHaves, issue 1966) the list is held, too, while it would leave out any user
// of the list in force or of the list the bus loaded (listed are the users of the list composed now): a bootstrap placement
// adds logins and never takes one away.
func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost, listed []string, digest string) string {
p.hashHold = nil
if f := in.ReadFailed(); len(f) > 0 {
return "held: the list in force is kept while a read of this pass failed (" + strings.Join(f, "; ") + ")"
@@ -320,6 +324,18 @@ func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest
default:
had = bus
}
if in.ByHaves != "" {
var gone []string
for _, u := range append(slices.Clone(had), bus...) {
if !slices.Contains(listed, u) && !slices.Contains(gone, u) {
gone = append(gone, u)
}
}
if len(gone) > 0 {
slices.Sort(gone)
return "held: " + in.ByHaves + ", and the list composed so would drop " + strings.Join(gone, ", ") + " of the list in force"
}
}
var unhashed []string
for _, u := range leftOut {
if slices.Contains(had, u) {