mesh-grants places modules by the node-engines' haves while the assigner's assignments view does not answer (issue 1966, bootstrap recovery) #170

Merged
mesh-tools merged 3 commits from fix/1966-grants-places-by-haves into main 2026-11-07 22:05:11 +00:00
3 changed files with 251 additions and 14 deletions
+127 -7
View File
@@ -633,8 +633,9 @@ func TestGrants(t *testing.T) {
key, _ := builds.BuilderBuildKey("mod-x", fp)
b.put(grants.ReadBuilds, key, map[string]any{"v": keys.Version, "state": "done", "manifest": digest})
}, []string{"a.mod-x", "a.nats", "b.mod-y", "node.a", "node.b"}, "people"},
{"assignments refused is a gap; the machines come from the haves", func(b *fakeBus) { b.refused[grants.ReadAssignments] = true },
[]string{"node.a", "node.b"}, grants.ReadAssignments},
// Issue 1966: refused, each module is placed where its have says it runs (the bootstrap fallback); a has nats.
{"assignments refused is a gap; the machines and the modules come from the haves", func(b *fakeBus) { b.refused[grants.ReadAssignments] = true },
[]string{"a.nats", "node.a", "node.b"}, grants.ReadAssignments},
} {
t.Run(row.name, func(t *testing.T) {
b := fixture()
@@ -1310,10 +1311,17 @@ func TestGrants(t *testing.T) {
t.Errorf("read: assigned %v, placed %v, gaps %v", read.Assigned, read.Placed, read.Gaps)
}
b.refused[grants.ReadAssignments] = true
// The assignments alone places (issue 774, 774/S29): not read, nothing is placed, and it is a failed read.
// The assignments alone places (issue 774, 774/S29), but for the bootstrap (issue 1966): not read, a module is placed
// where its have says it runs, said as a gap that is no failed read; with the haves refused too nothing is placed,
// and it is a failed read.
if read := grants.ReadInputs(b, &grants.Manifests{}); !slices.ContainsFunc(read.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadAssignments && !g.Failed }) ||
!slices.Equal(read.Assigned["a"], []string{"nats"}) || len(read.Assigned) != 1 || read.ByHaves == "" || read.BusNode != "a" {
t.Errorf("assignments not on the bus is not placed by the haves: %v, %v", read.Gaps, read.Assigned)
}
b.refused[grants.ReadHave] = true
if read := grants.ReadInputs(b, &grants.Manifests{}); !slices.ContainsFunc(read.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadAssignments && g.Failed }) ||
len(read.Assigned) != 0 {
t.Errorf("assignments not on the bus is no failed read, or something is placed: %v, %v", read.Gaps, read.Assigned)
len(read.Assigned) != 0 || read.ByHaves != "" {
t.Errorf("assignments and haves not on the bus is no failed read, or something is placed: %v, %v", read.Gaps, read.Assigned)
}
// Issue 1945, the user list of 2026-10-27 09:21: the assignments' holds are the holders on record, not every holder.
// A mesh seat no assignments holds keeps the module claiming it as its holder, wherever assignments puts it.
@@ -2343,6 +2351,79 @@ func TestGrants(t *testing.T) {
}
})
// Issue 1966 (the registry delivery's second bootstrap loop): an assigner that serves no first views answers no
// assignments, so nothing was placed and no grant written, not even for the assigner that would serve them. While the
// view does not answer, each module is placed where its have says it runs: the grants are written at the indexer's
// trunk fingerprints, the memberships published, the buckets a placed module declares created and the list sealed; but
// no account is retired and no list sealed that drops a user of the list in force. Once the view answers, unused.
t.Run("bootstrap: the assignments view unanswered places by the haves, losing no login (issue 1966)", func(t *testing.T) {
b := fixture()
b.refused[grants.ReadAssignments] = true
b.manifestOf("mesh-delivery-assigner", `{"module":"mesh-delivery-assigner","version":"1","own-secrets":{"broker":"/var/lib/a/broker"},"state":[{"name":"assignments"}]}`)
b.manifestOf("mesh-delivery-builder", `{"module":"mesh-delivery-builder","version":"1","own-secrets":{"broker":"/var/lib/b/broker"},"state":[{"name":"builds"}]}`)
runs := nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
b.put(grants.ReadHave, "a.mesh-delivery-assigner", runs)
b.put(grants.ReadHave, "b.mesh-delivery-builder", runs)
removed := runs
removed.Phase = nodeengineclient.PhaseRemoved
b.put(grants.ReadHave, "b.mod-y", removed)
busKey, err := seal.GenerateKeyPair()
if err != nil {
t.Fatal(err)
}
b.put(grants.ReadHave, "a.nats", nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK,
UserList: &nodeengine.UserListWritten{Source: grantsstate.GrantsUserListSource, Revision: 8, Wanted: "sha256:aa", Digest: "sha256:aa", Users: 5,
Key: busKey.PublicKey, Follows: grantsstate.FollowGrants}})
hash := func(user string) {
b.put(grants.ReadAccounts, user, map[string]any{"user": user, "hash": "$2a$10$x", "generation": 1})
}
b.loaded = shadowOf(t, b)
for _, u := range b.loaded {
hash(u.User)
}
// The account of a user nothing composes now: a pass placed by the haves never retires it.
b.own[grants.StateAccounts] = map[string][]byte{"c.old": []byte(`{}`)}
p := partOn(b)
p.Create, p.Objects = b, &fakeObjects{}
p.Bus = grants.BusAddress{URL: "nats://a.mesh:4222", Fingerprint: "ab"}
res := p.Compose()
for _, m := range []string{"mesh-delivery-assigner", "mesh-delivery-builder"} {
key, _ := grantsstate.PartGrantKey(m, "src2:"+strings.Repeat("ab", 32))
if _, ok := b.own[grants.StateGrants][key]; !ok {
t.Fatalf("no grant of %s placed by its have: %+v, gaps %+v", m, res.Grants, res.UserList.Gaps)
}
}
if !slices.Contains(b.creates, "KV_mesh-delivery-assigner_assignments") || !slices.Contains(b.creates, "KV_mesh-delivery-builder_builds") {
t.Errorf("the placed modules' buckets were not created: %v (%v)", b.creates, res.NotAsserted)
}
if !slices.Contains(res.Published, "a.mesh-delivery-assigner") || !slices.Contains(res.Published, "b.mesh-delivery-builder") ||
slices.Contains(res.Published, "b.mod-y") {
t.Errorf("memberships published %v (a removed have places nothing)", res.Published)
}
if in := grants.ReadInputs(b, &grants.Manifests{}); slices.Contains(in.Assigned["b"], "mod-y") || !slices.Contains(in.Assigned["b"], "mesh-delivery-builder") {
t.Errorf("a removed have placed its module: b runs %v", in.Assigned["b"])
}
if sealed, _, _ := p.Sealed(); sealed.Digest == "" || sealed.Node != "a" {
t.Errorf("no list sealed to the bus node the haves name: %+v, %+v", sealed, res.Sealed)
}
if b.own[grants.StateAccounts]["c.old"] == nil || res.Accounts == nil ||
!slices.Contains(res.Accounts.Kept, "c.old: a read of this pass failed, so no account is retired") {
t.Errorf("a pass placed by the haves retired an account: %+v", res.Accounts)
}
// A user of the loaded list the haves do not compose: the list composed now is held, never sealed without it.
b.loaded = append(b.loaded, grants.User{User: "c.old"})
b.put(grants.ReadHave, "b.mod-y", runs)
hash("b.mod-y")
if res := p.Compose(); res.Sealed == nil || !strings.Contains(res.Sealed.Why, "would drop c.old of the list in force") {
t.Errorf("a list dropping a loaded user was not held: %+v", res.Sealed)
}
// The view answers: the assignments alone places again.
delete(b.refused, grants.ReadAssignments)
if in := grants.ReadInputs(b, &grants.Manifests{}); in.ByHaves != "" || slices.Contains(in.Assigned["a"], "mesh-delivery-assigner") {
t.Errorf("an answering assignments view still placed by the haves: %q, %v", in.ByHaves, in.Assigned)
}
})
// ADR 0590 clause 5's test for this repository (issue 1126): no module.json here names mesh-controller, the lease
// read and the builder's consume of artifacts-gone included. Read as text, as mesh-catalog's controllerdeps check reads
// its own: mesh-sdk names no module, so it holds no such check (issue 774, the operator 2026-11-03).
@@ -2464,6 +2545,42 @@ func TestGrants(t *testing.T) {
if !strings.Contains(refused, "manifest of mod-x: it waits for the seat list") {
t.Errorf("another module is not held: %v", p.Last().Refused)
}
// Issue 1966: the assignments view unanswered, the indexer placed by its have alone (its real manifest, claiming
// mesh-catalog and serving seat-definitions) is composed to serve the seat's verbs, in the list and its membership,
// while the seat list stays unread: the bootstrap reaches the verb that reads the list without opening the gate.
indexer, err := os.ReadFile("../mesh-delivery-indexer/module.json")
if err != nil {
t.Fatal(err)
}
fb := fixture()
fb.refused[grants.ReadAssignments] = true
fb.manifestOf("mesh-delivery-indexer", string(indexer))
fb.put(grants.ReadHave, "novox.node", map[string]any{"v": 1})
fb.put(grants.ReadHave, "novox.mesh-delivery-indexer", nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
gate := &seatGate{}
fp := partOn(fb)
fp.SeatList = gate
res := fp.Compose()
if strings.Contains(strings.Join(res.Refused, "\n"), "manifest of mesh-delivery-indexer:") {
t.Fatalf("the indexer is held in a pass placed by the haves: %v", res.Refused)
}
listRecs, _ := grants.ListRecords(grants.ReadInputs(fb, &grants.Manifests{}))
users, err := grants.ComposeUsers(listRecs)
if err != nil {
t.Fatal(err)
}
at := slices.IndexFunc(users, func(u grants.User) bool { return u.User == "novox.mesh-delivery-indexer" })
if at < 0 || !slices.ContainsFunc(users[at].Subscribe, func(s string) bool { return strings.HasPrefix(s, "mesh.seat.mesh-catalog.") }) {
t.Fatalf("novox.mesh-delivery-indexer is not composed to hear the seat's verbs: %+v", users)
}
raw := fb.own[grants.StateMemberships]["novox.mesh-delivery-indexer"]
if !slices.Contains(res.Published, "novox.mesh-delivery-indexer") || !strings.Contains(string(raw), "seat-definitions") {
t.Errorf("the indexer's membership serving seat-definitions was not published: %v, %s", res.Published, raw)
}
if !strings.Contains(strings.Join(res.Refused, "\n"), "manifest of mod-x: it waits for the seat list") {
t.Errorf("the seat list was opened by a pass placed by the haves: %v", res.Refused)
}
})
// DLV-G1, 1966/AC32-AC33 (novox/hq ADR 0595 §1, §7; design 25 § 12.2): the bus module's created writes the binding into
@@ -2653,8 +2770,11 @@ func TestGrants(t *testing.T) {
if !raised(grants.FirstViewUnanswered+".mesh-delivery-indexer.verdicts", occ.StateRaised) {
t.Fatalf("the refused raise was not said again at the pass: %+v", said)
}
if !slices.ContainsFunc(before.UserList.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadAssignments && g.Failed }) {
t.Fatalf("assignments answered empty at revision 0 was not a failed read: %+v", before.UserList.Gaps)
// Not answered above revision 0, the modules are placed by the node-engines' haves (issue 1966), said as its gap.
if !slices.ContainsFunc(before.UserList.Gaps, func(g grants.Gap) bool {
return g.Input == grants.ReadAssignments && strings.Contains(g.Why, "placed by the node-engines' haves")
}) {
t.Fatalf("assignments answered empty at revision 0 was not placed by the haves: %+v", before.UserList.Gaps)
}
if !raised(grants.FirstViewEmpty+".mesh-delivery-assigner.assignments", occ.StateRaised) {
t.Fatalf("no first-view-empty: %+v", said)
+105 -4
View File
@@ -167,6 +167,10 @@ type Inputs struct {
// Refused is every manifest the indexer holds that the part cannot read or mesh-sdk's reader refuses, assigned or
// not, each with why: its module counts as placed nowhere, so it is said loudly, never passed over (issue 1945).
Refused []Gap
// ByHaves says why this pass placed the modules where the node-engines' haves say they run, the assignments view not
// answering (bootstrap only, issue 1966); empty when the assignments placed them. Such a pass retires nothing
// (ReadFailures) and seals no list dropping a user of the list in force (holdsBack).
ByHaves string
// Held is, by module, the manifest a held module's running build was judged on (its have's fingerprint, the builder's
// build of it): what the seats it defines, the streams others hear from it and its reads are composed from while its
// trunk manifest is refused or waits, so no other principal loses a grant to it (ADR 0601 clause 3).
@@ -321,9 +325,8 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
// **The seat's assignments alone places modules** (novox/hq ADR 0549 §5, ADR 0567 §9; issue 774): the controller's
// assignments are read no more, and the seats held once for the mesh are held where it says. Not read is a failed
// read: nothing is placed, so the list composed past it is not sealed.
if keys, err := r.Keys(ReadAssignments); err != nil {
in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "refused or absent: nothing is placed (" + err.Error() + ")", Failed: true})
} else {
keys, assignErr := r.Keys(ReadAssignments)
if assignErr == nil {
// Each node's keys in either form, assembled by the seat's one reader (meshmoduleassignerclient.AssignmentsOf; research 072
// row 37): its node key and a key per module, or its legacy key `assignments.<node>`.
values := map[string][]byte{}
@@ -367,6 +370,23 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
if err != nil {
in.Gaps = append(in.Gaps, Gap{Input: ReadHave, Why: "refused or absent: " + err.Error(), Failed: true})
}
switch {
case assignErr == nil:
case err != nil:
in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "refused or absent: nothing is placed (" + assignErr.Error() + ")", Failed: true})
default:
// **Bootstrap only** (issue 1966, the operator 2026-11-07 22:19, bending ADR 0567 §9 for it): while the assignments
// view does not answer (an assigner that serves no first views yet), each module is placed where a node-engine's have
// says it runs, so the grants of the assigner that serves the view can be written. Said as a gap naming the
// fallback, and kept as ByHaves: no account is retired and no hearing deleted in such a pass (ReadFailures), and the
// list is sealed only when it keeps every user of the list in force (holdsBack). Once the view answers, unused.
in.ByHaves = "the assignments view did not answer (" + assignErr.Error() + "), so each module is placed where a node-engine's have says it runs"
in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "refused or absent: placed by the node-engines' haves instead, until it answers (" + assignErr.Error() + ")"})
placeByHaves(r, &in, haveKeys)
for node := range in.Assigned {
nodes[node] = true
}
}
for _, k := range haveKeys {
if node, _, isNode, err := nodeengineclient.ParseHaveKey(k); err == nil && isNode {
nodes[node] = true
@@ -488,7 +508,16 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
}
// What each machine is wanted to run: the want's fingerprint, the builder's build of it, the manifest it built.
var fromTrunk []string
for key, raw := range each(r, ReadWants, &in.Gaps) {
wants := map[string][]byte{}
if in.ByHaves == "" {
wants = each(r, ReadWants, &in.Gaps)
} else {
// Placed by the haves, what runs is what each have's fingerprint names, not a want: the assigner's wants are its
// view too, and not read in such a pass.
in.Gaps = append(in.Gaps, Gap{Input: ReadWants, Why: "not read: the modules are placed by the node-engines' haves this pass, so what runs is composed from each have's fingerprint"})
fromTrunk = runningByHaves(r, &in, cache)
}
for key, raw := range wants {
node, module, err := meshwantsclient.ParseWantKey(key)
if err != nil || !slices.Contains(in.Assigned[node], module) {
continue
@@ -671,12 +700,84 @@ func (in Inputs) ReadFailed() []string {
// keeps, the hearings it withholds).
func (in Inputs) ReadFailures() []string {
out := in.ReadFailed()
if in.ByHaves != "" {
out = append(out, ReadAssignments+": "+in.ByHaves)
}
for _, g := range in.Refused {
out = append(out, g.Input+": "+g.Why)
}
return out
}
// placeByHaves places each module on the node whose node-engine's have says it runs (any phase but removed), and finds the
// bus node among them: the bootstrap placement while the assignments view does not answer (issue 1966). A have that does
// not read places nothing, said as a gap.
func placeByHaves(r Reader, in *Inputs, haveKeys []string) {
for _, k := range haveKeys {
node, module, isNode, err := nodeengineclient.ParseHaveKey(k)
if err != nil || isNode {
continue
}
raw, found, err := r.Get(ReadHave, k)
if err != nil || !found {
continue
}
h, err := nodeengineclient.DecodeHave(raw)
if err != nil {
in.Gaps = append(in.Gaps, Gap{Input: ReadHave + " " + k, Why: "does not read, so it places nothing: " + err.Error()})
continue
}
if h.Phase == nodeengineclient.PhaseRemoved || slices.Contains(in.Assigned[node], module) {
continue
}
in.Assigned[node] = append(in.Assigned[node], module)
if module == BusModule && (in.BusNode == "" || node < in.BusNode) {
in.BusNode = node
}
}
}
// runningByHaves is, in a pass placed by the haves, the manifest each placed module runs: its have's fingerprint, the
// builder's build of it, the manifest it built; the placed modules with none, `<node>.<module>`, to compose from the trunk.
func runningByHaves(r Reader, in *Inputs, cache *Manifests) []string {
var fromTrunk []string
for node, modules := range in.Assigned {
for _, module := range modules {
raw, found, err := r.Get(ReadHave, node+"."+module)
if err != nil || !found {
fromTrunk = append(fromTrunk, node+"."+module)
continue
}
h, err := nodeengineclient.DecodeHave(raw)
if err != nil || h.Fp == "" {
fromTrunk = append(fromTrunk, node+"."+module)
continue
}
bk, err := builds.BuilderBuildKey(module, h.Fp)
if err != nil {
fromTrunk = append(fromTrunk, node+"."+module)
continue
}
braw, found, err := r.Get(ReadBuilds, bk)
if err != nil || !found {
fromTrunk = append(fromTrunk, node+"."+module)
continue
}
b, err := builds.DecodeBuilderBuild(braw)
if err != nil || b.Manifest == "" {
fromTrunk = append(fromTrunk, node+"."+module)
continue
}
if m, err := cache.get(r, b.Manifest); err == nil {
in.Running[node+"."+module] = m
} else {
fromTrunk = append(fromTrunk, node+"."+module)
}
}
}
return fromTrunk
}
// heldManifests are the manifests the running builds of the held modules (in.Refused) were judged on, by module: the have
// of the module on a node names the fingerprint that runs, and the builder's build of it the manifest. A module that runs
// nowhere, or whose build or manifest cannot be read, has none, and that is said as a gap.
+19 -3
View File
@@ -104,10 +104,10 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut, dropped
why = "no node runs the bus's module, so the list is sealed to nobody"
case key == "":
why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)"
case cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, digest) != "":
case cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, listUsers(list), digest) != "":
// The list in force is never replaced by one composed past a failed read, or one that drops a user of it for want
// of a hash: that user's module would be locked out of the bus (review of the switch).
why = p.holdsBack(in, leftOut, dropped, bus, lost, digest)
why = p.holdsBack(in, leftOut, dropped, bus, lost, listUsers(list), digest)
held = why
p.keepHashHold(res, digest)
case cur.Digest != digest || cur.SealedTo != grantsstate.SealingKeyID(key) || cur.Node != in.BusNode:
@@ -298,7 +298,11 @@ func listUsers(list []byte) []string {
// loaded list not read holds it too, since that cannot be ruled out). The list in force is the one the part wrote, by the users
// it kept of it; where it kept none (a list written before this release), the users the bus loaded (bus, nil when
// they could not be read). Empty when it may.
func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest string) string {
//
// In a pass placed by the node-engines' haves (in.ByHaves, issue 1966) the list is held, too, while it would leave out any user
// of the list in force or of the list the bus loaded (listed are the users of the list composed now): a bootstrap placement
// adds logins and never takes one away.
func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost, listed []string, digest string) string {
p.hashHold = nil
if f := in.ReadFailed(); len(f) > 0 {
return "held: the list in force is kept while a read of this pass failed (" + strings.Join(f, "; ") + ")"
@@ -320,6 +324,18 @@ func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest
default:
had = bus
}
if in.ByHaves != "" {
var gone []string
for _, u := range append(slices.Clone(had), bus...) {
if !slices.Contains(listed, u) && !slices.Contains(gone, u) {
gone = append(gone, u)
}
}
if len(gone) > 0 {
slices.Sort(gone)
return "held: " + in.ByHaves + ", and the list composed so would drop " + strings.Join(gone, ", ") + " of the list in force"
}
}
var unhashed []string
for _, u := range leftOut {
if slices.Contains(had, u) {