diff --git a/modules/mesh-grants/grants_test.go b/modules/mesh-grants/grants_test.go index e2e0f5d1..661ef137 100644 --- a/modules/mesh-grants/grants_test.go +++ b/modules/mesh-grants/grants_test.go @@ -633,8 +633,9 @@ func TestGrants(t *testing.T) { key, _ := builds.BuilderBuildKey("mod-x", fp) b.put(grants.ReadBuilds, key, map[string]any{"v": keys.Version, "state": "done", "manifest": digest}) }, []string{"a.mod-x", "a.nats", "b.mod-y", "node.a", "node.b"}, "people"}, - {"assignments refused is a gap; the machines come from the haves", func(b *fakeBus) { b.refused[grants.ReadAssignments] = true }, - []string{"node.a", "node.b"}, grants.ReadAssignments}, + // Issue 1966: refused, each module is placed where its have says it runs (the bootstrap fallback); a has nats. + {"assignments refused is a gap; the machines and the modules come from the haves", func(b *fakeBus) { b.refused[grants.ReadAssignments] = true }, + []string{"a.nats", "node.a", "node.b"}, grants.ReadAssignments}, } { t.Run(row.name, func(t *testing.T) { b := fixture() @@ -1310,10 +1311,17 @@ func TestGrants(t *testing.T) { t.Errorf("read: assigned %v, placed %v, gaps %v", read.Assigned, read.Placed, read.Gaps) } b.refused[grants.ReadAssignments] = true - // The assignments alone places (issue 774, 774/S29): not read, nothing is placed, and it is a failed read. + // The assignments alone places (issue 774, 774/S29), but for the bootstrap (issue 1966): not read, a module is placed + // where its have says it runs, said as a gap that is no failed read; with the haves refused too nothing is placed, + // and it is a failed read. + if read := grants.ReadInputs(b, &grants.Manifests{}); !slices.ContainsFunc(read.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadAssignments && !g.Failed }) || + !slices.Equal(read.Assigned["a"], []string{"nats"}) || len(read.Assigned) != 1 || read.ByHaves == "" || read.BusNode != "a" { + t.Errorf("assignments not on the bus is not placed by the haves: %v, %v", read.Gaps, read.Assigned) + } + b.refused[grants.ReadHave] = true if read := grants.ReadInputs(b, &grants.Manifests{}); !slices.ContainsFunc(read.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadAssignments && g.Failed }) || - len(read.Assigned) != 0 { - t.Errorf("assignments not on the bus is no failed read, or something is placed: %v, %v", read.Gaps, read.Assigned) + len(read.Assigned) != 0 || read.ByHaves != "" { + t.Errorf("assignments and haves not on the bus is no failed read, or something is placed: %v, %v", read.Gaps, read.Assigned) } // Issue 1945, the user list of 2026-10-27 09:21: the assignments' holds are the holders on record, not every holder. // A mesh seat no assignments holds keeps the module claiming it as its holder, wherever assignments puts it. @@ -2343,6 +2351,79 @@ func TestGrants(t *testing.T) { } }) + // Issue 1966 (the registry delivery's second bootstrap loop): an assigner that serves no first views answers no + // assignments, so nothing was placed and no grant written, not even for the assigner that would serve them. While the + // view does not answer, each module is placed where its have says it runs: the grants are written at the indexer's + // trunk fingerprints, the memberships published, the buckets a placed module declares created and the list sealed; but + // no account is retired and no list sealed that drops a user of the list in force. Once the view answers, unused. + t.Run("bootstrap: the assignments view unanswered places by the haves, losing no login (issue 1966)", func(t *testing.T) { + b := fixture() + b.refused[grants.ReadAssignments] = true + b.manifestOf("mesh-delivery-assigner", `{"module":"mesh-delivery-assigner","version":"1","own-secrets":{"broker":"/var/lib/a/broker"},"state":[{"name":"assignments"}]}`) + b.manifestOf("mesh-delivery-builder", `{"module":"mesh-delivery-builder","version":"1","own-secrets":{"broker":"/var/lib/b/broker"},"state":[{"name":"builds"}]}`) + runs := nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} + b.put(grants.ReadHave, "a.mesh-delivery-assigner", runs) + b.put(grants.ReadHave, "b.mesh-delivery-builder", runs) + removed := runs + removed.Phase = nodeengineclient.PhaseRemoved + b.put(grants.ReadHave, "b.mod-y", removed) + busKey, err := seal.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + b.put(grants.ReadHave, "a.nats", nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK, + UserList: &nodeengine.UserListWritten{Source: grantsstate.GrantsUserListSource, Revision: 8, Wanted: "sha256:aa", Digest: "sha256:aa", Users: 5, + Key: busKey.PublicKey, Follows: grantsstate.FollowGrants}}) + hash := func(user string) { + b.put(grants.ReadAccounts, user, map[string]any{"user": user, "hash": "$2a$10$x", "generation": 1}) + } + b.loaded = shadowOf(t, b) + for _, u := range b.loaded { + hash(u.User) + } + // The account of a user nothing composes now: a pass placed by the haves never retires it. + b.own[grants.StateAccounts] = map[string][]byte{"c.old": []byte(`{}`)} + p := partOn(b) + p.Create, p.Objects = b, &fakeObjects{} + p.Bus = grants.BusAddress{URL: "nats://a.mesh:4222", Fingerprint: "ab"} + res := p.Compose() + for _, m := range []string{"mesh-delivery-assigner", "mesh-delivery-builder"} { + key, _ := grantsstate.PartGrantKey(m, "src2:"+strings.Repeat("ab", 32)) + if _, ok := b.own[grants.StateGrants][key]; !ok { + t.Fatalf("no grant of %s placed by its have: %+v, gaps %+v", m, res.Grants, res.UserList.Gaps) + } + } + if !slices.Contains(b.creates, "KV_mesh-delivery-assigner_assignments") || !slices.Contains(b.creates, "KV_mesh-delivery-builder_builds") { + t.Errorf("the placed modules' buckets were not created: %v (%v)", b.creates, res.NotAsserted) + } + if !slices.Contains(res.Published, "a.mesh-delivery-assigner") || !slices.Contains(res.Published, "b.mesh-delivery-builder") || + slices.Contains(res.Published, "b.mod-y") { + t.Errorf("memberships published %v (a removed have places nothing)", res.Published) + } + if in := grants.ReadInputs(b, &grants.Manifests{}); slices.Contains(in.Assigned["b"], "mod-y") || !slices.Contains(in.Assigned["b"], "mesh-delivery-builder") { + t.Errorf("a removed have placed its module: b runs %v", in.Assigned["b"]) + } + if sealed, _, _ := p.Sealed(); sealed.Digest == "" || sealed.Node != "a" { + t.Errorf("no list sealed to the bus node the haves name: %+v, %+v", sealed, res.Sealed) + } + if b.own[grants.StateAccounts]["c.old"] == nil || res.Accounts == nil || + !slices.Contains(res.Accounts.Kept, "c.old: a read of this pass failed, so no account is retired") { + t.Errorf("a pass placed by the haves retired an account: %+v", res.Accounts) + } + // A user of the loaded list the haves do not compose: the list composed now is held, never sealed without it. + b.loaded = append(b.loaded, grants.User{User: "c.old"}) + b.put(grants.ReadHave, "b.mod-y", runs) + hash("b.mod-y") + if res := p.Compose(); res.Sealed == nil || !strings.Contains(res.Sealed.Why, "would drop c.old of the list in force") { + t.Errorf("a list dropping a loaded user was not held: %+v", res.Sealed) + } + // The view answers: the assignments alone places again. + delete(b.refused, grants.ReadAssignments) + if in := grants.ReadInputs(b, &grants.Manifests{}); in.ByHaves != "" || slices.Contains(in.Assigned["a"], "mesh-delivery-assigner") { + t.Errorf("an answering assignments view still placed by the haves: %q, %v", in.ByHaves, in.Assigned) + } + }) + // ADR 0590 clause 5's test for this repository (issue 1126): no module.json here names mesh-controller, the lease // read and the builder's consume of artifacts-gone included. Read as text, as mesh-catalog's controllerdeps check reads // its own: mesh-sdk names no module, so it holds no such check (issue 774, the operator 2026-11-03). @@ -2464,6 +2545,42 @@ func TestGrants(t *testing.T) { if !strings.Contains(refused, "manifest of mod-x: it waits for the seat list") { t.Errorf("another module is not held: %v", p.Last().Refused) } + + // Issue 1966: the assignments view unanswered, the indexer placed by its have alone (its real manifest, claiming + // mesh-catalog and serving seat-definitions) is composed to serve the seat's verbs, in the list and its membership, + // while the seat list stays unread: the bootstrap reaches the verb that reads the list without opening the gate. + indexer, err := os.ReadFile("../mesh-delivery-indexer/module.json") + if err != nil { + t.Fatal(err) + } + fb := fixture() + fb.refused[grants.ReadAssignments] = true + fb.manifestOf("mesh-delivery-indexer", string(indexer)) + fb.put(grants.ReadHave, "novox.node", map[string]any{"v": 1}) + fb.put(grants.ReadHave, "novox.mesh-delivery-indexer", nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) + gate := &seatGate{} + fp := partOn(fb) + fp.SeatList = gate + res := fp.Compose() + if strings.Contains(strings.Join(res.Refused, "\n"), "manifest of mesh-delivery-indexer:") { + t.Fatalf("the indexer is held in a pass placed by the haves: %v", res.Refused) + } + listRecs, _ := grants.ListRecords(grants.ReadInputs(fb, &grants.Manifests{})) + users, err := grants.ComposeUsers(listRecs) + if err != nil { + t.Fatal(err) + } + at := slices.IndexFunc(users, func(u grants.User) bool { return u.User == "novox.mesh-delivery-indexer" }) + if at < 0 || !slices.ContainsFunc(users[at].Subscribe, func(s string) bool { return strings.HasPrefix(s, "mesh.seat.mesh-catalog.") }) { + t.Fatalf("novox.mesh-delivery-indexer is not composed to hear the seat's verbs: %+v", users) + } + raw := fb.own[grants.StateMemberships]["novox.mesh-delivery-indexer"] + if !slices.Contains(res.Published, "novox.mesh-delivery-indexer") || !strings.Contains(string(raw), "seat-definitions") { + t.Errorf("the indexer's membership serving seat-definitions was not published: %v, %s", res.Published, raw) + } + if !strings.Contains(strings.Join(res.Refused, "\n"), "manifest of mod-x: it waits for the seat list") { + t.Errorf("the seat list was opened by a pass placed by the haves: %v", res.Refused) + } }) // DLV-G1, 1966/AC32-AC33 (novox/hq ADR 0595 §1, §7; design 25 § 12.2): the bus module's created writes the binding into @@ -2653,8 +2770,11 @@ func TestGrants(t *testing.T) { if !raised(grants.FirstViewUnanswered+".mesh-delivery-indexer.verdicts", occ.StateRaised) { t.Fatalf("the refused raise was not said again at the pass: %+v", said) } - if !slices.ContainsFunc(before.UserList.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadAssignments && g.Failed }) { - t.Fatalf("assignments answered empty at revision 0 was not a failed read: %+v", before.UserList.Gaps) + // Not answered above revision 0, the modules are placed by the node-engines' haves (issue 1966), said as its gap. + if !slices.ContainsFunc(before.UserList.Gaps, func(g grants.Gap) bool { + return g.Input == grants.ReadAssignments && strings.Contains(g.Why, "placed by the node-engines' haves") + }) { + t.Fatalf("assignments answered empty at revision 0 was not placed by the haves: %+v", before.UserList.Gaps) } if !raised(grants.FirstViewEmpty+".mesh-delivery-assigner.assignments", occ.StateRaised) { t.Fatalf("no first-view-empty: %+v", said) diff --git a/modules/mesh-grants/internal/grants/inputs.go b/modules/mesh-grants/internal/grants/inputs.go index e8a7b85d..663c9e43 100644 --- a/modules/mesh-grants/internal/grants/inputs.go +++ b/modules/mesh-grants/internal/grants/inputs.go @@ -167,6 +167,10 @@ type Inputs struct { // Refused is every manifest the indexer holds that the part cannot read or mesh-sdk's reader refuses, assigned or // not, each with why: its module counts as placed nowhere, so it is said loudly, never passed over (issue 1945). Refused []Gap + // ByHaves says why this pass placed the modules where the node-engines' haves say they run, the assignments view not + // answering (bootstrap only, issue 1966); empty when the assignments placed them. Such a pass retires nothing + // (ReadFailures) and seals no list dropping a user of the list in force (holdsBack). + ByHaves string // Held is, by module, the manifest a held module's running build was judged on (its have's fingerprint, the builder's // build of it): what the seats it defines, the streams others hear from it and its reads are composed from while its // trunk manifest is refused or waits, so no other principal loses a grant to it (ADR 0601 clause 3). @@ -321,9 +325,8 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { // **The seat's assignments alone places modules** (novox/hq ADR 0549 §5, ADR 0567 §9; issue 774): the controller's // assignments are read no more, and the seats held once for the mesh are held where it says. Not read is a failed // read: nothing is placed, so the list composed past it is not sealed. - if keys, err := r.Keys(ReadAssignments); err != nil { - in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "refused or absent: nothing is placed (" + err.Error() + ")", Failed: true}) - } else { + keys, assignErr := r.Keys(ReadAssignments) + if assignErr == nil { // Each node's keys in either form, assembled by the seat's one reader (meshmoduleassignerclient.AssignmentsOf; research 072 // row 37): its node key and a key per module, or its legacy key `assignments.`. values := map[string][]byte{} @@ -367,6 +370,23 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { if err != nil { in.Gaps = append(in.Gaps, Gap{Input: ReadHave, Why: "refused or absent: " + err.Error(), Failed: true}) } + switch { + case assignErr == nil: + case err != nil: + in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "refused or absent: nothing is placed (" + assignErr.Error() + ")", Failed: true}) + default: + // **Bootstrap only** (issue 1966, the operator 2026-11-07 22:19, bending ADR 0567 §9 for it): while the assignments + // view does not answer (an assigner that serves no first views yet), each module is placed where a node-engine's have + // says it runs, so the grants of the assigner that serves the view can be written. Said as a gap naming the + // fallback, and kept as ByHaves: no account is retired and no hearing deleted in such a pass (ReadFailures), and the + // list is sealed only when it keeps every user of the list in force (holdsBack). Once the view answers, unused. + in.ByHaves = "the assignments view did not answer (" + assignErr.Error() + "), so each module is placed where a node-engine's have says it runs" + in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "refused or absent: placed by the node-engines' haves instead, until it answers (" + assignErr.Error() + ")"}) + placeByHaves(r, &in, haveKeys) + for node := range in.Assigned { + nodes[node] = true + } + } for _, k := range haveKeys { if node, _, isNode, err := nodeengineclient.ParseHaveKey(k); err == nil && isNode { nodes[node] = true @@ -488,7 +508,16 @@ func ReadInputs(r Reader, cache *Manifests) Inputs { } // What each machine is wanted to run: the want's fingerprint, the builder's build of it, the manifest it built. var fromTrunk []string - for key, raw := range each(r, ReadWants, &in.Gaps) { + wants := map[string][]byte{} + if in.ByHaves == "" { + wants = each(r, ReadWants, &in.Gaps) + } else { + // Placed by the haves, what runs is what each have's fingerprint names, not a want: the assigner's wants are its + // view too, and not read in such a pass. + in.Gaps = append(in.Gaps, Gap{Input: ReadWants, Why: "not read: the modules are placed by the node-engines' haves this pass, so what runs is composed from each have's fingerprint"}) + fromTrunk = runningByHaves(r, &in, cache) + } + for key, raw := range wants { node, module, err := meshwantsclient.ParseWantKey(key) if err != nil || !slices.Contains(in.Assigned[node], module) { continue @@ -671,12 +700,84 @@ func (in Inputs) ReadFailed() []string { // keeps, the hearings it withholds). func (in Inputs) ReadFailures() []string { out := in.ReadFailed() + if in.ByHaves != "" { + out = append(out, ReadAssignments+": "+in.ByHaves) + } for _, g := range in.Refused { out = append(out, g.Input+": "+g.Why) } return out } +// placeByHaves places each module on the node whose node-engine's have says it runs (any phase but removed), and finds the +// bus node among them: the bootstrap placement while the assignments view does not answer (issue 1966). A have that does +// not read places nothing, said as a gap. +func placeByHaves(r Reader, in *Inputs, haveKeys []string) { + for _, k := range haveKeys { + node, module, isNode, err := nodeengineclient.ParseHaveKey(k) + if err != nil || isNode { + continue + } + raw, found, err := r.Get(ReadHave, k) + if err != nil || !found { + continue + } + h, err := nodeengineclient.DecodeHave(raw) + if err != nil { + in.Gaps = append(in.Gaps, Gap{Input: ReadHave + " " + k, Why: "does not read, so it places nothing: " + err.Error()}) + continue + } + if h.Phase == nodeengineclient.PhaseRemoved || slices.Contains(in.Assigned[node], module) { + continue + } + in.Assigned[node] = append(in.Assigned[node], module) + if module == BusModule && (in.BusNode == "" || node < in.BusNode) { + in.BusNode = node + } + } +} + +// runningByHaves is, in a pass placed by the haves, the manifest each placed module runs: its have's fingerprint, the +// builder's build of it, the manifest it built; the placed modules with none, `.`, to compose from the trunk. +func runningByHaves(r Reader, in *Inputs, cache *Manifests) []string { + var fromTrunk []string + for node, modules := range in.Assigned { + for _, module := range modules { + raw, found, err := r.Get(ReadHave, node+"."+module) + if err != nil || !found { + fromTrunk = append(fromTrunk, node+"."+module) + continue + } + h, err := nodeengineclient.DecodeHave(raw) + if err != nil || h.Fp == "" { + fromTrunk = append(fromTrunk, node+"."+module) + continue + } + bk, err := builds.BuilderBuildKey(module, h.Fp) + if err != nil { + fromTrunk = append(fromTrunk, node+"."+module) + continue + } + braw, found, err := r.Get(ReadBuilds, bk) + if err != nil || !found { + fromTrunk = append(fromTrunk, node+"."+module) + continue + } + b, err := builds.DecodeBuilderBuild(braw) + if err != nil || b.Manifest == "" { + fromTrunk = append(fromTrunk, node+"."+module) + continue + } + if m, err := cache.get(r, b.Manifest); err == nil { + in.Running[node+"."+module] = m + } else { + fromTrunk = append(fromTrunk, node+"."+module) + } + } + } + return fromTrunk +} + // heldManifests are the manifests the running builds of the held modules (in.Refused) were judged on, by module: the have // of the module on a node names the fingerprint that runs, and the builder's build of it the manifest. A module that runs // nowhere, or whose build or manifest cannot be read, has none, and that is said as a gap. diff --git a/modules/mesh-grants/internal/grants/sealed.go b/modules/mesh-grants/internal/grants/sealed.go index 01b491b9..139642f3 100644 --- a/modules/mesh-grants/internal/grants/sealed.go +++ b/modules/mesh-grants/internal/grants/sealed.go @@ -104,10 +104,10 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut, dropped why = "no node runs the bus's module, so the list is sealed to nobody" case key == "": why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)" - case cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, digest) != "": + case cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, listUsers(list), digest) != "": // The list in force is never replaced by one composed past a failed read, or one that drops a user of it for want // of a hash: that user's module would be locked out of the bus (review of the switch). - why = p.holdsBack(in, leftOut, dropped, bus, lost, digest) + why = p.holdsBack(in, leftOut, dropped, bus, lost, listUsers(list), digest) held = why p.keepHashHold(res, digest) case cur.Digest != digest || cur.SealedTo != grantsstate.SealingKeyID(key) || cur.Node != in.BusNode: @@ -298,7 +298,11 @@ func listUsers(list []byte) []string { // loaded list not read holds it too, since that cannot be ruled out). The list in force is the one the part wrote, by the users // it kept of it; where it kept none (a list written before this release), the users the bus loaded (bus, nil when // they could not be read). Empty when it may. -func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest string) string { +// +// In a pass placed by the node-engines' haves (in.ByHaves, issue 1966) the list is held, too, while it would leave out any user +// of the list in force or of the list the bus loaded (listed are the users of the list composed now): a bootstrap placement +// adds logins and never takes one away. +func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost, listed []string, digest string) string { p.hashHold = nil if f := in.ReadFailed(); len(f) > 0 { return "held: the list in force is kept while a read of this pass failed (" + strings.Join(f, "; ") + ")" @@ -320,6 +324,18 @@ func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest default: had = bus } + if in.ByHaves != "" { + var gone []string + for _, u := range append(slices.Clone(had), bus...) { + if !slices.Contains(listed, u) && !slices.Contains(gone, u) { + gone = append(gone, u) + } + } + if len(gone) > 0 { + slices.Sort(gone) + return "held: " + in.ByHaves + ", and the list composed so would drop " + strings.Join(gone, ", ") + " of the list in force" + } + } var unhashed []string for _, u := range leftOut { if slices.Contains(had, u) {