Merge pull request 'mesh-grants places modules by the node-engines' haves while the assigner's assignments view does not answer (issue 1966, bootstrap recovery)' (#170) from fix/1966-grants-places-by-haves into main

modules/mesh-grants/grants_test.go

@@ -633,8 +633,9 @@ func TestGrants(t *testing.T) {
 				key, _ := builds.BuilderBuildKey("mod-x", fp)
 				b.put(grants.ReadBuilds, key, map[string]any{"v": keys.Version, "state": "done", "manifest": digest})
 			}, []string{"a.mod-x", "a.nats", "b.mod-y", "node.a", "node.b"}, "people"},
-			{"assignments refused is a gap; the machines come from the haves", func(b *fakeBus) { b.refused[grants.ReadAssignments] = true },
-				[]string{"node.a", "node.b"}, grants.ReadAssignments},
+			// Issue 1966: refused, each module is placed where its have says it runs (the bootstrap fallback); a has nats.
+			{"assignments refused is a gap; the machines and the modules come from the haves", func(b *fakeBus) { b.refused[grants.ReadAssignments] = true },
+				[]string{"a.nats", "node.a", "node.b"}, grants.ReadAssignments},
 		} {
 			t.Run(row.name, func(t *testing.T) {
 				b := fixture()
@@ -1310,10 +1311,17 @@ func TestGrants(t *testing.T) {
 			t.Errorf("read: assigned %v, placed %v, gaps %v", read.Assigned, read.Placed, read.Gaps)
 		}
 		b.refused[grants.ReadAssignments] = true
-		// The assignments alone places (issue 774, 774/S29): not read, nothing is placed, and it is a failed read.
+		// The assignments alone places (issue 774, 774/S29), but for the bootstrap (issue 1966): not read, a module is placed
+		// where its have says it runs, said as a gap that is no failed read; with the haves refused too nothing is placed,
+		// and it is a failed read.
+		if read := grants.ReadInputs(b, &grants.Manifests{}); !slices.ContainsFunc(read.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadAssignments && !g.Failed }) ||
+			!slices.Equal(read.Assigned["a"], []string{"nats"}) || len(read.Assigned) != 1 || read.ByHaves == "" || read.BusNode != "a" {
+			t.Errorf("assignments not on the bus is not placed by the haves: %v, %v", read.Gaps, read.Assigned)
+		}
+		b.refused[grants.ReadHave] = true
 		if read := grants.ReadInputs(b, &grants.Manifests{}); !slices.ContainsFunc(read.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadAssignments && g.Failed }) ||
-			len(read.Assigned) != 0 {
-			t.Errorf("assignments not on the bus is no failed read, or something is placed: %v, %v", read.Gaps, read.Assigned)
+			len(read.Assigned) != 0 || read.ByHaves != "" {
+			t.Errorf("assignments and haves not on the bus is no failed read, or something is placed: %v, %v", read.Gaps, read.Assigned)
 		}
 		// Issue 1945, the user list of 2026-10-27 09:21: the assignments' holds are the holders on record, not every holder.
 		// A mesh seat no assignments holds keeps the module claiming it as its holder, wherever assignments puts it.
@@ -2343,6 +2351,79 @@ func TestGrants(t *testing.T) {
 		}
 	})
 
+	// Issue 1966 (the registry delivery's second bootstrap loop): an assigner that serves no first views answers no
+	// assignments, so nothing was placed and no grant written, not even for the assigner that would serve them. While the
+	// view does not answer, each module is placed where its have says it runs: the grants are written at the indexer's
+	// trunk fingerprints, the memberships published, the buckets a placed module declares created and the list sealed; but
+	// no account is retired and no list sealed that drops a user of the list in force. Once the view answers, unused.
+	t.Run("bootstrap: the assignments view unanswered places by the haves, losing no login (issue 1966)", func(t *testing.T) {
+		b := fixture()
+		b.refused[grants.ReadAssignments] = true
+		b.manifestOf("mesh-delivery-assigner", `{"module":"mesh-delivery-assigner","version":"1","own-secrets":{"broker":"/var/lib/a/broker"},"state":[{"name":"assignments"}]}`)
+		b.manifestOf("mesh-delivery-builder", `{"module":"mesh-delivery-builder","version":"1","own-secrets":{"broker":"/var/lib/b/broker"},"state":[{"name":"builds"}]}`)
+		runs := nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
+		b.put(grants.ReadHave, "a.mesh-delivery-assigner", runs)
+		b.put(grants.ReadHave, "b.mesh-delivery-builder", runs)
+		removed := runs
+		removed.Phase = nodeengineclient.PhaseRemoved
+		b.put(grants.ReadHave, "b.mod-y", removed)
+		busKey, err := seal.GenerateKeyPair()
+		if err != nil {
+			t.Fatal(err)
+		}
+		b.put(grants.ReadHave, "a.nats", nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK,
+			UserList: &nodeengine.UserListWritten{Source: grantsstate.GrantsUserListSource, Revision: 8, Wanted: "sha256:aa", Digest: "sha256:aa", Users: 5,
+				Key: busKey.PublicKey, Follows: grantsstate.FollowGrants}})
+		hash := func(user string) {
+			b.put(grants.ReadAccounts, user, map[string]any{"user": user, "hash": "$2a$10$x", "generation": 1})
+		}
+		b.loaded = shadowOf(t, b)
+		for _, u := range b.loaded {
+			hash(u.User)
+		}
+		// The account of a user nothing composes now: a pass placed by the haves never retires it.
+		b.own[grants.StateAccounts] = map[string][]byte{"c.old": []byte(`{}`)}
+		p := partOn(b)
+		p.Create, p.Objects = b, &fakeObjects{}
+		p.Bus = grants.BusAddress{URL: "nats://a.mesh:4222", Fingerprint: "ab"}
+		res := p.Compose()
+		for _, m := range []string{"mesh-delivery-assigner", "mesh-delivery-builder"} {
+			key, _ := grantsstate.PartGrantKey(m, "src2:"+strings.Repeat("ab", 32))
+			if _, ok := b.own[grants.StateGrants][key]; !ok {
+				t.Fatalf("no grant of %s placed by its have: %+v, gaps %+v", m, res.Grants, res.UserList.Gaps)
+			}
+		}
+		if !slices.Contains(b.creates, "KV_mesh-delivery-assigner_assignments") || !slices.Contains(b.creates, "KV_mesh-delivery-builder_builds") {
+			t.Errorf("the placed modules' buckets were not created: %v (%v)", b.creates, res.NotAsserted)
+		}
+		if !slices.Contains(res.Published, "a.mesh-delivery-assigner") || !slices.Contains(res.Published, "b.mesh-delivery-builder") ||
+			slices.Contains(res.Published, "b.mod-y") {
+			t.Errorf("memberships published %v (a removed have places nothing)", res.Published)
+		}
+		if in := grants.ReadInputs(b, &grants.Manifests{}); slices.Contains(in.Assigned["b"], "mod-y") || !slices.Contains(in.Assigned["b"], "mesh-delivery-builder") {
+			t.Errorf("a removed have placed its module: b runs %v", in.Assigned["b"])
+		}
+		if sealed, _, _ := p.Sealed(); sealed.Digest == "" || sealed.Node != "a" {
+			t.Errorf("no list sealed to the bus node the haves name: %+v, %+v", sealed, res.Sealed)
+		}
+		if b.own[grants.StateAccounts]["c.old"] == nil || res.Accounts == nil ||
+			!slices.Contains(res.Accounts.Kept, "c.old: a read of this pass failed, so no account is retired") {
+			t.Errorf("a pass placed by the haves retired an account: %+v", res.Accounts)
+		}
+		// A user of the loaded list the haves do not compose: the list composed now is held, never sealed without it.
+		b.loaded = append(b.loaded, grants.User{User: "c.old"})
+		b.put(grants.ReadHave, "b.mod-y", runs)
+		hash("b.mod-y")
+		if res := p.Compose(); res.Sealed == nil || !strings.Contains(res.Sealed.Why, "would drop c.old of the list in force") {
+			t.Errorf("a list dropping a loaded user was not held: %+v", res.Sealed)
+		}
+		// The view answers: the assignments alone places again.
+		delete(b.refused, grants.ReadAssignments)
+		if in := grants.ReadInputs(b, &grants.Manifests{}); in.ByHaves != "" || slices.Contains(in.Assigned["a"], "mesh-delivery-assigner") {
+			t.Errorf("an answering assignments view still placed by the haves: %q, %v", in.ByHaves, in.Assigned)
+		}
+	})
+
 	// ADR 0590 clause 5's test for this repository (issue 1126): no module.json here names mesh-controller, the lease
 	// read and the builder's consume of artifacts-gone included. Read as text, as mesh-catalog's controllerdeps check reads
 	// its own: mesh-sdk names no module, so it holds no such check (issue 774, the operator 2026-11-03).
@@ -2464,6 +2545,42 @@ func TestGrants(t *testing.T) {
 		if !strings.Contains(refused, "manifest of mod-x: it waits for the seat list") {
 			t.Errorf("another module is not held: %v", p.Last().Refused)
 		}
+
+		// Issue 1966: the assignments view unanswered, the indexer placed by its have alone (its real manifest, claiming
+		// mesh-catalog and serving seat-definitions) is composed to serve the seat's verbs, in the list and its membership,
+		// while the seat list stays unread: the bootstrap reaches the verb that reads the list without opening the gate.
+		indexer, err := os.ReadFile("../mesh-delivery-indexer/module.json")
+		if err != nil {
+			t.Fatal(err)
+		}
+		fb := fixture()
+		fb.refused[grants.ReadAssignments] = true
+		fb.manifestOf("mesh-delivery-indexer", string(indexer))
+		fb.put(grants.ReadHave, "novox.node", map[string]any{"v": 1})
+		fb.put(grants.ReadHave, "novox.mesh-delivery-indexer", nodeengine.HaveByNodeModuleValue{V: keys.Version, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
+		gate := &seatGate{}
+		fp := partOn(fb)
+		fp.SeatList = gate
+		res := fp.Compose()
+		if strings.Contains(strings.Join(res.Refused, "\n"), "manifest of mesh-delivery-indexer:") {
+			t.Fatalf("the indexer is held in a pass placed by the haves: %v", res.Refused)
+		}
+		listRecs, _ := grants.ListRecords(grants.ReadInputs(fb, &grants.Manifests{}))
+		users, err := grants.ComposeUsers(listRecs)
+		if err != nil {
+			t.Fatal(err)
+		}
+		at := slices.IndexFunc(users, func(u grants.User) bool { return u.User == "novox.mesh-delivery-indexer" })
+		if at < 0 || !slices.ContainsFunc(users[at].Subscribe, func(s string) bool { return strings.HasPrefix(s, "mesh.seat.mesh-catalog.") }) {
+			t.Fatalf("novox.mesh-delivery-indexer is not composed to hear the seat's verbs: %+v", users)
+		}
+		raw := fb.own[grants.StateMemberships]["novox.mesh-delivery-indexer"]
+		if !slices.Contains(res.Published, "novox.mesh-delivery-indexer") || !strings.Contains(string(raw), "seat-definitions") {
+			t.Errorf("the indexer's membership serving seat-definitions was not published: %v, %s", res.Published, raw)
+		}
+		if !strings.Contains(strings.Join(res.Refused, "\n"), "manifest of mod-x: it waits for the seat list") {
+			t.Errorf("the seat list was opened by a pass placed by the haves: %v", res.Refused)
+		}
 	})
 
 	// DLV-G1, 1966/AC32-AC33 (novox/hq ADR 0595 §1, §7; design 25 § 12.2): the bus module's created writes the binding into
@@ -2653,8 +2770,11 @@ func TestGrants(t *testing.T) {
 		if !raised(grants.FirstViewUnanswered+".mesh-delivery-indexer.verdicts", occ.StateRaised) {
 			t.Fatalf("the refused raise was not said again at the pass: %+v", said)
 		}
-		if !slices.ContainsFunc(before.UserList.Gaps, func(g grants.Gap) bool { return g.Input == grants.ReadAssignments && g.Failed }) {
-			t.Fatalf("assignments answered empty at revision 0 was not a failed read: %+v", before.UserList.Gaps)
+		// Not answered above revision 0, the modules are placed by the node-engines' haves (issue 1966), said as its gap.
+		if !slices.ContainsFunc(before.UserList.Gaps, func(g grants.Gap) bool {
+			return g.Input == grants.ReadAssignments && strings.Contains(g.Why, "placed by the node-engines' haves")
+		}) {
+			t.Fatalf("assignments answered empty at revision 0 was not placed by the haves: %+v", before.UserList.Gaps)
 		}
 		if !raised(grants.FirstViewEmpty+".mesh-delivery-assigner.assignments", occ.StateRaised) {
 			t.Fatalf("no first-view-empty: %+v", said)

modules/mesh-grants/internal/grants/inputs.go

@@ -167,6 +167,10 @@ type Inputs struct {
 	// Refused is every manifest the indexer holds that the part cannot read or mesh-sdk's reader refuses, assigned or
 	// not, each with why: its module counts as placed nowhere, so it is said loudly, never passed over (issue 1945).
 	Refused []Gap
+	// ByHaves says why this pass placed the modules where the node-engines' haves say they run, the assignments view not
+	// answering (bootstrap only, issue 1966); empty when the assignments placed them. Such a pass retires nothing
+	// (ReadFailures) and seals no list dropping a user of the list in force (holdsBack).
+	ByHaves string
 	// Held is, by module, the manifest a held module's running build was judged on (its have's fingerprint, the builder's
 	// build of it): what the seats it defines, the streams others hear from it and its reads are composed from while its
 	// trunk manifest is refused or waits, so no other principal loses a grant to it (ADR 0601 clause 3).
@@ -321,9 +325,8 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
 	// **The seat's assignments alone places modules** (novox/hq ADR 0549 §5, ADR 0567 §9; issue 774): the controller's
 	// assignments are read no more, and the seats held once for the mesh are held where it says. Not read is a failed
 	// read: nothing is placed, so the list composed past it is not sealed.
-	if keys, err := r.Keys(ReadAssignments); err != nil {
-		in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "refused or absent: nothing is placed (" + err.Error() + ")", Failed: true})
-	} else {
+	keys, assignErr := r.Keys(ReadAssignments)
+	if assignErr == nil {
 		// Each node's keys in either form, assembled by the seat's one reader (meshmoduleassignerclient.AssignmentsOf; research 072
 		// row 37): its node key and a key per module, or its legacy key `assignments.<node>`.
 		values := map[string][]byte{}
@@ -367,6 +370,23 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
 	if err != nil {
 		in.Gaps = append(in.Gaps, Gap{Input: ReadHave, Why: "refused or absent: " + err.Error(), Failed: true})
 	}
+	switch {
+	case assignErr == nil:
+	case err != nil:
+		in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "refused or absent: nothing is placed (" + assignErr.Error() + ")", Failed: true})
+	default:
+		// **Bootstrap only** (issue 1966, the operator 2026-11-07 22:19, bending ADR 0567 §9 for it): while the assignments
+		// view does not answer (an assigner that serves no first views yet), each module is placed where a node-engine's have
+		// says it runs, so the grants of the assigner that serves the view can be written. Said as a gap naming the
+		// fallback, and kept as ByHaves: no account is retired and no hearing deleted in such a pass (ReadFailures), and the
+		// list is sealed only when it keeps every user of the list in force (holdsBack). Once the view answers, unused.
+		in.ByHaves = "the assignments view did not answer (" + assignErr.Error() + "), so each module is placed where a node-engine's have says it runs"
+		in.Gaps = append(in.Gaps, Gap{Input: ReadAssignments, Why: "refused or absent: placed by the node-engines' haves instead, until it answers (" + assignErr.Error() + ")"})
+		placeByHaves(r, &in, haveKeys)
+		for node := range in.Assigned {
+			nodes[node] = true
+		}
+	}
 	for _, k := range haveKeys {
 		if node, _, isNode, err := nodeengineclient.ParseHaveKey(k); err == nil && isNode {
 			nodes[node] = true
@@ -488,7 +508,16 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
 	}
 	// What each machine is wanted to run: the want's fingerprint, the builder's build of it, the manifest it built.
 	var fromTrunk []string
-	for key, raw := range each(r, ReadWants, &in.Gaps) {
+	wants := map[string][]byte{}
+	if in.ByHaves == "" {
+		wants = each(r, ReadWants, &in.Gaps)
+	} else {
+		// Placed by the haves, what runs is what each have's fingerprint names, not a want: the assigner's wants are its
+		// view too, and not read in such a pass.
+		in.Gaps = append(in.Gaps, Gap{Input: ReadWants, Why: "not read: the modules are placed by the node-engines' haves this pass, so what runs is composed from each have's fingerprint"})
+		fromTrunk = runningByHaves(r, &in, cache)
+	}
+	for key, raw := range wants {
 		node, module, err := meshwantsclient.ParseWantKey(key)
 		if err != nil || !slices.Contains(in.Assigned[node], module) {
 			continue
@@ -671,12 +700,84 @@ func (in Inputs) ReadFailed() []string {
 // keeps, the hearings it withholds).
 func (in Inputs) ReadFailures() []string {
 	out := in.ReadFailed()
+	if in.ByHaves != "" {
+		out = append(out, ReadAssignments+": "+in.ByHaves)
+	}
 	for _, g := range in.Refused {
 		out = append(out, g.Input+": "+g.Why)
 	}
 	return out
 }
 
+// placeByHaves places each module on the node whose node-engine's have says it runs (any phase but removed), and finds the
+// bus node among them: the bootstrap placement while the assignments view does not answer (issue 1966). A have that does
+// not read places nothing, said as a gap.
+func placeByHaves(r Reader, in *Inputs, haveKeys []string) {
+	for _, k := range haveKeys {
+		node, module, isNode, err := nodeengineclient.ParseHaveKey(k)
+		if err != nil || isNode {
+			continue
+		}
+		raw, found, err := r.Get(ReadHave, k)
+		if err != nil || !found {
+			continue
+		}
+		h, err := nodeengineclient.DecodeHave(raw)
+		if err != nil {
+			in.Gaps = append(in.Gaps, Gap{Input: ReadHave + " " + k, Why: "does not read, so it places nothing: " + err.Error()})
+			continue
+		}
+		if h.Phase == nodeengineclient.PhaseRemoved || slices.Contains(in.Assigned[node], module) {
+			continue
+		}
+		in.Assigned[node] = append(in.Assigned[node], module)
+		if module == BusModule && (in.BusNode == "" || node < in.BusNode) {
+			in.BusNode = node
+		}
+	}
+}
+
+// runningByHaves is, in a pass placed by the haves, the manifest each placed module runs: its have's fingerprint, the
+// builder's build of it, the manifest it built; the placed modules with none, `<node>.<module>`, to compose from the trunk.
+func runningByHaves(r Reader, in *Inputs, cache *Manifests) []string {
+	var fromTrunk []string
+	for node, modules := range in.Assigned {
+		for _, module := range modules {
+			raw, found, err := r.Get(ReadHave, node+"."+module)
+			if err != nil || !found {
+				fromTrunk = append(fromTrunk, node+"."+module)
+				continue
+			}
+			h, err := nodeengineclient.DecodeHave(raw)
+			if err != nil || h.Fp == "" {
+				fromTrunk = append(fromTrunk, node+"."+module)
+				continue
+			}
+			bk, err := builds.BuilderBuildKey(module, h.Fp)
+			if err != nil {
+				fromTrunk = append(fromTrunk, node+"."+module)
+				continue
+			}
+			braw, found, err := r.Get(ReadBuilds, bk)
+			if err != nil || !found {
+				fromTrunk = append(fromTrunk, node+"."+module)
+				continue
+			}
+			b, err := builds.DecodeBuilderBuild(braw)
+			if err != nil || b.Manifest == "" {
+				fromTrunk = append(fromTrunk, node+"."+module)
+				continue
+			}
+			if m, err := cache.get(r, b.Manifest); err == nil {
+				in.Running[node+"."+module] = m
+			} else {
+				fromTrunk = append(fromTrunk, node+"."+module)
+			}
+		}
+	}
+	return fromTrunk
+}
+
 // heldManifests are the manifests the running builds of the held modules (in.Refused) were judged on, by module: the have
 // of the module on a node names the fingerprint that runs, and the builder's build of it the manifest. A module that runs
 // nowhere, or whose build or manifest cannot be read, has none, and that is said as a gap.

modules/mesh-grants/internal/grants/sealed.go

@@ -104,10 +104,10 @@ func (p *Part) writeSealed(res *Result, list []byte, users int, leftOut, dropped
 		why = "no node runs the bus's module, so the list is sealed to nobody"
 	case key == "":
 		why = "the bus node's have of the bus module names no key yet (its node-engine says it from this release)"
-	case cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, digest) != "":
+	case cur.Digest != digest && p.holdsBack(in, leftOut, dropped, bus, lost, listUsers(list), digest) != "":
 		// The list in force is never replaced by one composed past a failed read, or one that drops a user of it for want
 		// of a hash: that user's module would be locked out of the bus (review of the switch).
-		why = p.holdsBack(in, leftOut, dropped, bus, lost, digest)
+		why = p.holdsBack(in, leftOut, dropped, bus, lost, listUsers(list), digest)
 		held = why
 		p.keepHashHold(res, digest)
 	case cur.Digest != digest || cur.SealedTo != grantsstate.SealingKeyID(key) || cur.Node != in.BusNode:
@@ -298,7 +298,11 @@ func listUsers(list []byte) []string {
 // loaded list not read holds it too, since that cannot be ruled out). The list in force is the one the part wrote, by the users
 // it kept of it; where it kept none (a list written before this release), the users the bus loaded (bus, nil when
 // they could not be read). Empty when it may.
-func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest string) string {
+//
+// In a pass placed by the node-engines' haves (in.ByHaves, issue 1966) the list is held, too, while it would leave out any user
+// of the list in force or of the list the bus loaded (listed are the users of the list composed now): a bootstrap placement
+// adds logins and never takes one away.
+func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost, listed []string, digest string) string {
 	p.hashHold = nil
 	if f := in.ReadFailed(); len(f) > 0 {
 		return "held: the list in force is kept while a read of this pass failed (" + strings.Join(f, "; ") + ")"
@@ -320,6 +324,18 @@ func (p *Part) holdsBack(in Inputs, leftOut, dropped, bus, lost []string, digest
 	default:
 		had = bus
 	}
+	if in.ByHaves != "" {
+		var gone []string
+		for _, u := range append(slices.Clone(had), bus...) {
+			if !slices.Contains(listed, u) && !slices.Contains(gone, u) {
+				gone = append(gone, u)
+			}
+		}
+		if len(gone) > 0 {
+			slices.Sort(gone)
+			return "held: " + in.ByHaves + ", and the list composed so would drop " + strings.Join(gone, ", ") + " of the list in force"
+		}
+	}
 	var unhashed []string
 	for _, u := range leftOut {
 		if slices.Contains(had, u) {