mesh-grants: judge manifests against the predefined keys on the bus, and say every refused one

A manifest falling back to a predefined key (claude-code, dunst) was parsed
with no list, refused silently, and its module counted as placed nowhere, so
three memberships would lose claude-code's reach addresses and were held back.
The part now reads the list the controller mirrors (mesh-controller.predefined,
key list), follows it, and lists every manifest it cannot read in its gaps,
its pass (refused) and its log.

Issue: 1945

modules/mesh-grants/grants_test.go

@@ -414,6 +414,70 @@ func TestGrants(t *testing.T) {
 		}
 	})
 
+	t.Run("manifests judged against the published list of predefined keys; every refusal listed", func(t *testing.T) {
+		// mod-p's value falls back to a predefined key, as claude-code's managed_settings does (issue 1945); mod-z's manifest
+		// carries a key mesh-sdk's reader refuses, and mod-z is assigned nowhere.
+		const pManifest = `{"module":"mod-p","version":"1","own-secrets":{"broker":"/var/lib/p/broker"},` +
+			`"config":{"size":{"type":"number","description":"The size text shows in, in points.","predefined":"font-size","default":12,"runtime":true}}}`
+		const zManifest = `{"module":"mod-z","version":"1","no-such-key":true}`
+		published := map[string]any{"keys": map[string]any{"font-size": map[string]any{"type": "number",
+			"description": "The size a desktop program shows its text in, in points.", "minimum": 6, "maximum": 72}},
+			"repository": "mesh-catalog", "commit": "c1", "generation": 7, "build": "b1"}
+		for _, row := range []struct {
+			name    string
+			list    bool
+			placed  bool
+			refused []string
+		}{
+			{"a fallback to a predefined key is read against the list on the bus, and its module placed", true, true,
+				[]string{"manifest of mod-z"}},
+			{"with no list on the bus the fallback is refused, said, and its module placed nowhere", false, false,
+				[]string{"manifest of mod-p", "manifest of mod-z"}},
+		} {
+			t.Run(row.name, func(t *testing.T) {
+				b := fixture()
+				b.manifestOf("mod-p", pManifest)
+				b.manifestOf("mod-z", zManifest)
+				b.assign("b", "mod-p")
+				if row.list {
+					b.put(grants.ReadPredefinedKeys, grants.PredefinedListKey, published)
+				}
+				in := grants.ReadInputs(b, &grants.Manifests{})
+				var refused []string
+				for _, g := range in.Refused {
+					refused = append(refused, g.Input)
+				}
+				if !slices.Equal(refused, row.refused) {
+					t.Errorf("refused = %v, want %v", in.Refused, row.refused)
+				}
+				if !row.list && !gapNamed(in.Gaps, grants.ReadPredefinedKeys) {
+					t.Errorf("gaps %v do not say the list is missing", in.Gaps)
+				}
+				recs, _ := grants.Records(in)
+				users, err := grants.ComposeUsers(recs)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if got := slices.Contains(names(users), "b.mod-p"); got != row.placed {
+					t.Errorf("b.mod-p composed %v, want %v: %v", got, row.placed, names(users))
+				}
+				// The pass and the tool listing the held memberships both say every refusal.
+				part := partOn(b)
+				res := part.Pass()
+				if len(res.Refused) != len(row.refused) || !strings.HasPrefix(res.Refused[len(res.Refused)-1], "manifest of mod-z: ") {
+					t.Errorf("the pass says refused %v, want %v", res.Refused, row.refused)
+				}
+				held, err := part.Held()
+				if err != nil {
+					t.Fatal(err)
+				}
+				if got, _ := held.(map[string]any)["refused"].([]string); len(got) != len(row.refused) {
+					t.Errorf("grants_membership lists refused %v, want %v", got, row.refused)
+				}
+			})
+		}
+	})
+
 	t.Run("credentials", func(t *testing.T) {
 		shadow := []grants.User{{User: "a.held"}, {User: "a.new"}, {User: "a.bare"}}
 		loaded := []grants.User{{User: "a.held"}, {User: "a.bare"}, {User: "enrol.x"}, {User: "person.op"}}

modules/mesh-grants/internal/grants/follow.go

@@ -17,8 +17,9 @@ type Watcher interface {
 
 // Followed are the states whose change composes again: placement, the indexer's manifests by module, the resolved
 // config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's
-// wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts.
-var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts}
+// wants (the manifest each machine is wanted to run), the vault's hashes of the bus accounts, and the list of predefined
+// keys the manifests are judged against.
+var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts, ReadPredefinedKeys}
 
 // Follower composes again on every change it hears, never on a timer. It never stops for a refusal: a watch it cannot
 // open is said, composed without, and opened again at the next wake (any change it does hear, or a reconnect).
@@ -39,6 +40,7 @@ type Follower struct {
 	// held and notAsserted are what the last pass held back and could not create, said again only when they change.
 	held        string
 	notAsserted string
+	refusedMans string
 }
 
 func (f *Follower) init() {
@@ -150,6 +152,18 @@ func (f *Follower) open() {
 // the pass could not create, each with why (issue 1790), when the set changes from the last pass's; and every stream it
 // created, one line per module.
 func (f *Follower) said(res Result) {
+	if refused := strings.Join(res.Refused, "\n"); refused != f.refusedMans {
+		f.refusedMans = refused
+		if len(res.Refused) == 0 {
+			f.logf("every manifest the indexer holds is read")
+		} else {
+			f.logf("%d manifests cannot be read, so their modules count as placed nowhere (grants_membership with no node and module lists them):",
+				len(res.Refused))
+			for _, r := range res.Refused {
+				f.logf("refused %s", r)
+			}
+		}
+	}
 	if held := strings.Join(res.NotPublished, "\n"); held != f.held {
 		f.held = held
 		if len(res.NotPublished) == 0 {

modules/mesh-grants/internal/grants/inputs.go

@@ -32,11 +32,19 @@ const (
 	ReadBuilds          = "mesh-delivery-builder.builds"
 	// ReadAccounts is the vault's bcrypt hashes of the bus accounts it holds, by bus user (ADR 0547 clause 9).
 	ReadAccounts = "mesh-vault.bus-accounts"
+	// ReadPredefinedKeys is the list of predefined keys the mesh published (the newest trunk build's of the catalogue,
+	// ADR 0348 §7), which the controller mirrors under one key, PredefinedListKey (mesh-sdk buscompose.PredefinedBucket):
+	// a manifest whose config value falls back to a predefined key is judged against it, as the controller judges it
+	// (issue 1945).
+	ReadPredefinedKeys = "mesh-controller.predefined"
 )
 
+// PredefinedListKey is the one key of the controller's predefined mirror.
+const PredefinedListKey = "list"
+
 // Reads is every state the part reads, in the manifest's order.
 var Reads = []string{ReadAssignments, ReadControllerGrant, ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds,
-	ReadAccounts}
+	ReadAccounts, ReadPredefinedKeys}
 
 // The bus module's tools the part calls, as its manifest's invokes name them: read-only, each.
 const (
@@ -114,7 +122,43 @@ type Inputs struct {
 	Loaded  *delivery.UserListWritten
 	// Hashes are the vault's hashes of the bus accounts, by bus user (ReadAccounts); kept in memory, never written.
 	Hashes map[string]HeldHash
-	Gaps   []Gap
+	// Predefined is the list of predefined keys the mesh published, nil when it cannot be read; PredefinedFrom says
+	// which build of the catalogue it came from.
+	Predefined     catalogue.PredefinedList
+	PredefinedFrom string
+	// Refused is every manifest the indexer holds that the part cannot read or mesh-sdk's reader refuses, assigned or
+	// not, each with why: its module counts as placed nowhere, so it is said loudly, never passed over (issue 1945).
+	Refused []Gap
+	Gaps    []Gap
+}
+
+// predefinedMirrored is the controller's predefined mirror's value (mesh-controller internal/inventory PredefinedMirrored).
+type predefinedMirrored struct {
+	Keys       catalogue.PredefinedList `json:"keys"`
+	Repository string                   `json:"repository"`
+	Commit     string                   `json:"commit"`
+	Generation int                      `json:"generation"`
+	Build      string                   `json:"build"`
+}
+
+// readPredefined reads the list of predefined keys the mesh published; what cannot be read is a gap and no list, so a
+// manifest falling back to a key is refused, and said, as with no list anywhere.
+func readPredefined(r Reader, gaps *[]Gap) (catalogue.PredefinedList, string) {
+	raw, found, err := r.Get(ReadPredefinedKeys, PredefinedListKey)
+	var v predefinedMirrored
+	switch {
+	case err != nil:
+		*gaps = append(*gaps, Gap{Input: ReadPredefinedKeys, Why: "refused or absent: " + err.Error() +
+			": a manifest falling back to a predefined key is refused"})
+		return nil, ""
+	case !found:
+		*gaps = append(*gaps, Gap{Input: ReadPredefinedKeys, Why: "the controller keeps no list yet: a manifest falling back to a predefined key is refused"})
+		return nil, ""
+	case json.Unmarshal(raw, &v) != nil || v.Keys == nil:
+		*gaps = append(*gaps, Gap{Input: ReadPredefinedKeys, Why: "holds no list of keys: a manifest falling back to a predefined key is refused"})
+		return nil, ""
+	}
+	return v.Keys, fmt.Sprintf("%s@%s#%d", v.Repository, v.Commit, v.Generation)
 }
 
 // HeldHash is one key of the vault's bus-accounts: the hash, and what it is of. The hash is never said or written by
@@ -143,16 +187,18 @@ func readHashes(r Reader, gaps *[]Gap) map[string]HeldHash {
 // BusModule is the bus's module: it holds mesh-broker.
 const BusModule = "nats"
 
-// Manifests caches manifests by digest: a digest names one manifest for ever.
+// Manifests caches manifests by digest and the list of predefined keys they were judged against: a digest names one
+// manifest for ever, and a newer list judges it again.
 type Manifests struct {
 	mu     sync.Mutex
 	parsed map[string]catalogue.Manifest
 }
 
-func (c *Manifests) get(r Reader, digest string) (catalogue.Manifest, error) {
+func (c *Manifests) get(r Reader, digest string, predefined catalogue.PredefinedList, from string) (catalogue.Manifest, error) {
+	key := digest + " " + from
 	// The lock guards the map alone, never a read on the bus: a tool never waits on a pass's round trip.
 	c.mu.Lock()
-	m, ok := c.parsed[digest]
+	m, ok := c.parsed[key]
 	c.mu.Unlock()
 	if ok {
 		return m, nil
@@ -168,7 +214,9 @@ func (c *Manifests) get(r Reader, digest string) (catalogue.Manifest, error) {
 	if err != nil {
 		return catalogue.Manifest{}, err
 	}
-	m, err = catalogue.ParseManifest([]byte(v.JSON))
+	// Judged against the list the mesh published, read from the bus, as the controller judges it (ADR 0348 §7): never
+	// against this process's own, which holds none (issue 1945).
+	m, err = catalogue.ParseManifestWith([]byte(v.JSON), predefined)
 	if err != nil {
 		return catalogue.Manifest{}, fmt.Errorf("the manifest %s is refused by mesh-sdk's reader: %w", digest, err)
 	}
@@ -177,7 +225,7 @@ func (c *Manifests) get(r Reader, digest string) (catalogue.Manifest, error) {
 	if c.parsed == nil {
 		c.parsed = map[string]catalogue.Manifest{}
 	}
-	c.parsed[digest] = m
+	c.parsed[key] = m
 	return m, nil
 }
 
@@ -207,6 +255,7 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
 	in := Inputs{Assigned: map[string][]string{}, Manifests: map[string]catalogue.Manifest{}, Digests: map[string]string{},
 		Running: map[string]catalogue.Manifest{},
 		Values:  map[string]any{}, Gaps: append([]Gap(nil), standingGaps...)}
+	in.Predefined, in.PredefinedFrom = readPredefined(r, &in.Gaps)
 	nodes := map[string]bool{}
 	for _, raw := range each(r, ReadAssignments, &in.Gaps) {
 		var a assignment
@@ -269,17 +318,15 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
 		if err == nil && v.Manifest == "" {
 			err = fmt.Errorf("the indexer's value names no manifest digest")
 		}
-		if err != nil {
-			if wanted[module] {
-				in.Gaps = append(in.Gaps, Gap{Input: "manifest of " + module, Why: err.Error()})
-			}
-			continue
+		var m catalogue.Manifest
+		if err == nil {
+			m, err = cache.get(r, v.Manifest, in.Predefined, in.PredefinedFrom)
 		}
-		m, err := cache.get(r, v.Manifest)
 		if err != nil {
-			if wanted[module] {
-				in.Gaps = append(in.Gaps, Gap{Input: "manifest of " + module, Why: err.Error()})
-			}
+			// Every manifest, assigned or not: one refused declares no seat, so its holders compose as placed nowhere.
+			g := Gap{Input: "manifest of " + module, Why: err.Error()}
+			in.Refused = append(in.Refused, g)
+			in.Gaps = append(in.Gaps, g)
 			continue
 		}
 		in.Manifests[module] = m
@@ -320,7 +367,7 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
 			fromTrunk = append(fromTrunk, node+"."+module)
 			continue
 		}
-		if m, err := cache.get(r, b.Manifest); err == nil {
+		if m, err := cache.get(r, b.Manifest, in.Predefined, in.PredefinedFrom); err == nil {
 			in.Running[node+"."+module] = m
 		} else {
 			fromTrunk = append(fromTrunk, node+"."+module)

modules/mesh-grants/internal/grants/part.go

@@ -96,8 +96,11 @@ type Result struct {
 	AssertionsWritten  int           `json:"assertions_written"`
 	// Asserted is each module whose declared streams this pass created, `<module> at <digest>: created …`, and
 	// NotAsserted each creation refused, `<module> <stream>: <why>` (issue 1790).
-	Asserted            []string `json:"asserted,omitempty"`
-	NotAsserted         []string `json:"not_asserted,omitempty"`
+	Asserted    []string `json:"asserted,omitempty"`
+	NotAsserted []string `json:"not_asserted,omitempty"`
+	// Refused is every manifest the indexer holds that this pass could not read, `manifest of <module>: <why>`: its
+	// module counts as placed nowhere (issue 1945).
+	Refused             []string `json:"refused,omitempty"`
 	Errors              []string `json:"errors,omitempty"`
 	PublishedOrAsserted string   `json:"published_or_asserted"`
 }
@@ -165,6 +168,9 @@ func (p *Part) Compose() Result {
 		p.mu.Unlock()
 	}()
 	in := ReadInputs(p.Read, &p.cache)
+	for _, g := range in.Refused {
+		res.Refused = append(res.Refused, g.Input+": "+g.Why)
+	}
 	recs, left := Records(in)
 	st := UserListState{V: 1, Mode: Shadow, LeftOut: left, Loaded: in.Loaded, BusNode: in.BusNode,
 		Gaps: append(append([]Gap(nil), in.Gaps...), MembershipGaps...), At: p.now()}

modules/mesh-grants/internal/grants/tools.go

@@ -116,7 +116,11 @@ func (p *Part) Held() (any, error) {
 		key, why, _ := strings.Cut(n, ": ")
 		out = append(out, held{Membership: key, Why: why})
 	}
-	return map[string]any{"held": out, "count": len(out), "pass": rev, "published": res.Published}, nil
+	refused := res.Refused
+	if refused == nil {
+		refused = []string{}
+	}
+	return map[string]any{"held": out, "count": len(out), "pass": rev, "published": res.Published, "refused": refused}, nil
 }
 
 // Pass composes and keeps the result as the last pass, for the tools to read.
@@ -198,7 +202,9 @@ func (p *Part) AssertOne(module string) (any, error) {
 	if err != nil || v.Manifest == "" {
 		return nil, fmt.Errorf("the indexer names no manifest of %s", module)
 	}
-	m, err := p.cache.get(p.Read, v.Manifest)
+	var gaps []Gap
+	list, from := readPredefined(p.Read, &gaps)
+	m, err := p.cache.get(p.Read, v.Manifest, list, from)
 	if err != nil {
 		return nil, err
 	}

modules/mesh-grants/module.json

@@ -11,7 +11,8 @@
     "mesh-delivery-indexer.manifests",
     "mesh-delivery-indexer.modules",
     "node-engine.have",
-    "mesh-vault.bus-accounts"
+    "mesh-vault.bus-accounts",
+    "mesh-controller.predefined"
   ],
   "invokes": [
     "nats.nats_streams",