cmd/mesh-controller/compose_input.go

@@ -35,10 +35,17 @@ func readComposeInput(ctx context.Context, open *stores) (*meshInput, error) {
 	if err != nil {
 		return nil, err
 	}
-	// What each node resolves to, with no credential and no settings given: the modules whose settings are read, and the
+	// What each node resolves to, with no credential given (the settings are read before it): the modules whose settings are read, and the
 	// needs whose pair credentials are asked of the stores. Nothing is written by it.
+	unread, err := readSettings(ctx, open, &in)
+	if err != nil {
+		return nil, err
+	}
 	plans := map[string]compose.Resolved{}
 	for _, n := range in.Nodes {
+		if shape.refused[n.Name] != nil {
+			continue // refused for what it keeps: nothing of it is read further
+		}
 		resolved, err := compose.Resolve(in, n.Name)
 		switch {
 		case compose.IsUnresolvable(err):
@@ -50,15 +57,16 @@ func readComposeInput(ctx context.Context, open *stores) (*meshInput, error) {
 		}
 		plans[n.Name] = resolved
 	}
-	if err := readSettings(ctx, open, &in, plans); err != nil {
+	if err := unreadSettings(unread, plans); err != nil {
 		return nil, err
 	}
-	wouldMake, err := readCredentialsAndBuilds(ctx, open, &in, plans, shape)
+	var busRefused error
+	wouldMake, err := readCredentialsAndBuilds(ctx, open, &in, plans, shape, &busRefused)
 	if err != nil {
 		return nil, err
 	}
 	composeInputBuilt()
-	return &meshInput{Input: in, wouldMake: wouldMake}, nil
+	return &meshInput{Input: in, wouldMake: wouldMake, busRefused: busRefused, refused: shape.refused}, nil
 }
 
 // meshInput is the Input an operation composes over, and the pair credentials its reading found none held for: the ones the
@@ -66,6 +74,11 @@ func readComposeInput(ctx context.Context, open *stores) (*meshInput, error) {
 type meshInput struct {
 	compose.Input
 	wouldMake []inventory.PairCredential
+	// busRefused is why the bus's user list cannot be composed (no user has a credential): the machine that runs the bus is
+	// refused with it, as the controller's own composition refused it, and no other machine is.
+	busRefused error
+	// refused is, by node, why that machine alone cannot be composed (nodeRefusal).
+	refused map[string]error
 }
 
 // nodeOf is the node of that name in an Input.
@@ -83,13 +96,16 @@ type inputShape struct {
 	sent    map[string]map[string]string // by node: module to the commit last sent
 	kept    map[string]map[string]string // by node: module to the commit kept
 	current map[string]inventory.CurrentBuild
+	// refused is, by node, why a recorded module cannot be kept at the build the machine runs (keepRecorded's refusal, in
+	// its words): that machine alone is refused with it, as planFor refused it, and nothing else of the mesh is.
+	refused map[string]error
 }
 
 // readMeshAndNodes is the mesh-wide and per-node half of the Input (issue 1382).
 func readMeshAndNodes(ctx context.Context, open *stores) (compose.Input, inputShape, error) {
 	inv := open.inventory
 	var in compose.Input
-	shape := inputShape{sent: map[string]map[string]string{}, kept: map[string]map[string]string{}}
+	shape := inputShape{sent: map[string]map[string]string{}, kept: map[string]map[string]string{}, refused: map[string]error{}}
 
 	shelf, err := inv.Catalogue(ctx)
 	if err != nil {
@@ -146,10 +162,14 @@ func readMeshAndNodes(ctx context.Context, open *stores) (compose.Input, inputSh
 	}
 
 	for _, n := range nodes {
-		node, err := readNode(ctx, open, n, placed[n.Name])
+		var refusal error
+		node, err := readNode(ctx, open, n, placed[n.Name], &refusal)
 		if err != nil {
 			return in, shape, err
 		}
+		if refusal != nil {
+			shape.refused[n.Name] = refusal
+		}
 		shape.sent[n.Name], shape.kept[n.Name] = node.SentBuilds, node.KeptBuilds
 		in.Nodes = append(in.Nodes, node)
 	}
@@ -162,7 +182,8 @@ func readMeshAndNodes(ctx context.Context, open *stores) (compose.Input, inputSh
 }
 
 // readNode is one node of the Input: everything the composition reads about it, in the order the controller read it.
-func readNode(ctx context.Context, open *stores, n inventory.Node, place inventory.Overlay) (compose.Node, error) {
+func readNode(ctx context.Context, open *stores, n inventory.Node, place inventory.Overlay,
+	refused *error) (compose.Node, error) {
 	inv := open.inventory
 	out := compose.Node{
 		Name: n.Name,
@@ -248,14 +269,15 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento
 	}
 	// A recorded module kept at the build this machine runs, in the context a send composes under (recorded_kept.go): its
 	// manifest and the commit, applied to this node's own resolution only.
+	// What cannot be kept refuses this machine alone (keepRecorded's refusal, as planFor returned it).
 	kept, err := recordedKept(ctx, open, n.Name)
 	if err != nil {
-		return out, err
-	}
-	if len(kept) > 0 {
+		*refused = err
+	} else if len(kept) > 0 {
 		out.KeptBuilds = kept
 		if out.Kept, err = keptManifests(ctx, open, n.Name, kept); err != nil {
-			return out, err
+			*refused = err
+			out.KeptBuilds, out.Kept = nil, nil
 		}
 	}
 	stated, has, err := inv.HealthOf(ctx, n.Name)
@@ -353,29 +375,36 @@ func readLicences(ctx context.Context, open *stores) ([]compose.Licence, error)
 	return out, nil
 }
 
-// readSettings is each node's settings layers for the modules it resolves to: the ones the composition reads for it and for
-// the rest of the mesh, and no module nothing resolves (a module's settings may fail to read where nothing runs it).
-func readSettings(ctx context.Context, open *stores, in *compose.Input, plans map[string]compose.Resolved) error {
-	modules := map[string]map[string]bool{}
-	for node, resolved := range plans {
-		modules[node] = map[string]bool{}
-		for _, m := range resolved.Plan.Modules {
-			modules[node][m.Module] = true
+// readSettings is each node's settings layers for every module that has any, read BEFORE any node is resolved: the first pass
+// resolves a node whose set takes a provider's served facts from the node that runs it, with that node's settings on them
+// (servedOnNode), so a mesh whose provider says ${setting:domain} resolves only with the settings in the Input. A module's
+// settings that fail to read are returned in unread, by node and module, and are an error only for a module a node resolves to
+// (unreadSettings): the controller read none for a module nothing runs.
+func readSettings(ctx context.Context, open *stores, in *compose.Input) (unread map[string]map[string]error, err error) {
+	have, err := open.inventory.ModulesWithSettings(ctx)
+	if err != nil {
+		return nil, err
+	}
+	names := make([]string, 0, len(have))
+	for module := range have {
+		if _, onShelf := in.Catalogue[module]; onShelf {
+			names = append(names, module)
 		}
 	}
-	// The rest of the mesh reads the settings of the modules that offer something on another node (servedOnNode), which are
-	// among the modules that node resolves to.
+	sort.Strings(names)
 	for i := range in.Nodes {
 		n := &in.Nodes[i]
-		names := make([]string, 0, len(modules[n.Name]))
-		for m := range modules[n.Name] {
-			names = append(names, m)
-		}
-		sort.Strings(names)
 		for _, module := range names {
 			layers, err := open.inventory.SettingsFor(ctx, n.Name, module)
 			if err != nil {
-				return err
+				if unread == nil {
+					unread = map[string]map[string]error{}
+				}
+				if unread[n.Name] == nil {
+					unread[n.Name] = map[string]error{}
+				}
+				unread[n.Name][module] = err
+				continue
 			}
 			if len(layers) == 0 {
 				continue
@@ -386,5 +415,22 @@ func readSettings(ctx context.Context, open *stores, in *compose.Input, plans ma
 			n.Settings[module] = layers
 		}
 	}
+	return unread, nil
+}
+
+// unreadSettings is the error of a module's settings that did not read, for a module the node resolves to.
+func unreadSettings(unread map[string]map[string]error, plans map[string]compose.Resolved) error {
+	names := make([]string, 0, len(plans))
+	for node := range plans {
+		names = append(names, node)
+	}
+	sort.Strings(names)
+	for _, node := range names {
+		for _, m := range plans[node].Plan.Modules {
+			if err := unread[node][m.Module]; err != nil {
+				return err
+			}
+		}
+	}
 	return nil
 }

cmd/mesh-controller/compose_input_credentials.go

@@ -16,11 +16,11 @@ import (
 // readCredentialsAndBuilds fills what the first half left: the credentials, the own secrets, the vault's boxes, the bus,
 // each node's serving key and certificate, and the builds.
 func readCredentialsAndBuilds(ctx context.Context, open *stores, in *compose.Input, plans map[string]compose.Resolved,
-	shape inputShape) ([]inventory.PairCredential, error) {
+	shape inputShape, busRefused *error) ([]inventory.PairCredential, error) {
 	if err := readIdentity(ctx, open, in); err != nil {
 		return nil, err
 	}
-	wouldMake, err := readCredentials(ctx, open, in, plans)
+	wouldMake, err := readCredentials(ctx, open, in, plans, busRefused)
 	if err != nil {
 		return nil, err
 	}
@@ -76,8 +76,8 @@ func readIdentity(ctx context.Context, open *stores, in *compose.Input) error {
 // consumers, the own secrets, the vault's boxes and the bus. It reads and never makes: a credential a send would make is
 // left out (the composition's would-make), a credential only a person gives and nobody gave is not-given, and one the send
 // would be refused is refused with the controller's words.
-func readCredentials(ctx context.Context, open *stores, in *compose.Input, plans map[string]compose.Resolved) (
-	[]inventory.PairCredential, error) {
+func readCredentials(ctx context.Context, open *stores, in *compose.Input, plans map[string]compose.Resolved,
+	busRefused *error) ([]inventory.PairCredential, error) {
 	var wouldMake []inventory.PairCredential
 	inv := open.inventory
 	shelf := in.Catalogue
@@ -182,6 +182,11 @@ func readCredentials(ctx context.Context, open *stores, in *compose.Input, plans
 		if err != nil && !strings.Contains(err.Error(), "not one user has a credential") {
 			return nil, err
 		}
+		if err != nil {
+			// The machine that runs the bus is refused, as the controller's own composition refused it: a list that
+			// refuses every connection in the mesh is never composed (refusedBus); the other machines compose.
+			*busRefused = err
+		}
 		in.BusUsers, in.BusMissing = list, missing
 	}
 	if in.BusMinted, err = inv.BusMinted(ctx); err != nil {

internal/inventory/read_state.go

@@ -232,3 +232,22 @@ func (i *Inventory) BusMinted(ctx context.Context) ([]string, error) {
 	sort.Strings(out)
 	return out, nil
 }
+
+// ModulesWithSettings is every module that has a settings row, mesh-wide or for a node: the modules whose layers
+// SettingsFor can answer with anything.
+func (i *Inventory) ModulesWithSettings(ctx context.Context) (map[string]bool, error) {
+	rows, err := i.store.Pool().Query(ctx, `select distinct module from settings`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := map[string]bool{}
+	for rows.Next() {
+		var module string
+		if err := rows.Scan(&module); err != nil {
+			return nil, err
+		}
+		out[module] = true
+	}
+	return out, rows.Err()
+}