Read the settings before the first pass resolves any machine, and refuse one machine alone for what it cannot keep or run (issues 1382, 1383)

The first pass of the adapter resolved every machine with no settings in the Input, so a provider whose served fact says ${setting:domain} could not be settled and the whole Input failed (the equality try on the live facts). The settings of every module that has any are read first; a module's settings that do not read are an error only for a machine that resolves to it. A recorded module that cannot be kept at the build a machine runs refuses that machine alone, as planFor did, and no longer the whole Input. The bus's refusal (no user has a credential) is carried for the machine that runs the bus.

cmd/mesh-controller/compose_input.go

@@ -35,10 +35,17 @@ func readComposeInput(ctx context.Context, open *stores) (*meshInput, error) {
 	if err != nil {
 		return nil, err
 	}
-	// What each node resolves to, with no credential and no settings given: the modules whose settings are read, and the
+	// What each node resolves to, with no credential given (the settings are read before it): the modules whose settings are read, and the
 	// needs whose pair credentials are asked of the stores. Nothing is written by it.
+	unread, err := readSettings(ctx, open, &in)
+	if err != nil {
+		return nil, err
+	}
 	plans := map[string]compose.Resolved{}
 	for _, n := range in.Nodes {
+		if shape.refused[n.Name] != nil {
+			continue // refused for what it keeps: nothing of it is read further
+		}
 		resolved, err := compose.Resolve(in, n.Name)
 		switch {
 		case compose.IsUnresolvable(err):
@@ -50,15 +57,16 @@ func readComposeInput(ctx context.Context, open *stores) (*meshInput, error) {
 		}
 		plans[n.Name] = resolved
 	}
-	if err := readSettings(ctx, open, &in, plans); err != nil {
+	if err := unreadSettings(unread, plans); err != nil {
 		return nil, err
 	}
-	wouldMake, err := readCredentialsAndBuilds(ctx, open, &in, plans, shape)
+	var busRefused error
+	wouldMake, err := readCredentialsAndBuilds(ctx, open, &in, plans, shape, &busRefused)
 	if err != nil {
 		return nil, err
 	}
 	composeInputBuilt()
-	return &meshInput{Input: in, wouldMake: wouldMake}, nil
+	return &meshInput{Input: in, wouldMake: wouldMake, busRefused: busRefused, refused: shape.refused}, nil
 }
 
 // meshInput is the Input an operation composes over, and the pair credentials its reading found none held for: the ones the
@@ -66,6 +74,11 @@ func readComposeInput(ctx context.Context, open *stores) (*meshInput, error) {
 type meshInput struct {
 	compose.Input
 	wouldMake []inventory.PairCredential
+	// busRefused is why the bus's user list cannot be composed (no user has a credential): the machine that runs the bus is
+	// refused with it, as the controller's own composition refused it, and no other machine is.
+	busRefused error
+	// refused is, by node, why that machine alone cannot be composed (nodeRefusal).
+	refused map[string]error
 }
 
 // nodeOf is the node of that name in an Input.
@@ -83,13 +96,16 @@ type inputShape struct {
 	sent    map[string]map[string]string // by node: module to the commit last sent
 	kept    map[string]map[string]string // by node: module to the commit kept
 	current map[string]inventory.CurrentBuild
+	// refused is, by node, why a recorded module cannot be kept at the build the machine runs (keepRecorded's refusal, in
+	// its words): that machine alone is refused with it, as planFor refused it, and nothing else of the mesh is.
+	refused map[string]error
 }
 
 // readMeshAndNodes is the mesh-wide and per-node half of the Input (issue 1382).
 func readMeshAndNodes(ctx context.Context, open *stores) (compose.Input, inputShape, error) {
 	inv := open.inventory
 	var in compose.Input
-	shape := inputShape{sent: map[string]map[string]string{}, kept: map[string]map[string]string{}}
+	shape := inputShape{sent: map[string]map[string]string{}, kept: map[string]map[string]string{}, refused: map[string]error{}}
 
 	shelf, err := inv.Catalogue(ctx)
 	if err != nil {
@@ -146,10 +162,14 @@ func readMeshAndNodes(ctx context.Context, open *stores) (compose.Input, inputSh
 	}
 
 	for _, n := range nodes {
-		node, err := readNode(ctx, open, n, placed[n.Name])
+		var refusal error
+		node, err := readNode(ctx, open, n, placed[n.Name], &refusal)
 		if err != nil {
 			return in, shape, err
 		}
+		if refusal != nil {
+			shape.refused[n.Name] = refusal
+		}
 		shape.sent[n.Name], shape.kept[n.Name] = node.SentBuilds, node.KeptBuilds
 		in.Nodes = append(in.Nodes, node)
 	}
@@ -162,7 +182,8 @@ func readMeshAndNodes(ctx context.Context, open *stores) (compose.Input, inputSh
 }
 
 // readNode is one node of the Input: everything the composition reads about it, in the order the controller read it.
-func readNode(ctx context.Context, open *stores, n inventory.Node, place inventory.Overlay) (compose.Node, error) {
+func readNode(ctx context.Context, open *stores, n inventory.Node, place inventory.Overlay,
+	refused *error) (compose.Node, error) {
 	inv := open.inventory
 	out := compose.Node{
 		Name: n.Name,
@@ -248,14 +269,15 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento
 	}
 	// A recorded module kept at the build this machine runs, in the context a send composes under (recorded_kept.go): its
 	// manifest and the commit, applied to this node's own resolution only.
+	// What cannot be kept refuses this machine alone (keepRecorded's refusal, as planFor returned it).
 	kept, err := recordedKept(ctx, open, n.Name)
 	if err != nil {
-		return out, err
+		*refused = err
-	}
+	} else if len(kept) > 0 {
-	if len(kept) > 0 {
 		out.KeptBuilds = kept
 		if out.Kept, err = keptManifests(ctx, open, n.Name, kept); err != nil {
-			return out, err
+			*refused = err
+			out.KeptBuilds, out.Kept = nil, nil
 		}
 	}
 	stated, has, err := inv.HealthOf(ctx, n.Name)
@@ -353,29 +375,36 @@ func readLicences(ctx context.Context, open *stores) ([]compose.Licence, error)
 	return out, nil
 }
 
-// readSettings is each node's settings layers for the modules it resolves to: the ones the composition reads for it and for
+// readSettings is each node's settings layers for every module that has any, read BEFORE any node is resolved: the first pass
-// the rest of the mesh, and no module nothing resolves (a module's settings may fail to read where nothing runs it).
+// resolves a node whose set takes a provider's served facts from the node that runs it, with that node's settings on them
-func readSettings(ctx context.Context, open *stores, in *compose.Input, plans map[string]compose.Resolved) error {
+// (servedOnNode), so a mesh whose provider says ${setting:domain} resolves only with the settings in the Input. A module's
-	modules := map[string]map[string]bool{}
+// settings that fail to read are returned in unread, by node and module, and are an error only for a module a node resolves to
-	for node, resolved := range plans {
+// (unreadSettings): the controller read none for a module nothing runs.
-		modules[node] = map[string]bool{}
+func readSettings(ctx context.Context, open *stores, in *compose.Input) (unread map[string]map[string]error, err error) {
-		for _, m := range resolved.Plan.Modules {
+	have, err := open.inventory.ModulesWithSettings(ctx)
-			modules[node][m.Module] = true
+	if err != nil {
+		return nil, err
+	}
+	names := make([]string, 0, len(have))
+	for module := range have {
+		if _, onShelf := in.Catalogue[module]; onShelf {
+			names = append(names, module)
 		}
 	}
-	// The rest of the mesh reads the settings of the modules that offer something on another node (servedOnNode), which are
+	sort.Strings(names)
-	// among the modules that node resolves to.
 	for i := range in.Nodes {
 		n := &in.Nodes[i]
-		names := make([]string, 0, len(modules[n.Name]))
-		for m := range modules[n.Name] {
-			names = append(names, m)
-		}
-		sort.Strings(names)
 		for _, module := range names {
 			layers, err := open.inventory.SettingsFor(ctx, n.Name, module)
 			if err != nil {
-				return err
+				if unread == nil {
+					unread = map[string]map[string]error{}
+				}
+				if unread[n.Name] == nil {
+					unread[n.Name] = map[string]error{}
+				}
+				unread[n.Name][module] = err
+				continue
 			}
 			if len(layers) == 0 {
 				continue
@@ -386,5 +415,22 @@ func readSettings(ctx context.Context, open *stores, in *compose.Input, plans ma
 			n.Settings[module] = layers
 		}
 	}
+	return unread, nil
+}
+
+// unreadSettings is the error of a module's settings that did not read, for a module the node resolves to.
+func unreadSettings(unread map[string]map[string]error, plans map[string]compose.Resolved) error {
+	names := make([]string, 0, len(plans))
+	for node := range plans {
+		names = append(names, node)
+	}
+	sort.Strings(names)
+	for _, node := range names {
+		for _, m := range plans[node].Plan.Modules {
+			if err := unread[node][m.Module]; err != nil {
+				return err
+			}
+		}
+	}
 	return nil
 }

cmd/mesh-controller/compose_input_credentials.go

@@ -16,11 +16,11 @@ import (
 // readCredentialsAndBuilds fills what the first half left: the credentials, the own secrets, the vault's boxes, the bus,
 // each node's serving key and certificate, and the builds.
 func readCredentialsAndBuilds(ctx context.Context, open *stores, in *compose.Input, plans map[string]compose.Resolved,
-	shape inputShape) ([]inventory.PairCredential, error) {
+	shape inputShape, busRefused *error) ([]inventory.PairCredential, error) {
 	if err := readIdentity(ctx, open, in); err != nil {
 		return nil, err
 	}
-	wouldMake, err := readCredentials(ctx, open, in, plans)
+	wouldMake, err := readCredentials(ctx, open, in, plans, busRefused)
 	if err != nil {
 		return nil, err
 	}
@@ -76,8 +76,8 @@ func readIdentity(ctx context.Context, open *stores, in *compose.Input) error {
 // consumers, the own secrets, the vault's boxes and the bus. It reads and never makes: a credential a send would make is
 // left out (the composition's would-make), a credential only a person gives and nobody gave is not-given, and one the send
 // would be refused is refused with the controller's words.
-func readCredentials(ctx context.Context, open *stores, in *compose.Input, plans map[string]compose.Resolved) (
+func readCredentials(ctx context.Context, open *stores, in *compose.Input, plans map[string]compose.Resolved,
-	[]inventory.PairCredential, error) {
+	busRefused *error) ([]inventory.PairCredential, error) {
 	var wouldMake []inventory.PairCredential
 	inv := open.inventory
 	shelf := in.Catalogue
@@ -182,6 +182,11 @@ func readCredentials(ctx context.Context, open *stores, in *compose.Input, plans
 		if err != nil && !strings.Contains(err.Error(), "not one user has a credential") {
 			return nil, err
 		}
+		if err != nil {
+			// The machine that runs the bus is refused, as the controller's own composition refused it: a list that
+			// refuses every connection in the mesh is never composed (refusedBus); the other machines compose.
+			*busRefused = err
+		}
 		in.BusUsers, in.BusMissing = list, missing
 	}
 	if in.BusMinted, err = inv.BusMinted(ctx); err != nil {

internal/inventory/read_state.go

@@ -232,3 +232,22 @@ func (i *Inventory) BusMinted(ctx context.Context) ([]string, error) {
 	sort.Strings(out)
 	return out, nil
 }
+
+// ModulesWithSettings is every module that has a settings row, mesh-wide or for a node: the modules whose layers
+// SettingsFor can answer with anything.
+func (i *Inventory) ModulesWithSettings(ctx context.Context) (map[string]bool, error) {
+	rows, err := i.store.Pool().Query(ctx, `select distinct module from settings`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := map[string]bool{}
+	for rows.Next() {
+		var module string
+		if err := rows.Scan(&module); err != nil {
+			return nil, err
+		}
+		out[module] = true
+	}
+	return out, rows.Err()
+}