Where the answer to a requirement is allowed to live

Two different things were both written `requires`. A shell, a display
server and a private network have to be on the machine that needs them.
A database does not — it runs somewhere and is reached over the network.
Both were answered the same way, so requiring a database installed
PostgreSQL on every machine that ran a web application.

What a module provides now carries a scope, the same idea claims already
use, written short in the ordinary case:

  "provides": ["shell"]
  "provides": [{"name": "database", "scope": "mesh"}]

A mesh-scoped requirement is answered by finding the node already running
it — never by installing it here. Choosing a machine to put a database on
is a decision with consequences, and nothing resolving a web application
should make it silently. With nothing anywhere it refuses and says which
module to assign; with two it refuses and says how to choose.

Choosing is `pin <node> <provision> <from>`, kept per node because that
is the granularity the choice has. A pin at a machine that does not
provide it refuses rather than falling back — a fallback would quietly
move somebody's data. One provider does not overrule a pin either.

Resolving a node now needs to know what the others offer, and working
that out needs them resolved, so it is two passes: the first answers only
what each node offers, the second answers everything. Nothing is ever
declared from the first.

A node's plan says what it takes from elsewhere. It is the only part of a
set that stops working when a different machine goes away, and nothing
else in that output would have said so. It is also where a credential
will hang once there is a mechanism for handing one back.

One test found passing for the wrong reason: it read pins through a join
on the provider, which hides a dangling row whether or not it was cleaned
up. It counts rows now, and bites when the cascade is removed.

cmd/mesh-control/main.go

@@ -63,6 +63,10 @@ func run() error {
 	defer stop()
 
 	switch args[0] {
+	case "pin":
+		return pinCommand(ctx, args[1:], true)
+	case "unpin":
+		return pinCommand(ctx, args[1:], false)
 	case "migrate":
 		return migrate(ctx)
 	case "node":
@@ -127,6 +131,8 @@ func usage() {
   settings set <module> <file>   what a module's config should say, for the whole mesh
   settings set <module> <file> --node <n>   ...or for one machine
   settings clear <module> [--node <n>]      take a layer away
+  pin <node> <provision> <from>  which node this one gets a provision from
+  unpin <node> <provision>       put that question back
   plan <node> [--files|--json]     what that node would run, and why
   push [<node>]                  send a node everything it should be
   version                        what this binary is
@@ -850,7 +856,7 @@ func moduleCommand(ctx context.Context, args []string) error {
 			fmt.Printf(" from %s", short(*commit))
 		}
 		if len(m.Provides) > 0 {
-			fmt.Printf(", providing %s", strings.Join(m.Provides, ", "))
+			fmt.Printf(", providing %s", describeOffers(m.Provides))
 		}
 		fmt.Println()
 		for _, c := range m.Claims {
@@ -876,7 +882,7 @@ func moduleCommand(ctx context.Context, args []string) error {
 			m := shelf[n]
 			fmt.Printf("%-20s", m.Module)
 			if len(m.Provides) > 0 {
-				fmt.Printf("  provides %s", strings.Join(m.Provides, ", "))
+				fmt.Printf("  provides %s", describeOffers(m.Provides))
 			}
 			for _, c := range m.Claims {
 				fmt.Printf("  claims %s/%s", c.At(), c.Name)
@@ -979,31 +985,17 @@ func planFor(ctx context.Context, inv *inventory.Inventory, nodeName string) (ca
 		}
 	}
 
-	// What every other node already holds, so the claims wider than one machine can be checked.
-	// Resolved rather than read from a table: a claim is held by whatever a node actually runs,
-	// and a record of it would be a second answer that could disagree with the first.
-	var elsewhere []catalogue.Held
-	for _, p := range places {
-		if p.Name == nodeName {
-			continue
-		}
-		theirs, err := inv.Assigned(ctx, p.Name)
-		if err != nil || len(theirs) == 0 {
-			continue
-		}
-		theirCaps, _ := inv.ProfileOf(ctx, p.Name)
-		got, err := catalogue.Resolve(shelf, theirs,
-			catalogue.Node{Name: p.Name, Site: p.Site, Capabilities: theirCaps}, nil)
-		if err != nil {
-			// Their set does not resolve either. Not this node's problem to report, and their
-			// claims cannot be counted because nothing of theirs is running.
-			continue
-		}
-		elsewhere = append(elsewhere, got.Claims...)
+	world, err := theRestOfTheMesh(ctx, inv, shelf, nodeName)
+	if err != nil {
+		return catalogue.Resolution{}, nil, err
+	}
+	world.Pinned, err = inv.PinsFor(ctx, nodeName)
+	if err != nil {
+		return catalogue.Resolution{}, nil, err
 	}
 
 	resolved, err := catalogue.Resolve(shelf, assigned,
-		catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities}, elsewhere)
+		catalogue.Node{Name: nodeName, Site: site, Capabilities: capabilities}, world)
 	if err != nil {
 		return catalogue.Resolution{}, nil, err
 	}
@@ -1033,6 +1025,82 @@ func planFor(ctx context.Context, inv *inventory.Inventory, nodeName string) (ca
 	return resolved, settings, nil
 }
 
+// theRestOfTheMesh is what every other node holds and offers.
+//
+// Two things at once because they come from the same place — resolving the other nodes — and
+// because both are facts about what is actually running rather than records that could disagree
+// with it. A claim is held by whatever a node runs; a provision is offered by whatever a node
+// runs; neither is a table somebody keeps up to date.
+//
+// **Two passes over the others.** What a node offers the mesh needs that node resolved, and
+// resolving it may need what the mesh offers. So the first pass takes brokered requirements on
+// trust and answers only *what does each node offer*; the second answers everything with that in
+// hand. Nothing is ever declared from the first.
+func theRestOfTheMesh(ctx context.Context, inv *inventory.Inventory,
+	shelf map[string]catalogue.Manifest, exclude string) (catalogue.World, error) {
+
+	// Every node, not only the placed ones. A machine that was never put on the private network
+	// still runs modules, still holds claims, and still offers whatever it offers.
+	nodes, err := inv.Nodes(ctx)
+	if err != nil {
+		return catalogue.World{}, err
+	}
+	places, err := inv.Overlays(ctx)
+	if err != nil {
+		return catalogue.World{}, err
+	}
+	siteOf := map[string]string{}
+	for _, p := range places {
+		siteOf[p.Name] = p.Site
+	}
+
+	type candidate struct {
+		node     catalogue.Node
+		assigned []string
+	}
+	var others []candidate
+	for _, n := range nodes {
+		if n.Name == exclude {
+			continue
+		}
+		theirs, err := inv.Assigned(ctx, n.Name)
+		if err != nil || len(theirs) == 0 {
+			continue
+		}
+		caps, _ := inv.ProfileOf(ctx, n.Name)
+		others = append(others, candidate{
+			catalogue.Node{Name: n.Name, Site: siteOf[n.Name], Capabilities: caps}, theirs})
+	}
+
+	offered := map[string][]string{}
+	for _, o := range others {
+		got, err := catalogue.Resolve(shelf, o.assigned, o.node, catalogue.World{Unchecked: true})
+		if err != nil {
+			// Their set does not resolve for some other reason. Not this node's problem to
+			// report, and nothing of theirs is running, so it offers nothing.
+			continue
+		}
+		for _, m := range got.Modules {
+			for _, name := range m.OffersAt(catalogue.ScopeMesh) {
+				offered[name] = append(offered[name], o.node.Name)
+			}
+		}
+	}
+	for k := range offered {
+		sort.Strings(offered[k])
+	}
+
+	world := catalogue.World{Offered: offered}
+	for _, o := range others {
+		got, err := catalogue.Resolve(shelf, o.assigned, o.node, world)
+		if err != nil {
+			continue
+		}
+		world.Held = append(world.Held, got.Claims...)
+	}
+	return world, nil
+}
+
 func planCommand(ctx context.Context, args []string) error {
 	set := flag.NewFlagSet("plan", flag.ContinueOnError)
 	// Because "one resource" does not tell you whether the settings landed. Being able to read
@@ -1090,6 +1158,12 @@ func planCommand(ctx context.Context, args []string) error {
 	for _, c := range plan.Claims {
 		fmt.Printf("  holds %s, one per %s\n", c.Claim, c.Scope)
 	}
+	// What this machine depends on that is not on it. Worth saying out loud: it is the only part
+	// of a node's set that stops working when a *different* machine goes away, and nothing else
+	// in this output would have told anybody that.
+	for _, n := range plan.Needs {
+		fmt.Printf("  needs %s from %s, for %s\n", n.Name, n.From, n.For)
+	}
 	gens, err := generators(ctx, inv)
 	if err != nil {
 		return err
@@ -1350,3 +1424,58 @@ func settingsCommand(ctx context.Context, args []string) error {
 		return fmt.Errorf("settings has no %q; it has set and clear", args[0])
 	}
 }
+
+// describeOffers says what a module provides, and marks the ones answered from anywhere in the
+// mesh — because "provides a database" and "provides a shell" are read the same way and mean
+// entirely different things about where the answer has to be.
+func describeOffers(offers []catalogue.Offer) string {
+	var out []string
+	for _, o := range offers {
+		if o.At() == catalogue.ScopeMesh {
+			out = append(out, o.Name+" (from anywhere in the mesh)")
+			continue
+		}
+		out = append(out, o.Name)
+	}
+	return strings.Join(out, ", ")
+}
+
+// pinCommand says which node a machine gets a provision from.
+//
+// Needed only when more than one could answer, and recordable before that -- a mesh with one
+// database should not change where an existing machine gets its data the day a second one
+// arrives.
+func pinCommand(ctx context.Context, args []string, setting bool) error {
+	if setting && len(args) != 3 {
+		return errors.New("pin <node> <provision> <from-node>")
+	}
+	if !setting && len(args) != 2 {
+		return errors.New("unpin <node> <provision>")
+	}
+	inv, err := openInventory(ctx)
+	if err != nil {
+		return err
+	}
+	defer inv.Close()
+
+	if !setting {
+		if err := inv.UnpinProvision(ctx, args[0], args[1]); err != nil {
+			return err
+		}
+		fmt.Printf("%s is no longer told where to get %s from\n", args[0], args[1])
+		return nil
+	}
+	if args[0] == args[2] {
+		// Allowed by nothing here, and worth saying rather than resolving into a confusing
+		// refusal later: a node providing something to itself is a node-scoped provision, and
+		// this field is for the other kind.
+		return fmt.Errorf("%s cannot get %s from itself; that would be a provision this machine "+
+			"provides, which does not need saying", args[0], args[1])
+	}
+	if err := inv.PinProvision(ctx, args[0], args[1], args[2]); err != nil {
+		return err
+	}
+	fmt.Printf("%s gets %s from %s\n", args[0], args[1], args[2])
+	fmt.Printf("  run `push %s` to send it\n", args[0])
+	return nil
+}

internal/catalogue/brokered_test.go

@@ -0,0 +1,179 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// Where the answer to a requirement is allowed to live.
+//
+// A shell, a display server and a private network have to be on the machine that needs them. A
+// database does not — it runs somewhere and is reached over the network. Both were written
+// `requires`, so both were answered the same way, and the second answer was to install PostgreSQL
+// on every machine that runs a web application.
+
+func brokeredShelf() map[string]Manifest {
+	return shelf(
+		Manifest{Module: "postgres", Version: "1", Provides: FromAnywhere("database")},
+		Manifest{Module: "meshboard", Version: "1", Requires: []string{"database"}},
+	)
+}
+
+func TestADatabaseIsNotInstalledOnEveryMachineThatUsesOne(t *testing.T) {
+	// The fault this whole distinction exists for.
+	got, err := Resolve(brokeredShelf(), []string{"meshboard"}, workstation(),
+		World{Offered: map[string][]string{"database": {"anchor"}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if have := strings.Join(names(got), " "); strings.Contains(have, "postgres") {
+		t.Fatalf("using a database installed one here: %s", have)
+	}
+}
+
+func TestWhatAMachineTakesFromElsewhereIsRecorded(t *testing.T) {
+	// It is the only part of a node's set that stops working when a *different* machine goes
+	// away, and it is where a credential will have to be handed back.
+	got, err := Resolve(brokeredShelf(), []string{"meshboard"}, workstation(),
+		World{Offered: map[string][]string{"database": {"anchor"}}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got.Needs) != 1 {
+		t.Fatalf("got %v", got.Needs)
+	}
+	if got.Needs[0].Name != "database" || got.Needs[0].From != "anchor" {
+		t.Fatalf("got %v", got.Needs[0])
+	}
+	if got.Needs[0].For != "meshboard" {
+		t.Fatalf("it does not say what wanted it: %v", got.Needs[0])
+	}
+}
+
+func TestNothingInTheMeshProvidingItIsRefusedWithSomewhereToPutIt(t *testing.T) {
+	// Refused rather than installed here. Choosing a machine to put a database on is a decision
+	// with consequences, and nothing resolving a web application should make it silently.
+	_, err := Resolve(brokeredShelf(), []string{"meshboard"}, workstation(), World{})
+	if err == nil {
+		t.Fatal("a database was found in a mesh that has none")
+	}
+	if !strings.Contains(err.Error(), "assign") || !strings.Contains(err.Error(), "postgres") {
+		t.Fatalf("the refusal does not say what to do: %v", err)
+	}
+}
+
+func TestTwoNodesProvidingItIsRefusedRatherThanPicked(t *testing.T) {
+	// Same rule as everywhere else. Picking one would be a guess about which database a person
+	// meant, and the wrong guess is somebody's data in the wrong place.
+	_, err := Resolve(brokeredShelf(), []string{"meshboard"}, workstation(),
+		World{Offered: map[string][]string{"database": {"anchor", "archive"}}})
+	if err == nil {
+		t.Fatal("one of two databases was picked silently")
+	}
+	for _, want := range []string{"anchor", "archive", "pin"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Fatalf("the refusal does not name %s: %v", want, err)
+		}
+	}
+}
+
+func TestSayingWhichOneSettlesIt(t *testing.T) {
+	got, err := Resolve(brokeredShelf(), []string{"meshboard"}, workstation(),
+		World{
+			Offered: map[string][]string{"database": {"anchor", "archive"}},
+			Pinned:  map[string]string{"database": "archive"},
+		})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got.Needs) != 1 || got.Needs[0].From != "archive" {
+		t.Fatalf("the choice was not taken: %v", got.Needs)
+	}
+}
+
+func TestBeingPointedAtAMachineThatDoesNotProvideItIsRefused(t *testing.T) {
+	// Rather than falling back to one that does. A fallback would quietly move somebody's data to
+	// a machine they did not choose, which is the whole reason the question is asked.
+	_, err := Resolve(brokeredShelf(), []string{"meshboard"}, workstation(),
+		World{
+			Offered: map[string][]string{"database": {"anchor", "archive"}},
+			Pinned:  map[string]string{"database": "somewhere-else"},
+		})
+	if err == nil {
+		t.Fatal("a machine was silently given a different database from the one chosen")
+	}
+	if !strings.Contains(err.Error(), "somewhere-else") {
+		t.Fatalf("the refusal does not say what was chosen: %v", err)
+	}
+}
+
+func TestOneProviderDoesNotOverruleAChoice(t *testing.T) {
+	// A single answer is normally taken silently. Not when somebody said they wanted a different
+	// one -- that is the mesh overruling a person, which it does nowhere else.
+	_, err := Resolve(brokeredShelf(), []string{"meshboard"}, workstation(),
+		World{
+			Offered: map[string][]string{"database": {"anchor"}},
+			Pinned:  map[string]string{"database": "archive"},
+		})
+	if err == nil {
+		t.Fatal("the only database was used although another was chosen")
+	}
+	if !strings.Contains(err.Error(), "only anchor provides it") {
+		t.Fatalf("the refusal does not say what is available: %v", err)
+	}
+}
+
+func TestACatalogueThatDisagreesAboutScopeIsRefused(t *testing.T) {
+	// If one module says a database is local and another says it is anywhere, the same
+	// requirement means two things depending on which one happens to answer it.
+	_, err := Resolve(shelf(
+		Manifest{Module: "postgres", Version: "1", Provides: FromAnywhere("database")},
+		Manifest{Module: "sqlite", Version: "1", Provides: Offers("database")},
+		Manifest{Module: "meshboard", Version: "1", Requires: []string{"database"}},
+	), []string{"meshboard"}, workstation(), World{})
+	if err == nil {
+		t.Fatal("a catalogue that disagrees about where a database lives was accepted")
+	}
+	if !strings.Contains(err.Error(), "two things") {
+		t.Fatalf("unhelpful refusal: %v", err)
+	}
+}
+
+func TestALocalRequirementIsStillAnsweredLocally(t *testing.T) {
+	// The change must not have made everything brokered. A shell is still installed here.
+	got, err := Resolve(shelf(
+		Manifest{Module: "zsh", Version: "1", Provides: Offers("shell")},
+		Manifest{Module: "tools", Version: "1", Requires: []string{"shell"}},
+	), []string{"tools"}, workstation(), World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if have := strings.Join(names(got), " "); !strings.Contains(have, "zsh") {
+		t.Fatalf("a shell was not installed on the machine that needs one: %s", have)
+	}
+}
+
+func TestTheShortFormStillMeansHere(t *testing.T) {
+	// `"provides": ["shell"]` must keep meaning what it meant, or every existing manifest
+	// silently changes meaning.
+	m, err := ParseManifest([]byte(`{"module":"zsh","version":"1","provides":["shell"]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(m.Provides) != 1 || m.Provides[0].At() != ScopeNode {
+		t.Fatalf("a plain name is no longer node-scoped: %v", m.Provides)
+	}
+}
+
+func TestASiteScopedProvisionIsRefused(t *testing.T) {
+	// Meaningful for a claim — one DHCP server per segment — and not yet meaningful for a
+	// provision, because nothing knows how to reach "the one at my site".
+	_, err := ParseManifest([]byte(
+		`{"module":"dns","version":"1","provides":[{"name":"resolver","scope":"site"}]}`))
+	if err == nil {
+		t.Fatal("a site-scoped provision was accepted")
+	}
+	if !strings.Contains(err.Error(), "site") {
+		t.Fatalf("unhelpful refusal: %v", err)
+	}
+}

internal/catalogue/computed_test.go

@@ -32,13 +32,13 @@ func (f *fake) Resources(node string) ([]map[string]any, bool, error) {
 
 func computedShelf() map[string]Manifest {
 	return shelf(Manifest{Module: "mesh-network", Computed: "mesh-network",
-		Provides: []string{"private-network"}})
+		Provides: Offers("private-network")})
 }
 
 func TestAComputedModuleIsAskedAboutTheNodeItIsFor(t *testing.T) {
 	// The generator gets a node name, not a plan. Everything it needs is the whole mesh, which
 	// it was built with — this is the one thing a node could never work out for itself.
-	got, err := Resolve(computedShelf(), []string{"mesh-network"}, workstation(), nil)
+	got, err := Resolve(computedShelf(), []string{"mesh-network"}, workstation(), World{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -58,7 +58,7 @@ func TestAComputedModuleIsAskedAboutTheNodeItIsFor(t *testing.T) {
 func TestAMachineNobodyGaveItGetsNothing(t *testing.T) {
 	// The whole point of making the network a module. Before this, every machine with an address
 	// was on the private network and there was no way to say one should stay off.
-	got, err := Resolve(computedShelf(), nil, workstation(), nil)
+	got, err := Resolve(computedShelf(), nil, workstation(), World{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -78,7 +78,7 @@ func TestAMachineNobodyGaveItGetsNothing(t *testing.T) {
 func TestAssignedAndNotYetPartOfItIsNotAFailure(t *testing.T) {
 	// A machine given the network module before it has a place on it. Brief and ordinary — the
 	// answer is "nothing yet", and treating it as an error would make an ordering a fault.
-	got, _ := Resolve(computedShelf(), []string{"mesh-network"}, workstation(), nil)
+	got, _ := Resolve(computedShelf(), []string{"mesh-network"}, workstation(), World{})
 	gen := &fake{on: map[string]bool{}}
 	out, err := got.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": gen}})
 	if err != nil {
@@ -92,8 +92,8 @@ func TestAssignedAndNotYetPartOfItIsNotAFailure(t *testing.T) {
 func TestAGeneratorThisControlPlaneDoesNotHaveIsRefused(t *testing.T) {
 	// Sending a machine a module with no files would look like it worked. Named in the message,
 	// because the only fix is a control plane that has it.
-	got, _ := Resolve(shelf(Manifest{Module: "weather", Computed: "the-weather"}),
-		[]string{"weather"}, workstation(), nil)
+	got, _ := Resolve(shelf(Manifest{Module: "weather", Computed: "the-weather"}), []string{"weather"}, workstation(), World{})
+
 	_, err := got.Declaration(Rendering{Generators: map[string]Generator{"mesh-network": &fake{}}})
 	if err == nil {
 		t.Fatal("a module computed by nothing was accepted")
@@ -119,7 +119,7 @@ func TestAModuleIsEitherWrittenOrComputedNotBoth(t *testing.T) {
 func TestSettingsApplyToAComputedModuleToo(t *testing.T) {
 	// Being computed is about where the files come from, not about whether they are configurable.
 	// A line drawn there would be arbitrary and nobody could predict it.
-	got, _ := Resolve(computedShelf(), []string{"mesh-network"}, workstation(), nil)
+	got, _ := Resolve(computedShelf(), []string{"mesh-network"}, workstation(), World{})
 	gen := &fake{on: map[string]bool{"workstation": true}}
 	out, err := got.Declaration(Rendering{
 		Generators: map[string]Generator{"mesh-network": gen},
@@ -143,10 +143,11 @@ func TestTwoWaysToBeOnAPrivateNetworkRefuseAndNameBoth(t *testing.T) {
 	// both is caught rather than producing a machine on two networks that each half-work.
 	_, err := Resolve(shelf(
 		Manifest{Module: "mesh-network", Computed: "mesh-network",
-			Provides: []string{"private-network"}},
-		Manifest{Module: "tailscale", Provides: []string{"private-network"}},
+			Provides: Offers("private-network")},
+		Manifest{Module: "tailscale", Provides: Offers("private-network")},
 		Manifest{Module: "backups", Requires: []string{"private-network"}},
-	), []string{"backups"}, workstation(), nil)
+	), []string{"backups"}, workstation(), World{})
+
 	if err == nil {
 		t.Fatal("two answers to one requirement were taken silently")
 	}

internal/catalogue/contributes_test.go

@@ -23,7 +23,7 @@ func published(module, host string, port int) Manifest {
 
 func proxy() Manifest {
 	return Manifest{Module: "traefik", Version: "1",
-		Provides: []string{"reverse-proxy"},
+		Provides: Offers("reverse-proxy"),
 		Receives: map[string]string{"reverse-proxy": "/etc/traefik/mesh.json"},
 		Resources: []map[string]any{
 			{"id": "up", "type": "service", "unit": "traefik", "state": "running",
@@ -58,8 +58,8 @@ func received(t *testing.T, out []map[string]any) []Contribution {
 func TestTheFileTheProviderGetsIsMachineReadable(t *testing.T) {
 	// It is written for a program, and the first version put a `//` header above the JSON — a
 	// file that says "do not edit" to a person and fails to parse for the thing meant to read it.
-	got, _ := Resolve(shelf(proxy(), published("board", "board", 8080)),
-		[]string{"board"}, workstation(), nil)
+	got, _ := Resolve(shelf(proxy(), published("board", "board", 8080)), []string{"board"}, workstation(), World{})
+
 	for _, r := range mustDeclare(t, got) {
 		if r["path"] != "/etc/traefik/mesh.json" {
 			continue
@@ -79,8 +79,8 @@ func TestTheFileTheProviderGetsIsMachineReadable(t *testing.T) {
 }
 
 func TestAModuleTellsItsProviderWhatItNeeds(t *testing.T) {
-	got, err := Resolve(shelf(proxy(), published("board", "board", 8080)),
-		[]string{"board"}, workstation(), nil)
+	got, err := Resolve(shelf(proxy(), published("board", "board", 8080)), []string{"board"}, workstation(), World{})
+
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -100,8 +100,8 @@ func TestAModuleTellsItsProviderWhatItNeeds(t *testing.T) {
 func TestContributingToSomethingIsRequiringIt(t *testing.T) {
 	// Asking to be published means a publisher must exist. A module that had to say both would
 	// eventually say one, and the failure would be a machine where nothing serves the route.
-	got, err := Resolve(shelf(proxy(), published("board", "board", 8080)),
-		[]string{"board"}, workstation(), nil)
+	got, err := Resolve(shelf(proxy(), published("board", "board", 8080)), []string{"board"}, workstation(), World{})
+
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -111,8 +111,8 @@ func TestContributingToSomethingIsRequiringIt(t *testing.T) {
 }
 
 func TestNothingToContributeToIsRefused(t *testing.T) {
-	_, err := Resolve(shelf(published("board", "board", 8080)),
-		[]string{"board"}, workstation(), nil)
+	_, err := Resolve(shelf(published("board", "board", 8080)), []string{"board"}, workstation(), World{})
+
 	if err == nil {
 		t.Fatal("a module was published through a proxy that does not exist")
 	}
@@ -125,7 +125,8 @@ func TestEveryPublisherOnTheMachineIsInOneFile(t *testing.T) {
 	got, err := Resolve(shelf(proxy(),
 		published("board", "board", 8080),
 		published("archive", "archive", 9000),
-	), []string{"board", "archive"}, workstation(), nil)
+	), []string{"board", "archive"}, workstation(), World{})
+
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -144,7 +145,7 @@ func TestAProviderWithNoConsumersStillGetsTheFile(t *testing.T) {
 	// Empty rather than absent. A provider that finds no file cannot tell "nothing asked for me"
 	// from "the mesh never wrote it", and those want completely different responses — the same
 	// rule the host follows about a service that does not exist.
-	got, err := Resolve(shelf(proxy()), []string{"traefik"}, workstation(), nil)
+	got, err := Resolve(shelf(proxy()), []string{"traefik"}, workstation(), World{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -157,8 +158,8 @@ func TestTheProviderCanReloadWhenTheRoutesChange(t *testing.T) {
 	// A proxy that got a new route and did not reload is a route that silently does not work.
 	// The same fault the private network had when a peer list changed under a running interface,
 	// which is why the received file has a name a module can point at.
-	got, _ := Resolve(shelf(proxy(), published("board", "board", 8080)),
-		[]string{"board"}, workstation(), nil)
+	got, _ := Resolve(shelf(proxy(), published("board", "board", 8080)), []string{"board"}, workstation(), World{})
+
 	out := mustDeclare(t, got)
 	for _, r := range out {
 		if r["type"] != "service" {
@@ -179,8 +180,8 @@ func TestTheProviderCanReloadWhenTheRoutesChange(t *testing.T) {
 func TestARouteCanBeSetPerMesh(t *testing.T) {
 	// The hostname is exactly the kind of thing that differs between one mesh and the next. A
 	// module whose route could not be set would have to be edited to be reused anywhere.
-	got, _ := Resolve(shelf(proxy(), published("board", "board", 8080)),
-		[]string{"board"}, workstation(), nil)
+	got, _ := Resolve(shelf(proxy(), published("board", "board", 8080)), []string{"board"}, workstation(), World{})
+
 	out, err := got.Declaration(Rendering{Settings: SettingsBy{
 		"board": {{From: "the mesh", Values: map[string]any{"host": "dashboard"}}},
 	}})

internal/catalogue/domain_test.go

@@ -16,10 +16,10 @@ func networkingShelf(extra ...Manifest) map[string]Manifest {
 	base := []Manifest{
 		{Module: "networking", Requires: []string{"private-network", "name-resolution"}},
 		{Module: "mesh-wireguard", Computed: "mesh-wireguard",
-			Provides: []string{"private-network", "mesh-addressing"},
+			Provides: Offers("private-network", "mesh-addressing"),
 			Claims:   []Claim{{Name: "the-private-network", Scope: ScopeNode}}},
 		{Module: "mesh-names", Computed: "mesh-names",
-			Provides: []string{"name-resolution"}, Requires: []string{"mesh-addressing"}},
+			Provides: Offers("name-resolution"), Requires: []string{"mesh-addressing"}},
 	}
 	return shelf(append(base, extra...)...)
 }
@@ -27,7 +27,7 @@ func networkingShelf(extra ...Manifest) map[string]Manifest {
 func TestOneWordBringsUpTheNetwork(t *testing.T) {
 	// The case that has to stay easy. Nothing is asked, because with one answer to each
 	// requirement there was never a question.
-	got, err := Resolve(networkingShelf(), []string{"networking"}, workstation(), nil)
+	got, err := Resolve(networkingShelf(), []string{"networking"}, workstation(), World{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -43,9 +43,10 @@ func TestASecondVPNTurnsItIntoAChoice(t *testing.T) {
 	// And the choice is offered rather than made. A default here would be the flavor field coming
 	// back under another name.
 	_, err := Resolve(networkingShelf(
-		Manifest{Module: "tailscale", Provides: []string{"private-network"},
+		Manifest{Module: "tailscale", Provides: Offers("private-network"),
 			Claims: []Claim{{Name: "the-private-network", Scope: ScopeNode}}},
-	), []string{"networking"}, workstation(), nil)
+	), []string{"networking"}, workstation(), World{})
+
 	if err == nil {
 		t.Fatal("two VPNs and one was picked silently")
 	}
@@ -60,9 +61,10 @@ func TestChoosingIsAssigning(t *testing.T) {
 	// No second verb. Assigning the one you want answers the requirement, and the bundle takes it.
 	got, err := Resolve(networkingShelf(
 		Manifest{Module: "tailscale",
-			Provides: []string{"private-network", "name-resolution"},
+			Provides: Offers("private-network", "name-resolution"),
 			Claims:   []Claim{{Name: "the-private-network", Scope: ScopeNode}}},
-	), []string{"networking", "tailscale"}, workstation(), nil)
+	), []string{"networking", "tailscale"}, workstation(), World{})
+
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -82,9 +84,10 @@ func TestChoosingOneVPNCannotDragTheOtherBackIn(t *testing.T) {
 	// The claim is what catches it. Providing a private network is not the singular part — a
 	// machine could run two VPNs for two purposes — but being *the* one the mesh runs over is.
 	_, err := Resolve(networkingShelf(
-		Manifest{Module: "tailscale", Provides: []string{"private-network"},
+		Manifest{Module: "tailscale", Provides: Offers("private-network"),
 			Claims: []Claim{{Name: "the-private-network", Scope: ScopeNode}}},
-	), []string{"networking", "tailscale"}, workstation(), nil)
+	), []string{"networking", "tailscale"}, workstation(), World{})
+
 	if err == nil {
 		t.Fatal("a machine was given two private networks without being told")
 	}
@@ -100,9 +103,10 @@ func TestNamesNeedTheMeshsOwnAddresses(t *testing.T) {
 	// mean nothing on it.
 	_, err := Resolve(shelf(
 		Manifest{Module: "mesh-names", Computed: "mesh-names",
-			Provides: []string{"name-resolution"}, Requires: []string{"mesh-addressing"}},
-		Manifest{Module: "tailscale", Provides: []string{"private-network"}},
-	), []string{"mesh-names", "tailscale"}, workstation(), nil)
+			Provides: Offers("name-resolution"), Requires: []string{"mesh-addressing"}},
+		Manifest{Module: "tailscale", Provides: Offers("private-network")},
+	), []string{"mesh-names", "tailscale"}, workstation(), World{})
+
 	if err == nil {
 		t.Fatal("the mesh's names were installed over a VPN whose addresses it does not hand out")
 	}
@@ -117,9 +121,10 @@ func TestARequirementWantedTwiceIsReportedOnce(t *testing.T) {
 	_, err := Resolve(shelf(
 		Manifest{Module: "one", Requires: []string{"shell"}},
 		Manifest{Module: "two", Requires: []string{"shell"}},
-		Manifest{Module: "bash", Provides: []string{"shell"}},
-		Manifest{Module: "zsh", Provides: []string{"shell"}},
-	), []string{"one", "two"}, workstation(), nil)
+		Manifest{Module: "bash", Provides: Offers("shell")},
+		Manifest{Module: "zsh", Provides: Offers("shell")},
+	), []string{"one", "two"}, workstation(), World{})
+
 	if err == nil {
 		t.Fatal("two shells and one was picked silently")
 	}

internal/catalogue/manifest.go

@@ -45,6 +45,63 @@ func (c Claim) At() string {
 	return c.Scope
 }
 
+// Offer is something a module provides, and where the answer to it may live.
+//
+// **The distinction this exists for:** a shell, a display server and a private network have to be
+// on the machine that needs them. A database, an object store and an identity provider do not —
+// they run somewhere in the mesh and are reached over it. Treating the second as the first
+// installs PostgreSQL on every machine that runs a web application, which is what happened until
+// this field existed.
+//
+// Written as a bare string in the ordinary case, because nearly everything is node-scoped and
+// making every manifest say so would bury the few that are not:
+//
+//	"provides": ["shell"]
+//	"provides": [{"name": "database", "scope": "mesh"}]
+type Offer struct {
+	Name string `json:"name"`
+	// Scope defaults to the node, which is where most things must be to be usable.
+	Scope string `json:"scope,omitempty"`
+}
+
+// At is this offer's scope, with the default applied.
+func (o Offer) At() string {
+	if o.Scope == "" {
+		return ScopeNode
+	}
+	return o.Scope
+}
+
+// UnmarshalJSON accepts a plain name as well as an object.
+func (o *Offer) UnmarshalJSON(raw []byte) error {
+	var plain string
+	if err := json.Unmarshal(raw, &plain); err == nil {
+		o.Name, o.Scope = plain, ""
+		return nil
+	}
+	var full struct {
+		Name  string `json:"name"`
+		Scope string `json:"scope,omitempty"`
+	}
+	if err := json.Unmarshal(raw, &full); err != nil {
+		return fmt.Errorf("a provided name is either a string or {name, scope}: %w", err)
+	}
+	o.Name, o.Scope = full.Name, full.Scope
+	return nil
+}
+
+// MarshalJSON writes back the short form when there is nothing else to say, so a manifest that
+// went through the mesh comes out looking like the one that went in.
+func (o Offer) MarshalJSON() ([]byte, error) {
+	if o.Scope == "" {
+		return json.Marshal(o.Name)
+	}
+	return json.Marshal(struct {
+		Name  string `json:"name"`
+		Scope string `json:"scope"`
+	}{o.Name, o.Scope})
+}
+
 // Manifest is everything a module says about itself.
 type Manifest struct {
 	Module  string `json:"module"`
@@ -52,7 +109,7 @@ type Manifest struct {
 
 	// Provides are the names other modules may require. A module always provides its own name;
 	// this is for the rest — `zsh` provides `shell`, `xorg` provides `display-server`.
-	Provides []string `json:"provides,omitempty"`
+	Provides []Offer `json:"provides,omitempty"`
 
 	// Requires are names that must be provided by something assigned to the same node.
 	Requires []string `json:"requires,omitempty"`
@@ -148,10 +205,19 @@ func ParseManifest(raw []byte) (Manifest, error) {
 		problems = append(problems, fmt.Sprintf(
 			"%q is not a usable module name: lower-case letters, digits, dashes and dots", m.Module))
 	}
-	for _, p := range m.Provides {
+	for _, offer := range m.Provides {
+		p := offer.Name
 		if !name.MatchString(p) {
 			problems = append(problems, fmt.Sprintf("%q is not a usable name to provide", p))
 		}
+		if s := offer.At(); s != ScopeNode && s != ScopeMesh {
+			// Site scope is meaningful for a claim — one DHCP server per segment — and is not
+			// yet meaningful for a provision, because nothing knows how to reach "the one at my
+			// site". Refused rather than silently treated as mesh-wide.
+			problems = append(problems, fmt.Sprintf(
+				"%s provides %q at scope %q; a provision is %q or %q",
+				m.Module, p, s, ScopeNode, ScopeMesh))
+		}
 		if p == m.Module {
 			// Harmless and worth saying: a module always provides its own name, so writing it
 			// suggests the author expected it not to.
@@ -238,7 +304,26 @@ func ParseManifest(raw []byte) (Manifest, error) {
 
 // Offers is everything this module can satisfy: its own name, and what it provides.
 func (m Manifest) Offers() []string {
-	out := append([]string{m.Module}, m.Provides...)
+	out := []string{m.Module}
+	for _, p := range m.Provides {
+		out = append(out, p.Name)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// OffersAt is what this module provides at one scope, with its own name counted as node-scoped:
+// a module is only ever itself on the machine it is installed on.
+func (m Manifest) OffersAt(scope string) []string {
+	var out []string
+	if scope == ScopeNode {
+		out = append(out, m.Module)
+	}
+	for _, p := range m.Provides {
+		if p.At() == scope {
+			out = append(out, p.Name)
+		}
+	}
 	sort.Strings(out)
 	return out
 }

internal/catalogue/provided_test.go

@@ -35,8 +35,8 @@ func provided(t *testing.T) map[string]catalogue.Manifest {
 }
 
 func TestTheShippedNetworkingModulesResolveOnTheirOwn(t *testing.T) {
-	got, err := catalogue.Resolve(provided(t), []string{overlay.Domain},
-		catalogue.Node{Name: "workstation", Site: "house"}, nil)
+	got, err := catalogue.Resolve(provided(t), []string{overlay.Domain}, catalogue.Node{Name: "workstation", Site: "house"}, catalogue.World{})
+
 	if err != nil {
 		t.Fatalf("assigning %s does not work out of the box: %v", overlay.Domain, err)
 	}
@@ -58,7 +58,8 @@ func TestTheShippedWireGuardModuleClaimsBeingTheNetwork(t *testing.T) {
 	_, err := catalogue.Resolve(
 		withTailscale(shipped),
 		[]string{overlay.Domain, "tailscale"},
-		catalogue.Node{Name: "workstation", Site: "house"}, nil)
+		catalogue.Node{Name: "workstation", Site: "house"}, catalogue.World{})
+
 	if err == nil {
 		t.Fatal("a machine was given two private networks and nobody was told")
 	}
@@ -72,8 +73,8 @@ func TestTheShippedNamesModuleNeedsTheMeshsOwnAddresses(t *testing.T) {
 	// is what stops a machine getting a hosts file that means nothing on it.
 	shipped := provided(t)
 	delete(shipped, overlay.Name)
-	_, err := catalogue.Resolve(shipped, []string{overlay.Names},
-		catalogue.Node{Name: "workstation", Site: "house"}, nil)
+	_, err := catalogue.Resolve(shipped, []string{overlay.Names}, catalogue.Node{Name: "workstation", Site: "house"}, catalogue.World{})
+
 	if err == nil {
 		t.Fatal("the mesh's names resolved with nothing handing out the mesh's addresses")
 	}
@@ -91,7 +92,7 @@ func withTailscale(shelf map[string]catalogue.Manifest) map[string]catalogue.Man
 	// both VPNs: the names then needed the mesh's addressing, and only WireGuard has it.
 	out["tailscale"] = catalogue.Manifest{
 		Module: "tailscale", Version: "1",
-		Provides: []string{overlay.Requirement},
+		Provides: catalogue.Offers(overlay.Requirement),
 		Claims:   []catalogue.Claim{{Name: overlay.TheNetwork, Scope: catalogue.ScopeNode}},
 	}
 	return out

internal/catalogue/resolve.go

@@ -24,6 +24,30 @@ type Node struct {
 	Capabilities map[string]bool
 }
 
+// World is what the rest of the mesh already has.
+//
+// Some requirements cannot be answered on the machine that has them — a database runs somewhere
+// and is reached over the network — so resolving one node needs to know what the others offer.
+type World struct {
+	// Held is the claims already taken, for the scopes wider than one node.
+	Held []Held
+	// Offered is what other nodes provide at mesh scope: the name, and which nodes provide it.
+	Offered map[string][]string
+	// Pinned is which node this machine was told to get a provision from, by name. Only consulted
+	// when more than one node could answer -- a choice recorded before it was needed should not
+	// start meaning something the day a second provider appears, and one recorded and then made
+	// unnecessary should not quietly stop applying either.
+	Pinned map[string]string
+	// Unchecked takes brokered requirements on trust instead of refusing when nothing answers
+	// them.
+	//
+	// For the first of two passes. Working out what a node offers the mesh needs that node
+	// resolved, and resolving it may need what the mesh offers — so the first pass answers only
+	// *what does each node offer*, and the second pass answers everything with that in hand. A
+	// declaration is never built from an unchecked resolution.
+	Unchecked bool
+}
+
 // Held is a claim somebody already has, used for the scopes wider than one node.
 type Held struct {
 	Claim  string
@@ -44,6 +68,20 @@ type Resolution struct {
 	Because map[string]string
 	// Claims is what this node's set holds, so wider scopes can be checked against it.
 	Claims []Held
+	// Needs is what this node's set gets from other nodes. Recorded rather than resolved away,
+	// because it is where a credential will have to be handed back once there is a mechanism for
+	// that, and because "what does this machine depend on that is not on it" has no other answer.
+	Needs []Needed
+}
+
+// Needed is one thing this node's set takes from elsewhere in the mesh.
+type Needed struct {
+	// Name is the provision, as required.
+	Name string
+	// From is the node providing it.
+	From string
+	// For is the module that wanted it.
+	For string
 }
 
 // Refusal is why a set of assignments cannot become a declaration.
@@ -64,9 +102,33 @@ var ErrAmbiguous = errors.New("more than one module provides that")
 // The catalogue is every module the mesh knows about; assigned is what a person put on this node.
 // What comes back is the closure — assigned modules plus everything they require — or a refusal
 // naming every reason it could not be closed.
-func Resolve(catalogue map[string]Manifest, assigned []string, node Node, elsewhere []Held) (Resolution, error) {
+func Resolve(catalogue map[string]Manifest, assigned []string, node Node, world World) (Resolution, error) {
+	elsewhere := world.Held
 	var problems []string
 
+	// Which names are answered from elsewhere in the mesh rather than from this machine. A
+	// property of the name, not of each provider: two modules disagreeing about whether a
+	// database is local would make the same requirement mean different things depending on which
+	// one happened to answer it.
+	brokered := map[string]bool{}
+	local := map[string]bool{}
+	for _, m := range catalogue {
+		for _, o := range m.Provides {
+			if o.At() == ScopeMesh {
+				brokered[o.Name] = true
+				continue
+			}
+			local[o.Name] = true
+		}
+	}
+	for want := range brokered {
+		if local[want] {
+			problems = append(problems, fmt.Sprintf(
+				"the catalogue disagrees about %q: some modules provide it here and others from "+
+					"anywhere in the mesh, so the same requirement would mean two things", want))
+		}
+	}
+
 	// What each name can be satisfied by. Built once from the whole catalogue, because "how many
 	// modules provide this" is the question the whole rule turns on.
 	offers := map[string][]string{}
@@ -82,6 +144,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, elsewh
 	chosen := map[string]bool{}
 	because := map[string]string{}
 	var order []string
+	var needs []Needed
 
 	// What the set already offers, which is the first thing a requirement is checked against.
 	//
@@ -120,6 +183,67 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, elsewh
 			continue
 		}
 
+		// Answered from elsewhere in the mesh, if it is that kind of name. **Never installed
+		// here.** Choosing a machine to put a database on is a decision with consequences
+		// nobody would want made silently by something resolving a web application.
+		if brokered[want] {
+			reported[want] = true
+			where := world.Offered[want]
+			switch {
+			case world.Unchecked:
+				// First pass. Whether anything answers this is exactly the question this pass
+				// exists to make answerable, so it is not asked here.
+			case len(where) == 0:
+				remedy := "and nothing in the catalogue could"
+				if answers := offers[want]; len(answers) == 1 {
+					remedy = "— assign " + answers[0] + " to a node"
+				} else if len(answers) > 1 {
+					remedy = "— assign one of these to a node: " + strings.Join(answers, ", ")
+				}
+				problems = append(problems, fmt.Sprintf(
+					"nothing in this mesh provides %q, wanted by %s %s",
+					want, because[want], remedy))
+			case len(where) == 1:
+				if chosenNode, pinned := world.Pinned[want]; pinned && chosenNode != where[0] {
+					// One provider, and it is not the one this machine was told to use. Silently
+					// using the other would be the mesh overruling a choice somebody made.
+					problems = append(problems, fmt.Sprintf(
+						"%s was told to get %q from %s, and only %s provides it",
+						node.Name, want, chosenNode, where[0]))
+					break
+				}
+				needs = append(needs, Needed{Name: want, From: where[0], For: because[want]})
+			default:
+				sorted := append([]string{}, where...)
+				sort.Strings(sorted)
+				chosenNode, pinned := world.Pinned[want]
+				if !pinned {
+					problems = append(problems, fmt.Sprintf(
+						"%d nodes provide %q, wanted by %s — say which with `pin %s %s <node>`: %s",
+						len(where), want, because[want], node.Name, want,
+						strings.Join(sorted, ", ")))
+					break
+				}
+				var offers bool
+				for _, w := range where {
+					if w == chosenNode {
+						offers = true
+					}
+				}
+				if !offers {
+					// Pointed at a machine that does not answer this. Refused rather than
+					// falling back to another: a fallback would quietly move somebody's data to
+					// a machine they did not choose, which is the whole reason this is asked.
+					problems = append(problems, fmt.Sprintf(
+						"%s was told to get %q from %s, and %s does not provide it — these do: %s",
+						node.Name, want, chosenNode, chosenNode, strings.Join(sorted, ", ")))
+					break
+				}
+				needs = append(needs, Needed{Name: want, From: chosenNode, For: because[want]})
+			}
+			continue
+		}
+
 		candidates := offers[want]
 		switch len(candidates) {
 		case 0:
@@ -159,7 +283,7 @@ func Resolve(catalogue map[string]Manifest, assigned []string, node Node, elsewh
 		}
 	}
 
-	resolution := Resolution{Node: node.Name, Because: because}
+	resolution := Resolution{Node: node.Name, Because: because, Needs: needs}
 	for _, n := range order {
 		resolution.Modules = append(resolution.Modules, catalogue[n])
 	}
@@ -437,3 +561,21 @@ func sortedKeys[V any](m map[string]V) []string {
 	sort.Strings(out)
 	return out
 }
+
+// Offers is a list of node-scoped provisions, which is what nearly everything is.
+func Offers(names ...string) []Offer {
+	out := make([]Offer, 0, len(names))
+	for _, n := range names {
+		out = append(out, Offer{Name: n})
+	}
+	return out
+}
+
+// FromAnywhere is a provision answered by whichever node in the mesh runs it.
+func FromAnywhere(names ...string) []Offer {
+	out := make([]Offer, 0, len(names))
+	for _, n := range names {
+		out = append(out, Offer{Name: n, Scope: ScopeMesh})
+	}
+	return out
+}

internal/catalogue/resolve_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func mod(name string, provides, requires, capabilities []string, claims ...Claim) Manifest {
-	return Manifest{Module: name, Provides: provides, Requires: requires,
+	return Manifest{Module: name, Provides: Offers(provides...), Requires: requires,
 		Capabilities: capabilities, Claims: claims}
 }
 
@@ -38,7 +38,8 @@ func TestARequirementWithOneAnswerIsTakenSilently(t *testing.T) {
 	got, err := Resolve(shelf(
 		mod("i3", nil, []string{"xorg"}, []string{"seat"}),
 		mod("xorg", []string{"display-server"}, nil, []string{"seat"}, Claim{Name: "the-seat"}),
-	), []string{"i3"}, workstation(), nil)
+	), []string{"i3"}, workstation(), World{})
+
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -58,7 +59,8 @@ func TestARequirementWithSeveralAnswersIsRefusedAndNamed(t *testing.T) {
 		mod("bash", []string{"shell"}, nil, nil),
 		mod("zsh", []string{"shell"}, nil, nil),
 		mod("fish", []string{"shell"}, nil, nil),
-	), []string{"editor"}, workstation(), nil)
+	), []string{"editor"}, workstation(), World{})
+
 	if err == nil {
 		t.Fatal("a requirement with three answers was resolved without asking")
 	}
@@ -70,8 +72,8 @@ func TestARequirementWithSeveralAnswersIsRefusedAndNamed(t *testing.T) {
 }
 
 func TestARequirementWithNoAnswerIsRefused(t *testing.T) {
-	_, err := Resolve(shelf(mod("i3", nil, []string{"xorg"}, nil)),
-		[]string{"i3"}, workstation(), nil)
+	_, err := Resolve(shelf(mod("i3", nil, []string{"xorg"}, nil)), []string{"i3"}, workstation(), World{})
+
 	if err == nil || !strings.Contains(err.Error(), "nothing provides") {
 		t.Fatalf("a requirement nothing satisfies gave %v", err)
 	}
@@ -84,7 +86,8 @@ func TestSeveralModulesMayProvideTheSameThingAndCoexist(t *testing.T) {
 		mod("bash", []string{"shell"}, nil, nil),
 		mod("zsh", []string{"shell"}, nil, nil),
 		mod("fish", []string{"shell"}, nil, nil),
-	), []string{"bash", "zsh", "fish"}, workstation(), nil)
+	), []string{"bash", "zsh", "fish"}, workstation(), World{})
+
 	if err != nil {
 		t.Fatalf("three shells could not coexist: %v", err)
 	}
@@ -99,7 +102,8 @@ func TestTwoModulesClaimingOneThingAreRefused(t *testing.T) {
 	_, err := Resolve(shelf(
 		mod("xorg", []string{"display-server"}, nil, nil, Claim{Name: "the-seat"}),
 		mod("wayland", []string{"display-server"}, nil, nil, Claim{Name: "the-seat"}),
-	), []string{"xorg", "wayland"}, workstation(), nil)
+	), []string{"xorg", "wayland"}, workstation(), World{})
+
 	if err == nil {
 		t.Fatal("two modules claiming the seat were both assigned")
 	}
@@ -118,11 +122,11 @@ func TestAThirdModuleNeedsNoChangeToTheOthers(t *testing.T) {
 		mod("mir", []string{"display-server"}, nil, nil, Claim{Name: "the-seat"}),
 	)
 	for _, pair := range [][]string{{"xorg", "mir"}, {"wayland", "mir"}} {
-		if _, err := Resolve(catalogue, pair, workstation(), nil); err == nil {
+		if _, err := Resolve(catalogue, pair, workstation(), World{}); err == nil {
 			t.Errorf("%v were both assigned", pair)
 		}
 	}
-	if _, err := Resolve(catalogue, []string{"mir"}, workstation(), nil); err != nil {
+	if _, err := Resolve(catalogue, []string{"mir"}, workstation(), World{}); err != nil {
 		t.Errorf("the newcomer alone was refused: %v", err)
 	}
 }
@@ -131,8 +135,8 @@ func TestAMissingCapabilityIsSaidToBeTheWrongMachine(t *testing.T) {
 	// The remedy differs from a missing module and the message has to say which. Nothing can be
 	// installed to give a server a seat.
 	server := Node{Name: "server", Capabilities: map[string]bool{"container-runtime": true}}
-	_, err := Resolve(shelf(mod("xorg", nil, nil, []string{"seat"}, Claim{Name: "the-seat"})),
-		[]string{"xorg"}, server, nil)
+	_, err := Resolve(shelf(mod("xorg", nil, nil, []string{"seat"}, Claim{Name: "the-seat"})), []string{"xorg"}, server, World{})
+
 	if err == nil {
 		t.Fatal("a display server was assigned to a machine with no seat")
 	}
@@ -146,7 +150,7 @@ func TestAMeshWideClaimIsHeldByOneNode(t *testing.T) {
 	// cannot.
 	_, err := Resolve(shelf(mod("hub", nil, nil, nil, Claim{Name: "the-hub", Scope: ScopeMesh})),
 		[]string{"hub"}, workstation(),
-		[]Held{{Claim: "the-hub", Scope: ScopeMesh, Node: "anchor", Module: "hub"}})
+		World{Held: []Held{{Claim: "the-hub", Scope: ScopeMesh, Node: "anchor", Module: "hub"}}})
 	if err == nil {
 		t.Fatal("two nodes both hold a mesh-wide claim")
 	}
@@ -161,12 +165,12 @@ func TestASiteClaimOnlyCollidesWithinThatSite(t *testing.T) {
 	catalogue := shelf(mod("dhcp", nil, nil, nil, Claim{Name: "dhcp", Scope: ScopeSite}))
 
 	elsewhere := []Held{{Claim: "dhcp", Scope: ScopeSite, Node: "other", Module: "dhcp", Site: "house"}}
-	if _, err := Resolve(catalogue, []string{"dhcp"}, workstation(), elsewhere); err == nil {
+	if _, err := Resolve(catalogue, []string{"dhcp"}, workstation(), World{Held: elsewhere}); err == nil {
 		t.Error("two DHCP servers at one site were allowed")
 	}
 
 	faraway := []Held{{Claim: "dhcp", Scope: ScopeSite, Node: "other", Module: "dhcp", Site: "office"}}
-	if _, err := Resolve(catalogue, []string{"dhcp"}, workstation(), faraway); err != nil {
+	if _, err := Resolve(catalogue, []string{"dhcp"}, workstation(), World{Held: faraway}); err != nil {
 		t.Errorf("a DHCP server at another site was treated as a collision: %v", err)
 	}
 }
@@ -179,7 +183,7 @@ func TestTwoModulesWritingOneFileAreRefusedWithoutAnyClaim(t *testing.T) {
 	b := mod("b", nil, nil, nil)
 	b.Resources = []map[string]any{{"id": "conf", "type": "file", "path": "/etc/thing.conf"}}
 
-	_, err := Resolve(shelf(a, b), []string{"a", "b"}, workstation(), nil)
+	_, err := Resolve(shelf(a, b), []string{"a", "b"}, workstation(), World{})
 	if err == nil {
 		t.Fatal("two modules writing the same file were both assigned")
 	}
@@ -196,7 +200,7 @@ func TestResourceIdentitiesCarryTheirModule(t *testing.T) {
 	b := mod("b", nil, nil, nil)
 	b.Resources = []map[string]any{{"id": "config", "type": "file", "path": "/etc/b"}}
 
-	got, err := Resolve(shelf(a, b), []string{"a", "b"}, workstation(), nil)
+	got, err := Resolve(shelf(a, b), []string{"a", "b"}, workstation(), World{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -222,7 +226,7 @@ func TestWhatAServiceReflectsIsQualifiedToo(t *testing.T) {
 		{"id": "svc", "type": "service", "unit": "thing.service", "state": "running",
 			"restart-on": []any{"conf"}},
 	}
-	got, err := Resolve(shelf(m), []string{"thing"}, workstation(), nil)
+	got, err := Resolve(shelf(m), []string{"thing"}, workstation(), World{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -243,7 +247,8 @@ func TestEveryReasonIsGivenAtOnce(t *testing.T) {
 	_, err := Resolve(shelf(
 		mod("xorg", nil, nil, []string{"seat"}, Claim{Name: "the-seat"}),
 		mod("wayland", nil, nil, []string{"seat"}, Claim{Name: "the-seat"}),
-	), []string{"xorg", "wayland"}, server, nil)
+	), []string{"xorg", "wayland"}, server, World{})
+
 	if err == nil {
 		t.Fatal("expected refusals")
 	}
@@ -258,7 +263,8 @@ func TestACycleStopsRatherThanRunsAway(t *testing.T) {
 	got, err := Resolve(shelf(
 		mod("a", nil, []string{"b"}, nil),
 		mod("b", nil, []string{"a"}, nil),
-	), []string{"a"}, workstation(), nil)
+	), []string{"a"}, workstation(), World{})
+
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -276,7 +282,8 @@ func TestChoosingOneSatisfiesTheRequirement(t *testing.T) {
 		mod("bash", []string{"shell"}, nil, nil),
 		mod("zsh", []string{"shell"}, nil, nil),
 		mod("fish", []string{"shell"}, nil, nil),
-	), []string{"editor", "zsh"}, workstation(), nil)
+	), []string{"editor", "zsh"}, workstation(), World{})
+
 	if err != nil {
 		t.Fatalf("choosing a shell did not satisfy the requirement for one: %v", err)
 	}
@@ -293,7 +300,8 @@ func TestChoosingSeveralIsStillFine(t *testing.T) {
 		mod("bash", []string{"shell"}, nil, nil),
 		mod("zsh", []string{"shell"}, nil, nil),
 		mod("fish", []string{"shell"}, nil, nil),
-	), []string{"editor", "zsh", "bash", "fish"}, workstation(), nil)
+	), []string{"editor", "zsh", "bash", "fish"}, workstation(), World{})
+
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -310,7 +318,8 @@ func TestARequirementNamingAModuleMeansThatModule(t *testing.T) {
 		mod("i3", nil, []string{"xorg"}, nil),
 		mod("xorg", []string{"display-server"}, nil, nil),
 		mod("wayland", []string{"display-server", "xorg"}, nil, nil),
-	), []string{"i3", "wayland"}, workstation(), nil)
+	), []string{"i3", "wayland"}, workstation(), World{})
+
 	// wayland claiming to provide "xorg" is a manifest saying something untrue; what matters is
 	// that a real xorg module still wins when it exists, and that the answer is not silent.
 	if err != nil && !strings.Contains(err.Error(), "xorg") {

internal/inventory/catalogue.go

@@ -436,3 +436,82 @@ func (i *Inventory) SettingsFor(ctx context.Context, nodeName, module string) ([
 	}
 	return layers, rows.Err()
 }
+
+// PinProvision records which node a machine gets a provision from.
+//
+// Only needed when more than one node could answer. Recordable before that, because a mesh with
+// one database should not change where an existing machine gets its data the day a second
+// arrives.
+func (i *Inventory) PinProvision(ctx context.Context, nodeName, provision, provider string) error {
+	node, err := i.NodeByName(ctx, nodeName)
+	if err != nil {
+		return err
+	}
+	from, err := i.NodeByName(ctx, provider)
+	if err != nil {
+		return err
+	}
+	_, err = i.store.Pool().Exec(ctx,
+		`insert into provision_pin (node, name, provider) values ($1, $2, $3)
+		 on conflict (node, name) do update set provider = excluded.provider, pinned_at = now()`,
+		node.ID, provision, from.ID)
+	return err
+}
+
+// UnpinProvision removes a choice, putting the question back.
+func (i *Inventory) UnpinProvision(ctx context.Context, nodeName, provision string) error {
+	node, err := i.NodeByName(ctx, nodeName)
+	if err != nil {
+		return err
+	}
+	tag, err := i.store.Pool().Exec(ctx,
+		`delete from provision_pin where node = $1 and name = $2`, node.ID, provision)
+	if err != nil {
+		return err
+	}
+	if tag.RowsAffected() == 0 {
+		return fmt.Errorf("%s was not told where to get %q from", nodeName, provision)
+	}
+	return nil
+}
+
+// PinsFor is what a node was told about where its provisions come from.
+func (i *Inventory) PinsFor(ctx context.Context, nodeName string) (map[string]string, error) {
+	node, err := i.NodeByName(ctx, nodeName)
+	if err != nil {
+		return nil, err
+	}
+	rows, err := i.store.Pool().Query(ctx,
+		`select p.name, n.name from provision_pin p join node n on n.id = p.provider
+		 where p.node = $1`, node.ID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	out := map[string]string{}
+	for rows.Next() {
+		var name, provider string
+		if err := rows.Scan(&name, &provider); err != nil {
+			return nil, err
+		}
+		out[name] = provider
+	}
+	return out, rows.Err()
+}
+
+// pinRows is how many pins a node holds, counted in the table rather than through the join that
+// reads them.
+//
+// For a test that would otherwise pass for the wrong reason: PinsFor joins on the provider, so a
+// pin left behind by a departed node is invisible through it whether it was cleaned up or not.
+func (i *Inventory) pinRows(ctx context.Context, nodeName string) (int, error) {
+	node, err := i.NodeByName(ctx, nodeName)
+	if err != nil {
+		return 0, err
+	}
+	var n int
+	err = i.store.Pool().QueryRow(ctx,
+		`select count(*) from provision_pin where node = $1`, node.ID).Scan(&n)
+	return n, err
+}

internal/inventory/catalogue_test.go

@@ -1,6 +1,7 @@
 package inventory
 
 import (
+	"context"
 	"errors"
 	"testing"
 
@@ -8,7 +9,7 @@ import (
 )
 
 func manifest(name string, provides, requires []string) catalogue.Manifest {
-	return catalogue.Manifest{Module: name, Provides: provides, Requires: requires}
+	return catalogue.Manifest{Module: name, Provides: catalogue.Offers(provides...), Requires: requires}
 }
 
 func TestAModuleRoundTripsWholeAndUnshredded(t *testing.T) {
@@ -17,7 +18,7 @@ func TestAModuleRoundTripsWholeAndUnshredded(t *testing.T) {
 	// stored — the module system is the thing most likely to grow.
 	inv := fresh(t)
 	m := catalogue.Manifest{
-		Module: "xorg", Provides: []string{"display-server"},
+		Module: "xorg", Provides: catalogue.Offers("display-server"),
 		Capabilities: []string{"seat"},
 		Claims:       []catalogue.Claim{{Name: "the-seat", Scope: catalogue.ScopeNode}},
 		Resources:    []map[string]any{{"id": "conf", "type": "file", "path": "/etc/X11/x.conf"}},
@@ -364,3 +365,66 @@ func TestASourceNobodyHasCheckedIsNotBehind(t *testing.T) {
 		t.Error("a module with no known head reports as behind")
 	}
 }
+
+func TestAPinSurvivesAndCanBeChanged(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	for _, n := range []string{"user", "first", "second"} {
+		if _, err := inv.AddNode(ctx, n); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.PinProvision(ctx, "user", "database", "first"); err != nil {
+		t.Fatal(err)
+	}
+	// Changing the answer replaces it rather than adding a second, or a machine would be told to
+	// use two databases and nothing would say which.
+	if err := inv.PinProvision(ctx, "user", "database", "second"); err != nil {
+		t.Fatal(err)
+	}
+	pins, err := inv.PinsFor(ctx, "user")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(pins) != 1 || pins["database"] != "second" {
+		t.Fatalf("got %v", pins)
+	}
+	if err := inv.UnpinProvision(ctx, "user", "database"); err != nil {
+		t.Fatal(err)
+	}
+	if pins, _ := inv.PinsFor(ctx, "user"); len(pins) != 0 {
+		t.Fatalf("the choice outlived being removed: %v", pins)
+	}
+	// Removing something that was never said is a mistake worth reporting, not a silent success.
+	if err := inv.UnpinProvision(ctx, "user", "database"); err == nil {
+		t.Fatal("unpinning something nobody pinned reported success")
+	}
+}
+
+func TestAPinGoesWhenTheProviderLeavesTheMesh(t *testing.T) {
+	// Otherwise a machine is pointed at something that no longer exists and reported as
+	// configured, which is the failure mode this whole project keeps refusing.
+	inv := fresh(t)
+	ctx := context.Background()
+	for _, n := range []string{"consumer", "provider"} {
+		if _, err := inv.AddNode(ctx, n); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.PinProvision(ctx, "consumer", "database", "provider"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'provider'`); err != nil {
+		t.Fatal(err)
+	}
+	// Counted in the table, not read through PinsFor. PinsFor joins on the provider, so a pin
+	// left behind by a departed node is invisible through it whether or not it was cleaned up —
+	// which made the first version of this test pass with the cascade removed.
+	rows, err := inv.pinRows(ctx, "consumer")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rows != 0 {
+		t.Fatalf("a choice outlived the machine it named: %d row(s) left", rows)
+	}
+}

internal/inventory/migrations/0008-which-provider.sql

@@ -0,0 +1,26 @@
+-- Which node a machine gets a provision from, when more than one could answer.
+--
+-- One database in a mesh needs no such record: there is one answer and the mesh takes it. Several
+-- is entirely ordinary, and then picking one is a choice with consequences -- somebody's data
+-- lands on the machine that was chosen -- so the mesh refuses to guess and this is where the
+-- answer is kept once a person gives it.
+--
+-- Per node rather than per mesh, because that is the granularity the choice actually has: two
+-- machines may reasonably use two different databases, and a mesh-wide answer could not say so.
+
+create table provision_pin (
+    node     uuid        not null references node(id) on delete cascade,
+    -- The provision as required -- `database`, not `postgres`. What is being chosen is which node
+    -- answers a requirement, and the module answering it may change without the choice changing.
+    name     text        not null,
+    -- The node it comes from. Not a module: the same module on two machines is two answers, and
+    -- which machine is the whole question.
+    provider uuid        not null references node(id) on delete cascade,
+    pinned_at timestamptz not null default now(),
+
+    primary key (node, name)
+);
+
+-- A pin naming a node that leaves the mesh goes with it. The alternative is a machine pointed at
+-- something that no longer exists, reported as configured.
+create index provision_pin_provider on provision_pin (provider);