Where the answer to a requirement is allowed to live

Two different things were both written `requires`. A shell, a display
server and a private network have to be on the machine that needs them.
A database does not — it runs somewhere and is reached over the network.
Both were answered the same way, so requiring a database installed
PostgreSQL on every machine that ran a web application.

What a module provides now carries a scope, the same idea claims already
use, written short in the ordinary case:

  "provides": ["shell"]
  "provides": [{"name": "database", "scope": "mesh"}]

A mesh-scoped requirement is answered by finding the node already running
it — never by installing it here. Choosing a machine to put a database on
is a decision with consequences, and nothing resolving a web application
should make it silently. With nothing anywhere it refuses and says which
module to assign; with two it refuses and says how to choose.

Choosing is `pin <node> <provision> <from>`, kept per node because that
is the granularity the choice has. A pin at a machine that does not
provide it refuses rather than falling back — a fallback would quietly
move somebody's data. One provider does not overrule a pin either.

Resolving a node now needs to know what the others offer, and working
that out needs them resolved, so it is two passes: the first answers only
what each node offers, the second answers everything. Nothing is ever
declared from the first.

A node's plan says what it takes from elsewhere. It is the only part of a
set that stops working when a different machine goes away, and nothing
else in that output would have said so. It is also where a credential
will hang once there is a mechanism for handing one back.

One test found passing for the wrong reason: it read pins through a join
on the provider, which hides a dangling row whether or not it was cleaned
up. It counts rows now, and bites when the cascade is removed.

internal/inventory/catalogue.go

@@ -436,3 +436,82 @@ func (i *Inventory) SettingsFor(ctx context.Context, nodeName, module string) ([
 	}
 	return layers, rows.Err()
 }
+
+// PinProvision records which node a machine gets a provision from.
+//
+// Only needed when more than one node could answer. Recordable before that, because a mesh with
+// one database should not change where an existing machine gets its data the day a second
+// arrives.
+func (i *Inventory) PinProvision(ctx context.Context, nodeName, provision, provider string) error {
+	node, err := i.NodeByName(ctx, nodeName)
+	if err != nil {
+		return err
+	}
+	from, err := i.NodeByName(ctx, provider)
+	if err != nil {
+		return err
+	}
+	_, err = i.store.Pool().Exec(ctx,
+		`insert into provision_pin (node, name, provider) values ($1, $2, $3)
+		 on conflict (node, name) do update set provider = excluded.provider, pinned_at = now()`,
+		node.ID, provision, from.ID)
+	return err
+}
+
+// UnpinProvision removes a choice, putting the question back.
+func (i *Inventory) UnpinProvision(ctx context.Context, nodeName, provision string) error {
+	node, err := i.NodeByName(ctx, nodeName)
+	if err != nil {
+		return err
+	}
+	tag, err := i.store.Pool().Exec(ctx,
+		`delete from provision_pin where node = $1 and name = $2`, node.ID, provision)
+	if err != nil {
+		return err
+	}
+	if tag.RowsAffected() == 0 {
+		return fmt.Errorf("%s was not told where to get %q from", nodeName, provision)
+	}
+	return nil
+}
+
+// PinsFor is what a node was told about where its provisions come from.
+func (i *Inventory) PinsFor(ctx context.Context, nodeName string) (map[string]string, error) {
+	node, err := i.NodeByName(ctx, nodeName)
+	if err != nil {
+		return nil, err
+	}
+	rows, err := i.store.Pool().Query(ctx,
+		`select p.name, n.name from provision_pin p join node n on n.id = p.provider
+		 where p.node = $1`, node.ID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	out := map[string]string{}
+	for rows.Next() {
+		var name, provider string
+		if err := rows.Scan(&name, &provider); err != nil {
+			return nil, err
+		}
+		out[name] = provider
+	}
+	return out, rows.Err()
+}
+
+// pinRows is how many pins a node holds, counted in the table rather than through the join that
+// reads them.
+//
+// For a test that would otherwise pass for the wrong reason: PinsFor joins on the provider, so a
+// pin left behind by a departed node is invisible through it whether it was cleaned up or not.
+func (i *Inventory) pinRows(ctx context.Context, nodeName string) (int, error) {
+	node, err := i.NodeByName(ctx, nodeName)
+	if err != nil {
+		return 0, err
+	}
+	var n int
+	err = i.store.Pool().QueryRow(ctx,
+		`select count(*) from provision_pin where node = $1`, node.ID).Scan(&n)
+	return n, err
+}

internal/inventory/catalogue_test.go

@@ -1,6 +1,7 @@
 package inventory
 
 import (
+	"context"
 	"errors"
 	"testing"
 
@@ -8,7 +9,7 @@ import (
 )
 
 func manifest(name string, provides, requires []string) catalogue.Manifest {
-	return catalogue.Manifest{Module: name, Provides: provides, Requires: requires}
+	return catalogue.Manifest{Module: name, Provides: catalogue.Offers(provides...), Requires: requires}
 }
 
 func TestAModuleRoundTripsWholeAndUnshredded(t *testing.T) {
@@ -17,7 +18,7 @@ func TestAModuleRoundTripsWholeAndUnshredded(t *testing.T) {
 	// stored — the module system is the thing most likely to grow.
 	inv := fresh(t)
 	m := catalogue.Manifest{
-		Module: "xorg", Provides: []string{"display-server"},
+		Module: "xorg", Provides: catalogue.Offers("display-server"),
 		Capabilities: []string{"seat"},
 		Claims:       []catalogue.Claim{{Name: "the-seat", Scope: catalogue.ScopeNode}},
 		Resources:    []map[string]any{{"id": "conf", "type": "file", "path": "/etc/X11/x.conf"}},
@@ -364,3 +365,66 @@ func TestASourceNobodyHasCheckedIsNotBehind(t *testing.T) {
 		t.Error("a module with no known head reports as behind")
 	}
 }
+
+func TestAPinSurvivesAndCanBeChanged(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	for _, n := range []string{"user", "first", "second"} {
+		if _, err := inv.AddNode(ctx, n); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.PinProvision(ctx, "user", "database", "first"); err != nil {
+		t.Fatal(err)
+	}
+	// Changing the answer replaces it rather than adding a second, or a machine would be told to
+	// use two databases and nothing would say which.
+	if err := inv.PinProvision(ctx, "user", "database", "second"); err != nil {
+		t.Fatal(err)
+	}
+	pins, err := inv.PinsFor(ctx, "user")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(pins) != 1 || pins["database"] != "second" {
+		t.Fatalf("got %v", pins)
+	}
+	if err := inv.UnpinProvision(ctx, "user", "database"); err != nil {
+		t.Fatal(err)
+	}
+	if pins, _ := inv.PinsFor(ctx, "user"); len(pins) != 0 {
+		t.Fatalf("the choice outlived being removed: %v", pins)
+	}
+	// Removing something that was never said is a mistake worth reporting, not a silent success.
+	if err := inv.UnpinProvision(ctx, "user", "database"); err == nil {
+		t.Fatal("unpinning something nobody pinned reported success")
+	}
+}
+
+func TestAPinGoesWhenTheProviderLeavesTheMesh(t *testing.T) {
+	// Otherwise a machine is pointed at something that no longer exists and reported as
+	// configured, which is the failure mode this whole project keeps refusing.
+	inv := fresh(t)
+	ctx := context.Background()
+	for _, n := range []string{"consumer", "provider"} {
+		if _, err := inv.AddNode(ctx, n); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.PinProvision(ctx, "consumer", "database", "provider"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'provider'`); err != nil {
+		t.Fatal(err)
+	}
+	// Counted in the table, not read through PinsFor. PinsFor joins on the provider, so a pin
+	// left behind by a departed node is invisible through it whether or not it was cleaned up —
+	// which made the first version of this test pass with the cascade removed.
+	rows, err := inv.pinRows(ctx, "consumer")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rows != 0 {
+		t.Fatalf("a choice outlived the machine it named: %d row(s) left", rows)
+	}
+}

internal/inventory/migrations/0008-which-provider.sql

@@ -0,0 +1,26 @@
+-- Which node a machine gets a provision from, when more than one could answer.
+--
+-- One database in a mesh needs no such record: there is one answer and the mesh takes it. Several
+-- is entirely ordinary, and then picking one is a choice with consequences -- somebody's data
+-- lands on the machine that was chosen -- so the mesh refuses to guess and this is where the
+-- answer is kept once a person gives it.
+--
+-- Per node rather than per mesh, because that is the granularity the choice actually has: two
+-- machines may reasonably use two different databases, and a mesh-wide answer could not say so.
+
+create table provision_pin (
+    node     uuid        not null references node(id) on delete cascade,
+    -- The provision as required -- `database`, not `postgres`. What is being chosen is which node
+    -- answers a requirement, and the module answering it may change without the choice changing.
+    name     text        not null,
+    -- The node it comes from. Not a module: the same module on two machines is two answers, and
+    -- which machine is the whole question.
+    provider uuid        not null references node(id) on delete cascade,
+    pinned_at timestamptz not null default now(),
+
+    primary key (node, name)
+);
+
+-- A pin naming a node that leaves the mesh goes with it. The alternative is a machine pointed at
+-- something that no longer exists, reported as configured.
+create index provision_pin_provider on provision_pin (provider);