A module may need a secret of its own, and the provisioner watches

Two things, both found by trying to write a real postgres module and
discovering it could not be said.

A database has a superuser password, a broker an administrator, a
registry an account. None of them is *for* anybody — they are not the
credential a consumer is given, and the mechanism that hands those out
has a consumer in the middle of it. So a module may declare what it needs
and where to put it, and the mesh generates one per node, seals it, and
reads it no more than it reads any other.

Per node, deliberately: a module running on three machines has three
passwords. One in the manifest instead would put the same secret on every
machine that ever runs it, in a file anybody can read, for ever. Made
once and kept, or a running database would be handed a password it was
not started with; remade when the machine's sealing key changes, like
everything else sealed here.

A need declared and not made is refused rather than skipped, because a
module whose own credential is silently absent starts, fails to
authenticate, and the reason is three layers from the machine reporting
it.

And the provisioner can watch. That is what lets it be a module rather
than a binary somebody places: run once, it needs invoking after every
declaration by a timer or a unit wired to a file; watching, it is an
ordinary long-running service the host already supervises. It polls
rather than watching the filesystem, because the host writes atomically —
the file is replaced, so a watch on the path stops seeing anything after
the first replacement, and a watcher that silently stops working is worse
than a poll. Credentials are compared by digest and never held: this runs
for as long as the machine is up.

cmd/mesh-control/main.go

@@ -1265,8 +1265,23 @@ func declarationWith(ctx context.Context, inv *inventory.Inventory, node string,
 	if err != nil {
 		return nil, err
 	}
-	return plan.Declaration(
+	// And each module's own secrets — a superuser password, an administrator, an account. Made
-		catalogue.Rendering{Settings: settings, Generators: gens, Grants: grants})
+	// per node, so a module running on three machines has three.
+	needed := map[string]map[string]string{}
+	for _, m := range plan.Modules {
+		for name := range m.Needs {
+			sealed, err := inv.SecretForModule(ctx, node, m.Module, name)
+			if err != nil {
+				return nil, err
+			}
+			if needed[m.Module] == nil {
+				needed[m.Module] = map[string]string{}
+			}
+			needed[m.Module][name] = sealed
+		}
+	}
+	return plan.Declaration(catalogue.Rendering{
+		Settings: settings, Generators: gens, Grants: grants, Needed: needed})
 }
 
 // grantsFor is every credential this node must create, because something elsewhere uses it.

examples/README.md

@@ -10,3 +10,16 @@ with the module that ships the software it configures.
 | | |
 |---|---|
 | `postgres-provisioner` | the last step of a credential: reads what the mesh delivered and makes PostgreSQL accept it |
+
+## Running the provisioner
+
+`--watch` reconciles now and again whenever what the mesh delivered changes. That is what lets it
+be a module: an ordinary long-running service the host supervises, rather than something that has
+to be invoked after every declaration by a timer or a unit wired to a file.
+
+It polls rather than watching the filesystem, because the host writes atomically — the file is
+replaced, so a watch on the path stops seeing anything after the first replacement. A watcher that
+silently stops working is worse than a poll.
+
+Credentials are compared by digest and never by content. This runs for as long as the machine is
+up, and a secret does not belong in a long-lived variable when a hash answers the same question.

examples/postgres-provisioner/Dockerfile

@@ -0,0 +1,18 @@
+# The provisioner, as a module ships one.
+#
+# Built here so a machine can be given it by the mesh rather than by somebody putting a binary on
+# it. Static and FROM scratch for the same reason the control plane's image is: it is fetched by
+# digest and run on a machine, and everything in it is something a person would have to audit.
+FROM golang:1.25-alpine AS build
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' \
+    -o /mesh-provision-postgres ./examples/postgres-provisioner
+
+FROM scratch
+COPY --from=build /mesh-provision-postgres /mesh-provision-postgres
+# Watching by default, because that is what makes it a module: an ordinary long-running service
+# the host supervises, rather than something invoked after every declaration.
+ENTRYPOINT ["/mesh-provision-postgres", "--watch"]

examples/postgres-provisioner/main.go

@@ -20,12 +20,17 @@ package main
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"sort"
 	"strings"
+	"syscall"
+	"time"
 
 	"github.com/jackc/pgx/v5"
 )
@@ -53,10 +58,97 @@ type manifest struct {
 }
 
 func main() {
-	if err := run(context.Background()); err != nil {
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	// Once, or whenever what it was given changes.
+	//
+	// **Watching is what lets this be a module.** Run once, it has to be invoked by something
+	// after every declaration — a timer that runs it when nothing changed, or a unit wired to
+	// restart on a file. Watching, it is an ordinary long-running service, which is a shape the
+	// mesh already delivers and the host already supervises.
+	//
+	// The file it watches is the manifest the mesh writes. A credential changing rewrites the
+	// file beside it and not the manifest, so the manifest is stamped whenever either is written
+	// — which is why this compares content rather than modification time.
+	if len(os.Args) > 1 && os.Args[1] == "--watch" {
+		if err := watch(ctx); err != nil {
 			fmt.Fprintf(os.Stderr, "mesh-provision-postgres: %v\n", err)
 			os.Exit(1)
 		}
+		return
+	}
+	if err := run(ctx); err != nil {
+		fmt.Fprintf(os.Stderr, "mesh-provision-postgres: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+// watch reconciles now, and again whenever what the mesh delivered changes.
+//
+// By polling rather than by watching the filesystem, because the file is replaced rather than
+// written in place — the host writes atomically, so an inotify watch on the path stops seeing
+// anything after the first replacement, which is a watcher that silently stops working.
+func watch(ctx context.Context) error {
+	const every = 10 * time.Second
+	var last string
+
+	for {
+		state, err := given()
+		switch {
+		case err != nil:
+			// Said and retried. A provisioner that exits because the mesh has not written
+			// anything yet is one that has to be restarted by hand after the first push.
+			fmt.Fprintf(os.Stderr, "cannot read what was granted: %v\n", err)
+		case state != last:
+			if err := run(ctx); err != nil {
+				// Reported and retried. The usual reason is that the database has not finished
+				// starting, and giving up would mean a module that works only if the two
+				// containers happen to come up in the right order.
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+			} else {
+				last = state
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-time.After(every):
+		}
+	}
+}
+
+// given is everything the mesh has delivered, as one string, so a change of any of it is one
+// comparison.
+//
+// The credentials are included by their **digest**, never their content: this is compared, logged
+// on nothing, and held in memory for as long as the process runs, and a secret does not belong in
+// any of that when a hash answers the same question.
+func given() (string, error) {
+	grants := os.Getenv("GRANTS")
+	if grants == "" {
+		grants = "/var/lib/postgres/grants"
+	}
+	entries, err := os.ReadDir(grants)
+	if err != nil {
+		return "", err
+	}
+	var names []string
+	for _, e := range entries {
+		names = append(names, e.Name())
+	}
+	sort.Strings(names)
+
+	sum := sha256.New()
+	for _, name := range names {
+		body, err := os.ReadFile(filepath.Join(grants, name))
+		if err != nil {
+			return "", err
+		}
+		fmt.Fprintf(sum, "%s:%x\n", name, sha256.Sum256(body))
+	}
+	return hex.EncodeToString(sum.Sum(nil)), nil
 }
 
 func run(ctx context.Context) error {

internal/catalogue/declaration.go

@@ -44,6 +44,10 @@ type Grant struct {
 
 // Rendering is everything needed to turn a resolution into the declaration a node is sent.
 type Rendering struct {
+	// Needed is each module's own secrets, sealed to this node, keyed by module and then by the
+	// name the module gave it.
+	Needed map[string]map[string]string
+
 	Settings   SettingsBy
 	Generators map[string]Generator
 	// Grants are the credentials this node must create, for the provisions it offers. Passed in
@@ -74,6 +78,19 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
 	var out []map[string]any
 	for _, m := range r.Modules {
 		resources := m.Resources
+		for _, name := range sortedKeys(m.Needs) {
+			sealed := with.Needed[m.Module][name]
+			if sealed == "" {
+				// Declared and not made. Refused rather than skipped: a module whose own
+				// credential is silently absent starts, fails to authenticate, and the reason is
+				// three layers away from the machine reporting it.
+				return nil, fmt.Errorf(
+					"%s needs a secret called %q and none was made for it", m.Module, name)
+			}
+			resources = append(append([]map[string]any{}, resources...), map[string]any{
+				"id": NeedID(name), "type": "file", "path": m.Needs[name], "sealed": sealed,
+			})
+		}
 		for _, to := range sortedKeys(m.Secrets) {
 			var found *Needed
 			for i, n := range r.Needs {

internal/catalogue/manifest.go

@@ -199,6 +199,19 @@ type Manifest struct {
 	// makes `restart-on` precise.
 	Secrets map[string]string `json:"secrets,omitempty"`
 
+	// Needs is a secret this module needs for itself, and where to put it.
+	//
+	// Not tied to a consumer. A database has a superuser password, a broker has an administrator,
+	// a registry has an account — each is a secret the module needs in order to be itself, and
+	// none of them is *for* anybody. Keyed by a name of the module's choosing, valued by the file
+	// it lands in.
+	//
+	// **Generated per node and sealed to it**, like everything else the mesh hands out, so a
+	// module running on three machines has three passwords and the mesh can read none of them. A
+	// manifest carrying one instead would put the same secret on every machine that ever runs the
+	// module, in a file anybody can read, for ever.
+	Needs map[string]string `json:"needs,omitempty"`
+
 	// Grants is a directory this module wants the credentials of its consumers written into, per
 	// provision it offers — one file per consumer, named for it, holding the value alone.
 	//
@@ -235,6 +248,9 @@ const (
 	ArtifactArchive = "archive"
 )
 
+// NeedID is the resource identity of the file a module's own secret lands in.
+func NeedID(name string) string { return "needs-" + name }
+
 // SecretID is the resource identity of the file a module is given a credential in.
 func SecretID(requirement string) string { return "secret-" + requirement }
 
@@ -374,6 +390,15 @@ func ParseManifest(raw []byte) (Manifest, error) {
 				"%s binds %q and does not require it", m.Module, to))
 		}
 	}
+	for name, where := range m.Needs {
+		if !strings.HasPrefix(where, "/") {
+			problems = append(problems, fmt.Sprintf(
+				"%s needs %q at %q, which is not an absolute path", m.Module, name, where))
+		}
+		if name == "" {
+			problems = append(problems, m.Module+" needs a secret with no name")
+		}
+	}
 	for to, where := range m.Secrets {
 		if !strings.HasPrefix(where, "/") {
 			problems = append(problems, fmt.Sprintf(

internal/catalogue/needs_test.go

@@ -0,0 +1,94 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// A secret a module needs in order to be itself.
+//
+// A database has a superuser password, a broker an administrator, a registry an account. None is
+// *for* anybody — it is not the credential a consumer is given, and the mechanism that hands
+// those out has a consumer in the middle of it.
+
+func needy() Manifest {
+	return Manifest{
+		Module: "postgres", Version: "1",
+		Needs: map[string]string{"superuser": "/var/lib/mesh/postgres/superuser"},
+		Resources: []map[string]any{
+			{"id": "store", "type": "container", "name": "mesh-postgres", "image": "postgres@sha256:x"},
+		},
+	}
+}
+
+func TestAModulesOwnSecretLandsSealed(t *testing.T) {
+	got, err := Resolve(shelf(needy()), []string{"postgres"}, reachable(), World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := got.Declaration(Rendering{
+		Needed: map[string]map[string]string{"postgres": {"superuser": "c2VhbGVk"}},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range out {
+		if r["path"] != "/var/lib/mesh/postgres/superuser" {
+			continue
+		}
+		if r["sealed"] != "c2VhbGVk" {
+			t.Fatalf("got %v", r)
+		}
+		if r["content"] != nil {
+			t.Fatal("a module's own secret was written in the clear")
+		}
+		return
+	}
+	t.Fatalf("no secret was written: %v", out)
+}
+
+func TestADeclaredNeedThatWasNotMadeIsRefused(t *testing.T) {
+	// Skipping it would start a database with no password it knows, which fails to authenticate
+	// three layers from the machine reporting it.
+	got, _ := Resolve(shelf(needy()), []string{"postgres"}, reachable(), World{})
+	_, err := got.Declaration(Rendering{})
+	if err == nil {
+		t.Fatal("a module needing a secret was declared without one")
+	}
+	if !strings.Contains(err.Error(), "superuser") {
+		t.Fatalf("the refusal does not name what is missing: %v", err)
+	}
+}
+
+func TestANeedIsAnAbsolutePath(t *testing.T) {
+	_, err := ParseManifest([]byte(`{"module":"postgres","version":"1",
+		"needs":{"superuser":"superuser.txt"}}`))
+	if err == nil {
+		t.Fatal("a relative path was accepted")
+	}
+	if !strings.Contains(err.Error(), "absolute path") {
+		t.Fatalf("unhelpful refusal: %v", err)
+	}
+}
+
+func TestAModuleMayNeedSeveralThings(t *testing.T) {
+	// A password and a token, say. Telling them apart is the module's business, not the mesh's.
+	m := needy()
+	m.Needs["replication"] = "/var/lib/mesh/postgres/replication"
+	got, _ := Resolve(shelf(m), []string{"postgres"}, reachable(), World{})
+	out, err := got.Declaration(Rendering{Needed: map[string]map[string]string{
+		"postgres": {"superuser": "b25l", "replication": "dHdv"},
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	seen := map[string]string{}
+	for _, r := range out {
+		if path, ok := r["path"].(string); ok && strings.Contains(path, "/var/lib/mesh/postgres/") {
+			seen[path], _ = r["sealed"].(string)
+		}
+	}
+	if len(seen) != 2 || seen["/var/lib/mesh/postgres/superuser"] == seen["/var/lib/mesh/postgres/replication"] {
+		t.Fatalf("two needs did not land as two secrets: %v", seen)
+	}
+}

internal/inventory/migrations/0012-a-modules-own-secret.sql

@@ -0,0 +1,29 @@
+-- A secret a module needs in order to be itself.
+--
+-- A database has a superuser password, a broker an administrator, a registry an account. None of
+-- them is *for* anybody -- they are not the credential a consumer is given, and the table holding
+-- those has a consumer in its key.
+--
+-- **One per node**, so a module running on three machines has three passwords. A manifest that
+-- carried one instead would put the same secret on every machine that ever runs the module, in a
+-- file anybody can read, for ever.
+--
+-- Sealed to the node before it is written, like everything else here: what is stored is unusable
+-- by whoever holds it, the mesh included.
+
+create table module_secret (
+    node     uuid        not null references node(id) on delete cascade,
+    module   text        not null references module(name) on delete cascade,
+    -- The module's own word for it. Two secrets in one module are ordinary -- a password and a
+    -- token, say -- and telling them apart is the module's business, not the mesh's.
+    name     text        not null,
+
+    sealed   text        not null,
+    -- Which key it was sealed to, so a node that regenerated its key can be told what it can no
+    -- longer open rather than discovering it as a service that will not start.
+    node_key text        not null,
+
+    made_at  timestamptz not null default now(),
+
+    primary key (node, module, name)
+);

internal/inventory/secrets.go

@@ -2,6 +2,7 @@ package inventory
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/novox/mesh-control/internal/secrets"
 )
@@ -133,3 +134,49 @@ func (i *Inventory) SecretsFrom(ctx context.Context, provider string) ([]Secret,
 	}
 	return out, rows.Err()
 }
+
+// SecretForModule is a secret a module needs in order to be itself, on one machine.
+//
+// Not the credential a consumer is given: a superuser password is not *for* anybody. Made once
+// and kept, because regenerating it on every declaration would change the password a running
+// database has already been started with — and remade when the node's sealing key changes, for
+// the same reason as everything else sealed here.
+func (i *Inventory) SecretForModule(ctx context.Context, node, module, name string) (string, error) {
+	key, err := i.SealingKeyOf(ctx, node)
+	if err != nil {
+		return "", err
+	}
+	if key == "" {
+		return "", fmt.Errorf(
+			"%s needs a secret and %s has no sealing key, so nothing can be sealed to it",
+			module, node)
+	}
+	record, err := i.NodeByName(ctx, node)
+	if err != nil {
+		return "", err
+	}
+
+	var sealed, against string
+	err = i.store.Pool().QueryRow(ctx,
+		`select sealed, node_key from module_secret where node = $1 and module = $2 and name = $3`,
+		record.ID, module, name).Scan(&sealed, &against)
+	if err == nil && against == key {
+		return sealed, nil
+	}
+
+	made, err := secrets.Make(key, key)
+	if err != nil {
+		return "", err
+	}
+	// Sealed once, to one recipient. Make seals to two ends because a provision has two; here
+	// both are the same machine, and only one copy is kept.
+	if _, err := i.store.Pool().Exec(ctx,
+		`insert into module_secret (node, module, name, sealed, node_key)
+		 values ($1, $2, $3, $4, $5)
+		 on conflict (node, module, name) do update set
+		   sealed = excluded.sealed, node_key = excluded.node_key, made_at = now()`,
+		record.ID, module, name, made.ForConsumer, key); err != nil {
+		return "", err
+	}
+	return made.ForConsumer, nil
+}

internal/inventory/secrets_test.go

@@ -5,6 +5,7 @@ import (
 	"crypto/ecdh"
 	"crypto/rand"
 	"encoding/base64"
+	"github.com/novox/mesh-control/internal/catalogue"
 	"strings"
 	"testing"
 
@@ -230,3 +231,97 @@ func TestSecretsGoWhenANodeLeaves(t *testing.T) {
 		t.Fatalf("%d credential(s) outlived the machine they were for", left)
 	}
 }
+
+func TestAModulesOwnSecretIsPerMachineAndKept(t *testing.T) {
+	// A module running on three machines has three passwords. One in the manifest instead would
+	// put the same secret on every machine that ever runs it, in a file anybody can read.
+	inv, ctx := twoNodesWithKeys(t)
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
+		Source{}); err != nil {
+		t.Fatal(err)
+	}
+
+	here, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
+	if err != nil {
+		t.Fatal(err)
+	}
+	there, err := inv.SecretForModule(ctx, "provider", "postgres", "superuser")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if here == there {
+		t.Fatal("two machines were given the same secret")
+	}
+
+	// Made once and kept, or a running database would be handed a password it was not started
+	// with on the next declaration.
+	again, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if again != here {
+		t.Fatal("asking twice made a second secret")
+	}
+}
+
+func TestTwoNeedsInOneModuleAreTwoSecrets(t *testing.T) {
+	inv, ctx := twoNodesWithKeys(t)
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
+		Source{}); err != nil {
+		t.Fatal(err)
+	}
+	one, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
+	if err != nil {
+		t.Fatal(err)
+	}
+	two, err := inv.SecretForModule(ctx, "consumer", "postgres", "replication")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if one == two {
+		t.Fatal("two names gave one secret")
+	}
+}
+
+func TestAModulesSecretIsRemadeWhenTheMachineRejoins(t *testing.T) {
+	// The node generated a new sealing key and can no longer open what was sealed to the old one.
+	inv, ctx := twoNodesWithKeys(t)
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
+		Source{}); err != nil {
+		t.Fatal(err)
+	}
+	before, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
+	if err != nil {
+		t.Fatal(err)
+	}
+	node, err := inv.NodeByName(ctx, "consumer")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fresh, _ := aSealingKey(t)
+	if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil {
+		t.Fatal(err)
+	}
+	after, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if after == before {
+		t.Fatal("a machine was handed a secret sealed to a key it no longer has")
+	}
+}
+
+func TestAMachineWithNoSealingKeyCannotBeGivenAModuleSecret(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	if _, err := inv.AddNode(ctx, "bare"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
+		Source{}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.SecretForModule(ctx, "bare", "postgres", "superuser"); err == nil {
+		t.Fatal("a secret was made for a machine that cannot open one")
+	}
+}