Rotate a credential and move both ends together

The invariant novox/hq ADR 0001 records as unowned, and it was measurably
false in HAL: a provision documented as never rotating minted a new password on
every adoption and updated only the provider's row. Consumers on three nodes
held dead credentials for two days while the mesh reported success. Nothing
enumerated who held the old one.

Three things make that impossible here. The holders are a set the mesh can name
— each pair has its own credential, so rotating one consumer touches one role
and the affected list is a query rather than an assumption. Both ends are
pushed by this command rather than a later one, because leaving the sending to
whoever remembered is the fault exactly. And it is all-or-nothing: if any
affected machine cannot be resolved, nothing is sent and the old credential
keeps working, which is a mesh that has not rotated rather than one that has
half-rotated.

The window is stated rather than hidden: a role's password changes on the
provider and the file changes on the consumer, and they cannot be simultaneous.

The provisioner now takes its superuser password from the file the mesh wrote,
which is how the mesh delivers one. Passing it through the environment needed a
person in the middle of the one path that exists so there is not one — and put
a superuser password where `docker inspect` prints it.

Makefile

@@ -42,6 +42,17 @@ builder-image:
 	@echo
 	@docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}}  {{.Size}} bytes'
 
+# The provisioner ships as an image too, because it is the thing that makes a sealed credential
+# true on a machine -- and the mesh cannot, having discarded the plaintext.
+PROVISIONER_IMAGE ?= mesh-provision-postgres:$(VERSION)
+PROVISIONER_DEV_TAG ?= mesh-provision-postgres:development
+
+provisioner-image:
+	docker build -f examples/postgres-provisioner/Dockerfile \
+		-t $(PROVISIONER_IMAGE) -t $(PROVISIONER_DEV_TAG) .
+	@echo
+	@docker image inspect $(PROVISIONER_IMAGE) --format 'built {{.RepoTags}}  {{.Size}} bytes'
+
 # The whole gate. Raises a database, runs everything against it, and takes it down again --
 # including when the tests fail, which is why the teardown is not conditional.
 check: fmt vet postgres

cmd/mesh-control/main.go

@@ -69,6 +69,8 @@ func run() error {
 		return buildCommand(ctx, args[1:])
 	case "builder":
 		return builderCommand(ctx, args[1:])
+	case "rotate":
+		return rotateCommand(ctx, args[1:])
 	case "builds":
 		return buildsCommand(ctx, args[1:])
 	case "pin":
@@ -142,6 +144,7 @@ func usage() {
   build <repository> [--ref R]   have a build machine build it, and record what came out
   builds [<module>]              what has been built lately, and what came of it
   builder issue <name>           a broker account for a build machine, scoped to build work
+  rotate <provision> [--consumer <n>]  a new credential for every holder, both ends at once
   pin <node> <provision> <from>  which node this one gets a provision from
   unpin <node> <provision>       put that question back
   plan <node> [--files|--json]     what that node would run, and why
@@ -1662,6 +1665,66 @@ func pushCommand(ctx context.Context, args []string) error {
 	return nil
 }
 
+// sendTo resolves and sends to exactly the machines named, or refuses without sending anything.
+//
+// The same all-or-nothing rule push follows, and for the same reason: a rotation that reached the
+// consumer and refused on the provider would leave one end holding a credential the other has
+// never heard of — which is the state this whole mechanism exists to make impossible.
+func sendTo(ctx context.Context, inv *inventory.Inventory, names []string) error {
+	ident, err := openIdentity(ctx)
+	if err != nil {
+		return err
+	}
+	defer ident.Close()
+
+	gens, err := generators(ctx, inv)
+	if err != nil {
+		return err
+	}
+
+	type ready struct {
+		node      string
+		resources []map[string]any
+	}
+	var sending []ready
+	var refusals []string
+	for _, name := range names {
+		plan, settings, err := planFor(ctx, inv, name)
+		if err != nil {
+			refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
+			continue
+		}
+		resources, err := declarationWith(ctx, inv, name, plan, settings, gens)
+		if err != nil {
+			refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
+			continue
+		}
+		sending = append(sending, ready{name, resources})
+	}
+	if len(refusals) > 0 {
+		return fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s",
+			len(refusals), strings.Join(refusals, "\n\n"))
+	}
+
+	server, err := link.Connect(nil, nil)
+	if err != nil {
+		return err
+	}
+	defer server.Close()
+
+	for _, s := range sending {
+		body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources})
+		if err != nil {
+			return err
+		}
+		if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
+			return err
+		}
+		fmt.Printf("  sent %s %d resource(s)\n", s.node, len(s.resources))
+	}
+	return nil
+}
+
 // short is a commit as a person refers to it.
 func short(commit string) string {
 	if len(commit) > 8 {

cmd/mesh-control/rotate.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"sort"
+)
+
+// rotateCommand replaces a credential and moves both ends together.
+//
+// **This is the invariant novox/hq ADR 0001 records as unowned, and it was measurably false.** On
+// 2026-08-22 `provision_ensure` — documented as never rotating an existing secret — minted a new
+// password on every adoption and updated only the provider's row. Consumers on three nodes held
+// dead credentials for two days; two rows for one provision were written 216 ms apart, so at most
+// one could match the live role. Nothing enumerated who held the old one, and nothing said so.
+//
+// Three things make that impossible here, and all three are deliberate:
+//
+// **The holders are a set the mesh can name.** Each pair has its own credential, so rotating one
+// consumer's password touches one role and leaves every other consumer alone — and the list of who
+// is affected is a query rather than an assumption.
+//
+// **Both ends are pushed by this command, not by a later one.** A rotation that changed the record
+// and left the sending to whoever remembered is the fault above, exactly.
+//
+// **It is all-or-nothing.** If any affected machine cannot be resolved, nothing is sent and the old
+// credential keeps working — which is a mesh that has not rotated, and is far better than one that
+// has half-rotated.
+func rotateCommand(ctx context.Context, args []string) error {
+	set := flag.NewFlagSet("rotate", flag.ContinueOnError)
+	// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
+	// and rotating the other nine would be a great deal of disruption for one suspicion.
+	only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
+	positionals, err := parseAround(set, args)
+	if err != nil {
+		return err
+	}
+	if len(positionals) != 1 {
+		return errors.New("rotate <provision> [--consumer <machine>]")
+	}
+	provision := positionals[0]
+
+	inv, err := openInventory(ctx)
+	if err != nil {
+		return err
+	}
+	defer inv.Close()
+
+	holders, err := inv.HoldersOf(ctx, provision, *only)
+	if err != nil {
+		return err
+	}
+	if len(holders) == 0 {
+		// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
+		// and a rotation somebody believes happened is worse than one they know did not.
+		if *only != "" {
+			return fmt.Errorf(
+				"%s holds no credential for %q, so there is nothing to rotate. `plan %s` says "+
+					"what it does hold", *only, provision, *only)
+		}
+		return fmt.Errorf(
+			"nothing in this mesh holds a credential for %q, so there is nothing to rotate",
+			provision)
+	}
+
+	// Every machine at both ends, named before anything changes. A person about to rotate a
+	// production credential is entitled to know the blast radius before it is the past tense.
+	affected := map[string]bool{}
+	for _, h := range holders {
+		affected[h.Consumer] = true
+		affected[h.Provider] = true
+	}
+	machines := make([]string, 0, len(affected))
+	for name := range affected {
+		machines = append(machines, name)
+	}
+	sort.Strings(machines)
+
+	fmt.Printf("rotating %s for %d holder(s):\n", provision, len(holders))
+	for _, h := range holders {
+		fmt.Printf("  %s from %s\n", h.Consumer, h.Provider)
+	}
+
+	for _, h := range holders {
+		if err := inv.RotateSecret(ctx, h.Provision, h.Consumer, h.Provider); err != nil {
+			// Partly rotated, and said so plainly. What is gone is remade on the next push, so
+			// the remedy is to run this again rather than to repair anything — but a machine
+			// whose secret was discarded and not resent is holding a credential the provider is
+			// about to stop honouring, and that is worth knowing now.
+			return fmt.Errorf(
+				"rotating %s for %s from %s: %w\n\nSome credentials were discarded and not yet "+
+					"sent. Run this again once the cause is fixed",
+				h.Provision, h.Consumer, h.Provider, err)
+		}
+	}
+
+	// **Both ends, in one send.** There is a window either way — a role's password changes on the
+	// provider and the file changes on the consumer, and they cannot be simultaneous — so the
+	// honest thing is to make it as short as the broker allows and to never leave it open across
+	// a command boundary, where it depends on somebody's memory.
+	fmt.Printf("\nsending to both ends:\n")
+	if err := sendTo(ctx, inv, machines); err != nil {
+		return fmt.Errorf(
+			"%w\n\nThe old credentials are gone from the mesh and the new ones have not arrived. "+
+				"Nothing on those machines has changed yet, so what is running keeps working "+
+				"until the provider next applies. Fix the cause and run `push --behind`", err)
+	}
+
+	fmt.Printf("\n%d machine(s) told. Until both ends have applied, a consumer whose password "+
+		"changed cannot authenticate — `status` says who is still behind\n", len(machines))
+	return nil
+}

examples/postgres-provisioner/main.go

@@ -24,6 +24,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"net/url"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -171,7 +172,11 @@ func run(ctx context.Context) error {
 		return fmt.Errorf("the manifest at %s is not readable: %w", grants, err)
 	}
 
-	db, err := pgx.Connect(ctx, os.Getenv("MESH_PROVISION_POSTGRES"))
+	where, err := connectionString()
+	if err != nil {
+		return err
+	}
+	db, err := pgx.Connect(ctx, where)
 	if err != nil {
 		return err
 	}
@@ -306,3 +311,51 @@ func sorted(given []contribution) []contribution {
 // to be safe today" is not a property anything should rest on.
 func quoteName(s string) string   { return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` }
 func quoteString(s string) string { return `'` + strings.ReplaceAll(s, `'`, `''`) + `'` }
+
+// connectionString is where this provisioner reaches the database it owns.
+//
+// **The password comes from a file**, because that is how the mesh delivers one. A module's own
+// secret — a superuser password here — is sealed to the machine and written by the host; a
+// provisioner told to take it from an environment variable would need somebody to read the file
+// and pass it in, which is a person in the middle of the one path that exists so there is not
+// one.
+//
+// It is also the difference between a credential that lives in a file and one that lives in a
+// process listing: `docker inspect` prints environment, and a superuser password printed by an
+// ordinary diagnostic is a superuser password in whatever collected that diagnostic.
+//
+// MESH_PROVISION_POSTGRES alone still works, for a provisioner somebody runs by hand.
+func connectionString() (string, error) {
+	where := strings.TrimSpace(os.Getenv("MESH_PROVISION_POSTGRES"))
+	if where == "" {
+		return "", fmt.Errorf(
+			"MESH_PROVISION_POSTGRES is not set, so this provisioner does not know which " +
+				"database it owns")
+	}
+	path := strings.TrimSpace(os.Getenv("MESH_PROVISION_PASSWORD_FILE"))
+	if path == "" {
+		return where, nil
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return "", fmt.Errorf(
+			"cannot read the password this provisioner was given at %s: %w", path, err)
+	}
+	password := strings.TrimSpace(string(raw))
+	if password == "" {
+		// An empty file connects as nobody and is refused by the database, three layers from
+		// here, as an authentication problem with no cause anybody changed.
+		return "", fmt.Errorf("%s is empty, so this provisioner has no password", path)
+	}
+
+	parsed, err := url.Parse(where)
+	if err != nil {
+		return "", fmt.Errorf("MESH_PROVISION_POSTGRES is not a URL: %w", err)
+	}
+	user := parsed.User.Username()
+	if user == "" {
+		user = "postgres"
+	}
+	parsed.User = url.UserPassword(user, password)
+	return parsed.String(), nil
+}

examples/postgres-provisioner/where_test.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// The password comes from a file, because that is how the mesh delivers one.
+//
+// A provisioner told to take a superuser password from an environment variable needs somebody to
+// read the sealed file and pass it in — a person in the middle of the one path that exists so
+// there is not one. It is also the difference between a credential in a file and one in a process
+// listing: `docker inspect` prints environment.
+func TestTheSuperuserPasswordComesFromTheFileTheMeshWrote(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "superuser")
+	if err := os.WriteFile(path, []byte("the-sealed-one\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("MESH_PROVISION_POSTGRES", "postgres://postgres@127.0.0.1:5433/postgres?sslmode=disable")
+	t.Setenv("MESH_PROVISION_PASSWORD_FILE", path)
+
+	where, err := connectionString()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(where, "the-sealed-one") {
+		t.Fatalf("the password the mesh wrote is not in the connection: %s", where)
+	}
+	if !strings.Contains(where, "127.0.0.1:5433") || !strings.Contains(where, "sslmode=disable") {
+		t.Fatalf("the rest of the connection was lost: %s", where)
+	}
+}
+
+// An empty file connects as nobody and is refused by the database three layers away, as an
+// authentication problem with no cause anybody changed.
+func TestAnEmptyPasswordFileIsRefusedHere(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "superuser")
+	if err := os.WriteFile(path, []byte("\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("MESH_PROVISION_POSTGRES", "postgres://postgres@127.0.0.1:5433/postgres")
+	t.Setenv("MESH_PROVISION_PASSWORD_FILE", path)
+	if _, err := connectionString(); err == nil {
+		t.Fatal("a provisioner with no password reported one")
+	}
+}
+
+// And a provisioner somebody runs by hand still works with the URL alone.
+func TestAConnectionWithNoPasswordFileIsLeftAlone(t *testing.T) {
+	t.Setenv("MESH_PROVISION_POSTGRES", "postgres://postgres:typed@127.0.0.1:5433/postgres")
+	t.Setenv("MESH_PROVISION_PASSWORD_FILE", "")
+	where, err := connectionString()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if where != "postgres://postgres:typed@127.0.0.1:5433/postgres" {
+		t.Fatalf("the connection was rewritten when it should have been left alone: %s", where)
+	}
+}
+
+func TestAProvisionerWithNoDatabaseSaysSo(t *testing.T) {
+	t.Setenv("MESH_PROVISION_POSTGRES", "")
+	t.Setenv("MESH_PROVISION_PASSWORD_FILE", "")
+	if _, err := connectionString(); err == nil {
+		t.Fatal("a provisioner that does not know which database it owns reported one")
+	}
+}

internal/inventory/secrets.go

@@ -230,3 +230,42 @@ func (i *Inventory) AcceptSecretForModule(ctx context.Context, node, module, nam
 		record.ID, module, name, sealed.ForConsumer, key)
 	return err
 }
+
+// Holder is one end-to-end credential: who gets it and who must create it.
+type Holder struct {
+	Provision string
+	Consumer  string
+	Provider  string
+}
+
+// HoldersOf is every pair sharing a credential for one provision.
+//
+// **The question rotation has to ask, and the one HAL could not.** There, a provision had a single
+// shared credential and rotating it updated the provider's row; nothing enumerated who else held
+// the old one, so three nodes carried dead credentials for two days and the mesh reported success
+// (novox/hq ADR 0001). Here each pair has its own credential, and this is the list that makes
+// "every consumer" a set the mesh can name rather than a hope.
+//
+// Empty consumer means all of them.
+func (i *Inventory) HoldersOf(ctx context.Context, provision, consumer string) ([]Holder, error) {
+	rows, err := i.store.Pool().Query(ctx,
+		`select s.name, c.name, p.name from secret s
+		   join node c on c.id = s.consumer
+		   join node p on p.id = s.provider
+		  where s.name = $1 and ($2 = '' or c.name = $2)
+		  order by c.name, p.name`, provision, consumer)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var out []Holder
+	for rows.Next() {
+		var h Holder
+		if err := rows.Scan(&h.Provision, &h.Consumer, &h.Provider); err != nil {
+			return nil, err
+		}
+		out = append(out, h)
+	}
+	return out, rows.Err()
+}

internal/inventory/secrets_test.go

@@ -446,3 +446,106 @@ func TestASecretTheMeshWasGivenSurvivesAnOrdinaryPush(t *testing.T) {
 		t.Fatal("a given secret changed between two pushes, so the machine was handed two")
 	}
 }
+
+// Rotation has to know who holds the old credential, and that was the half HAL could not answer.
+//
+// There a provision had one shared credential; rotating it updated the provider's row and nothing
+// enumerated the consumers, so three nodes carried dead credentials for two days while the mesh
+// reported success (novox/hq ADR 0001). Here the holders are a set, and this is the query that
+// makes "every consumer" nameable rather than hopeful.
+func TestEveryHolderOfACredentialCanBeNamed(t *testing.T) {
+	inv, ctx := twoNodesWithKeys(t)
+	third, err := inv.AddNode(ctx, "third")
+	if err != nil {
+		t.Fatal(err)
+	}
+	key, _ := aSealingKey(t)
+	if err := inv.RecordSealingKey(ctx, third.ID, key); err != nil {
+		t.Fatal(err)
+	}
+	for _, consumer := range []string{"consumer", "third"} {
+		if _, err := inv.SecretFor(ctx, "database", consumer, "provider"); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// And one for a different provision, which must not be swept up.
+	if _, err := inv.SecretFor(ctx, "cache", "consumer", "provider"); err != nil {
+		t.Fatal(err)
+	}
+
+	holders, err := inv.HoldersOf(ctx, "database", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(holders) != 2 {
+		t.Fatalf("a holder of the credential was not named: %+v", holders)
+	}
+	for _, h := range holders {
+		if h.Provision != "database" {
+			t.Fatalf("rotating one provision would have touched %q", h.Provision)
+		}
+	}
+
+	// One machine's, when that is what was asked for. Rotating the other nine because one is
+	// suspected is a great deal of disruption for one suspicion.
+	one, err := inv.HoldersOf(ctx, "database", "third")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(one) != 1 || one[0].Consumer != "third" {
+		t.Fatalf("asking for one machine's holder gave %+v", one)
+	}
+}
+
+// And rotating gives both ends a new credential, together — the same one.
+func TestRotatingGivesBothEndsTheSameNewCredential(t *testing.T) {
+	inv, ctx := twoNodesWithKeys(t)
+	before, err := inv.SecretFor(ctx, "database", "consumer", "provider")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RotateSecret(ctx, "database", "consumer", "provider"); err != nil {
+		t.Fatal(err)
+	}
+	after, err := inv.SecretFor(ctx, "database", "consumer", "provider")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if after.ForConsumer == before.ForConsumer {
+		t.Fatal("the consumer was handed the credential that was just rotated away")
+	}
+	if after.ForProvider == before.ForProvider {
+		t.Fatal("the provider was left creating the old password, which is the fault exactly")
+	}
+	// The two halves are the same secret sealed twice, which is the whole point: a provider
+	// creating one password and a consumer given another is a mesh that reports success and
+	// cannot connect.
+	if after.ForConsumer == after.ForProvider {
+		t.Fatal("both ends were sealed identically, so one of them cannot open it")
+	}
+
+	// Rotating one pair leaves every other holder alone. Otherwise "rotate this machine's
+	// credential" is a mesh-wide outage with a narrow name.
+	third, err := inv.AddNode(ctx, "third")
+	if err != nil {
+		t.Fatal(err)
+	}
+	key, _ := aSealingKey(t)
+	if err := inv.RecordSealingKey(ctx, third.ID, key); err != nil {
+		t.Fatal(err)
+	}
+	untouched, err := inv.SecretFor(ctx, "database", "third", "provider")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RotateSecret(ctx, "database", "consumer", "provider"); err != nil {
+		t.Fatal(err)
+	}
+	again, err := inv.SecretFor(ctx, "database", "third", "provider")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if again.ForConsumer != untouched.ForConsumer {
+		t.Fatal("rotating one machine's credential changed another machine's")
+	}
+}