diff --git a/Makefile b/Makefile index c116966..3c53ef2 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,17 @@ builder-image: @echo @docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes' +# The provisioner ships as an image too, because it is the thing that makes a sealed credential +# true on a machine -- and the mesh cannot, having discarded the plaintext. +PROVISIONER_IMAGE ?= mesh-provision-postgres:$(VERSION) +PROVISIONER_DEV_TAG ?= mesh-provision-postgres:development + +provisioner-image: + docker build -f examples/postgres-provisioner/Dockerfile \ + -t $(PROVISIONER_IMAGE) -t $(PROVISIONER_DEV_TAG) . + @echo + @docker image inspect $(PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes' + # The whole gate. Raises a database, runs everything against it, and takes it down again -- # including when the tests fail, which is why the teardown is not conditional. check: fmt vet postgres diff --git a/cmd/mesh-control/main.go b/cmd/mesh-control/main.go index d068d38..0368d7d 100644 --- a/cmd/mesh-control/main.go +++ b/cmd/mesh-control/main.go @@ -69,6 +69,8 @@ func run() error { return buildCommand(ctx, args[1:]) case "builder": return builderCommand(ctx, args[1:]) + case "rotate": + return rotateCommand(ctx, args[1:]) case "builds": return buildsCommand(ctx, args[1:]) case "pin": @@ -142,6 +144,7 @@ func usage() { build [--ref R] have a build machine build it, and record what came out builds [] what has been built lately, and what came of it builder issue a broker account for a build machine, scoped to build work + rotate [--consumer ] a new credential for every holder, both ends at once pin which node this one gets a provision from unpin put that question back plan [--files|--json] what that node would run, and why @@ -1662,6 +1665,66 @@ func pushCommand(ctx context.Context, args []string) error { return nil } +// sendTo resolves and sends to exactly the machines named, or refuses without sending anything. +// +// The same all-or-nothing rule push follows, and for the same reason: a rotation that reached the +// consumer and refused on the provider would leave one end holding a credential the other has +// never heard of — which is the state this whole mechanism exists to make impossible. +func sendTo(ctx context.Context, inv *inventory.Inventory, names []string) error { + ident, err := openIdentity(ctx) + if err != nil { + return err + } + defer ident.Close() + + gens, err := generators(ctx, inv) + if err != nil { + return err + } + + type ready struct { + node string + resources []map[string]any + } + var sending []ready + var refusals []string + for _, name := range names { + plan, settings, err := planFor(ctx, inv, name) + if err != nil { + refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) + continue + } + resources, err := declarationWith(ctx, inv, name, plan, settings, gens) + if err != nil { + refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) + continue + } + sending = append(sending, ready{name, resources}) + } + if len(refusals) > 0 { + return fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s", + len(refusals), strings.Join(refusals, "\n\n")) + } + + server, err := link.Connect(nil, nil) + if err != nil { + return err + } + defer server.Close() + + for _, s := range sending { + body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources}) + if err != nil { + return err + } + if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil { + return err + } + fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.resources)) + } + return nil +} + // short is a commit as a person refers to it. func short(commit string) string { if len(commit) > 8 { diff --git a/cmd/mesh-control/rotate.go b/cmd/mesh-control/rotate.go new file mode 100644 index 0000000..db303d5 --- /dev/null +++ b/cmd/mesh-control/rotate.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "sort" +) + +// rotateCommand replaces a credential and moves both ends together. +// +// **This is the invariant novox/hq ADR 0001 records as unowned, and it was measurably false.** On +// 2026-08-22 `provision_ensure` — documented as never rotating an existing secret — minted a new +// password on every adoption and updated only the provider's row. Consumers on three nodes held +// dead credentials for two days; two rows for one provision were written 216 ms apart, so at most +// one could match the live role. Nothing enumerated who held the old one, and nothing said so. +// +// Three things make that impossible here, and all three are deliberate: +// +// **The holders are a set the mesh can name.** Each pair has its own credential, so rotating one +// consumer's password touches one role and leaves every other consumer alone — and the list of who +// is affected is a query rather than an assumption. +// +// **Both ends are pushed by this command, not by a later one.** A rotation that changed the record +// and left the sending to whoever remembered is the fault above, exactly. +// +// **It is all-or-nothing.** If any affected machine cannot be resolved, nothing is sent and the old +// credential keeps working — which is a mesh that has not rotated, and is far better than one that +// has half-rotated. +func rotateCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("rotate", flag.ContinueOnError) + // One consumer rather than all of them. Ordinary: a credential is suspected on one machine, + // and rotating the other nine would be a great deal of disruption for one suspicion. + only := set.String("consumer", "", "only this machine's credential, rather than every holder's") + positionals, err := parseAround(set, args) + if err != nil { + return err + } + if len(positionals) != 1 { + return errors.New("rotate [--consumer ]") + } + provision := positionals[0] + + inv, err := openInventory(ctx) + if err != nil { + return err + } + defer inv.Close() + + holders, err := inv.HoldersOf(ctx, provision, *only) + if err != nil { + return err + } + if len(holders) == 0 { + // Said, not silent. "Nobody holds this" and "this did not run" must never look the same — + // and a rotation somebody believes happened is worse than one they know did not. + if *only != "" { + return fmt.Errorf( + "%s holds no credential for %q, so there is nothing to rotate. `plan %s` says "+ + "what it does hold", *only, provision, *only) + } + return fmt.Errorf( + "nothing in this mesh holds a credential for %q, so there is nothing to rotate", + provision) + } + + // Every machine at both ends, named before anything changes. A person about to rotate a + // production credential is entitled to know the blast radius before it is the past tense. + affected := map[string]bool{} + for _, h := range holders { + affected[h.Consumer] = true + affected[h.Provider] = true + } + machines := make([]string, 0, len(affected)) + for name := range affected { + machines = append(machines, name) + } + sort.Strings(machines) + + fmt.Printf("rotating %s for %d holder(s):\n", provision, len(holders)) + for _, h := range holders { + fmt.Printf(" %s from %s\n", h.Consumer, h.Provider) + } + + for _, h := range holders { + if err := inv.RotateSecret(ctx, h.Provision, h.Consumer, h.Provider); err != nil { + // Partly rotated, and said so plainly. What is gone is remade on the next push, so + // the remedy is to run this again rather than to repair anything — but a machine + // whose secret was discarded and not resent is holding a credential the provider is + // about to stop honouring, and that is worth knowing now. + return fmt.Errorf( + "rotating %s for %s from %s: %w\n\nSome credentials were discarded and not yet "+ + "sent. Run this again once the cause is fixed", + h.Provision, h.Consumer, h.Provider, err) + } + } + + // **Both ends, in one send.** There is a window either way — a role's password changes on the + // provider and the file changes on the consumer, and they cannot be simultaneous — so the + // honest thing is to make it as short as the broker allows and to never leave it open across + // a command boundary, where it depends on somebody's memory. + fmt.Printf("\nsending to both ends:\n") + if err := sendTo(ctx, inv, machines); err != nil { + return fmt.Errorf( + "%w\n\nThe old credentials are gone from the mesh and the new ones have not arrived. "+ + "Nothing on those machines has changed yet, so what is running keeps working "+ + "until the provider next applies. Fix the cause and run `push --behind`", err) + } + + fmt.Printf("\n%d machine(s) told. Until both ends have applied, a consumer whose password "+ + "changed cannot authenticate — `status` says who is still behind\n", len(machines)) + return nil +} diff --git a/examples/postgres-provisioner/main.go b/examples/postgres-provisioner/main.go index 3c522d6..7aadcc9 100644 --- a/examples/postgres-provisioner/main.go +++ b/examples/postgres-provisioner/main.go @@ -24,6 +24,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "net/url" "os" "os/signal" "path/filepath" @@ -171,7 +172,11 @@ func run(ctx context.Context) error { return fmt.Errorf("the manifest at %s is not readable: %w", grants, err) } - db, err := pgx.Connect(ctx, os.Getenv("MESH_PROVISION_POSTGRES")) + where, err := connectionString() + if err != nil { + return err + } + db, err := pgx.Connect(ctx, where) if err != nil { return err } @@ -306,3 +311,51 @@ func sorted(given []contribution) []contribution { // to be safe today" is not a property anything should rest on. func quoteName(s string) string { return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` } func quoteString(s string) string { return `'` + strings.ReplaceAll(s, `'`, `''`) + `'` } + +// connectionString is where this provisioner reaches the database it owns. +// +// **The password comes from a file**, because that is how the mesh delivers one. A module's own +// secret — a superuser password here — is sealed to the machine and written by the host; a +// provisioner told to take it from an environment variable would need somebody to read the file +// and pass it in, which is a person in the middle of the one path that exists so there is not +// one. +// +// It is also the difference between a credential that lives in a file and one that lives in a +// process listing: `docker inspect` prints environment, and a superuser password printed by an +// ordinary diagnostic is a superuser password in whatever collected that diagnostic. +// +// MESH_PROVISION_POSTGRES alone still works, for a provisioner somebody runs by hand. +func connectionString() (string, error) { + where := strings.TrimSpace(os.Getenv("MESH_PROVISION_POSTGRES")) + if where == "" { + return "", fmt.Errorf( + "MESH_PROVISION_POSTGRES is not set, so this provisioner does not know which " + + "database it owns") + } + path := strings.TrimSpace(os.Getenv("MESH_PROVISION_PASSWORD_FILE")) + if path == "" { + return where, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf( + "cannot read the password this provisioner was given at %s: %w", path, err) + } + password := strings.TrimSpace(string(raw)) + if password == "" { + // An empty file connects as nobody and is refused by the database, three layers from + // here, as an authentication problem with no cause anybody changed. + return "", fmt.Errorf("%s is empty, so this provisioner has no password", path) + } + + parsed, err := url.Parse(where) + if err != nil { + return "", fmt.Errorf("MESH_PROVISION_POSTGRES is not a URL: %w", err) + } + user := parsed.User.Username() + if user == "" { + user = "postgres" + } + parsed.User = url.UserPassword(user, password) + return parsed.String(), nil +} diff --git a/examples/postgres-provisioner/where_test.go b/examples/postgres-provisioner/where_test.go new file mode 100644 index 0000000..49288f6 --- /dev/null +++ b/examples/postgres-provisioner/where_test.go @@ -0,0 +1,69 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// The password comes from a file, because that is how the mesh delivers one. +// +// A provisioner told to take a superuser password from an environment variable needs somebody to +// read the sealed file and pass it in — a person in the middle of the one path that exists so +// there is not one. It is also the difference between a credential in a file and one in a process +// listing: `docker inspect` prints environment. +func TestTheSuperuserPasswordComesFromTheFileTheMeshWrote(t *testing.T) { + path := filepath.Join(t.TempDir(), "superuser") + if err := os.WriteFile(path, []byte("the-sealed-one\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("MESH_PROVISION_POSTGRES", "postgres://postgres@127.0.0.1:5433/postgres?sslmode=disable") + t.Setenv("MESH_PROVISION_PASSWORD_FILE", path) + + where, err := connectionString() + if err != nil { + t.Fatal(err) + } + if !strings.Contains(where, "the-sealed-one") { + t.Fatalf("the password the mesh wrote is not in the connection: %s", where) + } + if !strings.Contains(where, "127.0.0.1:5433") || !strings.Contains(where, "sslmode=disable") { + t.Fatalf("the rest of the connection was lost: %s", where) + } +} + +// An empty file connects as nobody and is refused by the database three layers away, as an +// authentication problem with no cause anybody changed. +func TestAnEmptyPasswordFileIsRefusedHere(t *testing.T) { + path := filepath.Join(t.TempDir(), "superuser") + if err := os.WriteFile(path, []byte("\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("MESH_PROVISION_POSTGRES", "postgres://postgres@127.0.0.1:5433/postgres") + t.Setenv("MESH_PROVISION_PASSWORD_FILE", path) + if _, err := connectionString(); err == nil { + t.Fatal("a provisioner with no password reported one") + } +} + +// And a provisioner somebody runs by hand still works with the URL alone. +func TestAConnectionWithNoPasswordFileIsLeftAlone(t *testing.T) { + t.Setenv("MESH_PROVISION_POSTGRES", "postgres://postgres:typed@127.0.0.1:5433/postgres") + t.Setenv("MESH_PROVISION_PASSWORD_FILE", "") + where, err := connectionString() + if err != nil { + t.Fatal(err) + } + if where != "postgres://postgres:typed@127.0.0.1:5433/postgres" { + t.Fatalf("the connection was rewritten when it should have been left alone: %s", where) + } +} + +func TestAProvisionerWithNoDatabaseSaysSo(t *testing.T) { + t.Setenv("MESH_PROVISION_POSTGRES", "") + t.Setenv("MESH_PROVISION_PASSWORD_FILE", "") + if _, err := connectionString(); err == nil { + t.Fatal("a provisioner that does not know which database it owns reported one") + } +} diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index 1a36fe3..dccc1b4 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -230,3 +230,42 @@ func (i *Inventory) AcceptSecretForModule(ctx context.Context, node, module, nam record.ID, module, name, sealed.ForConsumer, key) return err } + +// Holder is one end-to-end credential: who gets it and who must create it. +type Holder struct { + Provision string + Consumer string + Provider string +} + +// HoldersOf is every pair sharing a credential for one provision. +// +// **The question rotation has to ask, and the one HAL could not.** There, a provision had a single +// shared credential and rotating it updated the provider's row; nothing enumerated who else held +// the old one, so three nodes carried dead credentials for two days and the mesh reported success +// (novox/hq ADR 0001). Here each pair has its own credential, and this is the list that makes +// "every consumer" a set the mesh can name rather than a hope. +// +// Empty consumer means all of them. +func (i *Inventory) HoldersOf(ctx context.Context, provision, consumer string) ([]Holder, error) { + rows, err := i.store.Pool().Query(ctx, + `select s.name, c.name, p.name from secret s + join node c on c.id = s.consumer + join node p on p.id = s.provider + where s.name = $1 and ($2 = '' or c.name = $2) + order by c.name, p.name`, provision, consumer) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []Holder + for rows.Next() { + var h Holder + if err := rows.Scan(&h.Provision, &h.Consumer, &h.Provider); err != nil { + return nil, err + } + out = append(out, h) + } + return out, rows.Err() +} diff --git a/internal/inventory/secrets_test.go b/internal/inventory/secrets_test.go index e7f7ee8..9c70925 100644 --- a/internal/inventory/secrets_test.go +++ b/internal/inventory/secrets_test.go @@ -446,3 +446,106 @@ func TestASecretTheMeshWasGivenSurvivesAnOrdinaryPush(t *testing.T) { t.Fatal("a given secret changed between two pushes, so the machine was handed two") } } + +// Rotation has to know who holds the old credential, and that was the half HAL could not answer. +// +// There a provision had one shared credential; rotating it updated the provider's row and nothing +// enumerated the consumers, so three nodes carried dead credentials for two days while the mesh +// reported success (novox/hq ADR 0001). Here the holders are a set, and this is the query that +// makes "every consumer" nameable rather than hopeful. +func TestEveryHolderOfACredentialCanBeNamed(t *testing.T) { + inv, ctx := twoNodesWithKeys(t) + third, err := inv.AddNode(ctx, "third") + if err != nil { + t.Fatal(err) + } + key, _ := aSealingKey(t) + if err := inv.RecordSealingKey(ctx, third.ID, key); err != nil { + t.Fatal(err) + } + for _, consumer := range []string{"consumer", "third"} { + if _, err := inv.SecretFor(ctx, "database", consumer, "provider"); err != nil { + t.Fatal(err) + } + } + // And one for a different provision, which must not be swept up. + if _, err := inv.SecretFor(ctx, "cache", "consumer", "provider"); err != nil { + t.Fatal(err) + } + + holders, err := inv.HoldersOf(ctx, "database", "") + if err != nil { + t.Fatal(err) + } + if len(holders) != 2 { + t.Fatalf("a holder of the credential was not named: %+v", holders) + } + for _, h := range holders { + if h.Provision != "database" { + t.Fatalf("rotating one provision would have touched %q", h.Provision) + } + } + + // One machine's, when that is what was asked for. Rotating the other nine because one is + // suspected is a great deal of disruption for one suspicion. + one, err := inv.HoldersOf(ctx, "database", "third") + if err != nil { + t.Fatal(err) + } + if len(one) != 1 || one[0].Consumer != "third" { + t.Fatalf("asking for one machine's holder gave %+v", one) + } +} + +// And rotating gives both ends a new credential, together — the same one. +func TestRotatingGivesBothEndsTheSameNewCredential(t *testing.T) { + inv, ctx := twoNodesWithKeys(t) + before, err := inv.SecretFor(ctx, "database", "consumer", "provider") + if err != nil { + t.Fatal(err) + } + if err := inv.RotateSecret(ctx, "database", "consumer", "provider"); err != nil { + t.Fatal(err) + } + after, err := inv.SecretFor(ctx, "database", "consumer", "provider") + if err != nil { + t.Fatal(err) + } + if after.ForConsumer == before.ForConsumer { + t.Fatal("the consumer was handed the credential that was just rotated away") + } + if after.ForProvider == before.ForProvider { + t.Fatal("the provider was left creating the old password, which is the fault exactly") + } + // The two halves are the same secret sealed twice, which is the whole point: a provider + // creating one password and a consumer given another is a mesh that reports success and + // cannot connect. + if after.ForConsumer == after.ForProvider { + t.Fatal("both ends were sealed identically, so one of them cannot open it") + } + + // Rotating one pair leaves every other holder alone. Otherwise "rotate this machine's + // credential" is a mesh-wide outage with a narrow name. + third, err := inv.AddNode(ctx, "third") + if err != nil { + t.Fatal(err) + } + key, _ := aSealingKey(t) + if err := inv.RecordSealingKey(ctx, third.ID, key); err != nil { + t.Fatal(err) + } + untouched, err := inv.SecretFor(ctx, "database", "third", "provider") + if err != nil { + t.Fatal(err) + } + if err := inv.RotateSecret(ctx, "database", "consumer", "provider"); err != nil { + t.Fatal(err) + } + again, err := inv.SecretFor(ctx, "database", "third", "provider") + if err != nil { + t.Fatal(err) + } + if again.ForConsumer != untouched.ForConsumer { + t.Fatal("rotating one machine's credential changed another machine's") + } +}