Rotate a credential and move both ends together
The invariant novox/hq ADR 0001 records as unowned, and it was measurably false in HAL: a provision documented as never rotating minted a new password on every adoption and updated only the provider's row. Consumers on three nodes held dead credentials for two days while the mesh reported success. Nothing enumerated who held the old one. Three things make that impossible here. The holders are a set the mesh can name — each pair has its own credential, so rotating one consumer touches one role and the affected list is a query rather than an assumption. Both ends are pushed by this command rather than a later one, because leaving the sending to whoever remembered is the fault exactly. And it is all-or-nothing: if any affected machine cannot be resolved, nothing is sent and the old credential keeps working, which is a mesh that has not rotated rather than one that has half-rotated. The window is stated rather than hidden: a role's password changes on the provider and the file changes on the consumer, and they cannot be simultaneous. The provisioner now takes its superuser password from the file the mesh wrote, which is how the mesh delivers one. Passing it through the environment needed a person in the middle of the one path that exists so there is not one — and put a superuser password where `docker inspect` prints it.
This commit is contained in:
@@ -69,6 +69,8 @@ func run() error {
|
||||
return buildCommand(ctx, args[1:])
|
||||
case "builder":
|
||||
return builderCommand(ctx, args[1:])
|
||||
case "rotate":
|
||||
return rotateCommand(ctx, args[1:])
|
||||
case "builds":
|
||||
return buildsCommand(ctx, args[1:])
|
||||
case "pin":
|
||||
@@ -142,6 +144,7 @@ func usage() {
|
||||
build <repository> [--ref R] have a build machine build it, and record what came out
|
||||
builds [<module>] what has been built lately, and what came of it
|
||||
builder issue <name> a broker account for a build machine, scoped to build work
|
||||
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
|
||||
pin <node> <provision> <from> which node this one gets a provision from
|
||||
unpin <node> <provision> put that question back
|
||||
plan <node> [--files|--json] what that node would run, and why
|
||||
@@ -1662,6 +1665,66 @@ func pushCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// sendTo resolves and sends to exactly the machines named, or refuses without sending anything.
|
||||
//
|
||||
// The same all-or-nothing rule push follows, and for the same reason: a rotation that reached the
|
||||
// consumer and refused on the provider would leave one end holding a credential the other has
|
||||
// never heard of — which is the state this whole mechanism exists to make impossible.
|
||||
func sendTo(ctx context.Context, inv *inventory.Inventory, names []string) error {
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ident.Close()
|
||||
|
||||
gens, err := generators(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
type ready struct {
|
||||
node string
|
||||
resources []map[string]any
|
||||
}
|
||||
var sending []ready
|
||||
var refusals []string
|
||||
for _, name := range names {
|
||||
plan, settings, err := planFor(ctx, inv, name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
resources, err := declarationWith(ctx, inv, name, plan, settings, gens)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
sending = append(sending, ready{name, resources})
|
||||
}
|
||||
if len(refusals) > 0 {
|
||||
return fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s",
|
||||
len(refusals), strings.Join(refusals, "\n\n"))
|
||||
}
|
||||
|
||||
server, err := link.Connect(nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
for _, s := range sending {
|
||||
body, err := json.Marshal(map[string]any{"declaration": 1, "resources": s.resources})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := link.Declare(ctx, server.Channel(), ident, s.node, body, 15*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf(" sent %s %d resource(s)\n", s.node, len(s.resources))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// short is a commit as a person refers to it.
|
||||
func short(commit string) string {
|
||||
if len(commit) > 8 {
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// rotateCommand replaces a credential and moves both ends together.
|
||||
//
|
||||
// **This is the invariant novox/hq ADR 0001 records as unowned, and it was measurably false.** On
|
||||
// 2026-08-22 `provision_ensure` — documented as never rotating an existing secret — minted a new
|
||||
// password on every adoption and updated only the provider's row. Consumers on three nodes held
|
||||
// dead credentials for two days; two rows for one provision were written 216 ms apart, so at most
|
||||
// one could match the live role. Nothing enumerated who held the old one, and nothing said so.
|
||||
//
|
||||
// Three things make that impossible here, and all three are deliberate:
|
||||
//
|
||||
// **The holders are a set the mesh can name.** Each pair has its own credential, so rotating one
|
||||
// consumer's password touches one role and leaves every other consumer alone — and the list of who
|
||||
// is affected is a query rather than an assumption.
|
||||
//
|
||||
// **Both ends are pushed by this command, not by a later one.** A rotation that changed the record
|
||||
// and left the sending to whoever remembered is the fault above, exactly.
|
||||
//
|
||||
// **It is all-or-nothing.** If any affected machine cannot be resolved, nothing is sent and the old
|
||||
// credential keeps working — which is a mesh that has not rotated, and is far better than one that
|
||||
// has half-rotated.
|
||||
func rotateCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("rotate", flag.ContinueOnError)
|
||||
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
|
||||
// and rotating the other nine would be a great deal of disruption for one suspicion.
|
||||
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("rotate <provision> [--consumer <machine>]")
|
||||
}
|
||||
provision := positionals[0]
|
||||
|
||||
inv, err := openInventory(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer inv.Close()
|
||||
|
||||
holders, err := inv.HoldersOf(ctx, provision, *only)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(holders) == 0 {
|
||||
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
|
||||
// and a rotation somebody believes happened is worse than one they know did not.
|
||||
if *only != "" {
|
||||
return fmt.Errorf(
|
||||
"%s holds no credential for %q, so there is nothing to rotate. `plan %s` says "+
|
||||
"what it does hold", *only, provision, *only)
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"nothing in this mesh holds a credential for %q, so there is nothing to rotate",
|
||||
provision)
|
||||
}
|
||||
|
||||
// Every machine at both ends, named before anything changes. A person about to rotate a
|
||||
// production credential is entitled to know the blast radius before it is the past tense.
|
||||
affected := map[string]bool{}
|
||||
for _, h := range holders {
|
||||
affected[h.Consumer] = true
|
||||
affected[h.Provider] = true
|
||||
}
|
||||
machines := make([]string, 0, len(affected))
|
||||
for name := range affected {
|
||||
machines = append(machines, name)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
|
||||
fmt.Printf("rotating %s for %d holder(s):\n", provision, len(holders))
|
||||
for _, h := range holders {
|
||||
fmt.Printf(" %s from %s\n", h.Consumer, h.Provider)
|
||||
}
|
||||
|
||||
for _, h := range holders {
|
||||
if err := inv.RotateSecret(ctx, h.Provision, h.Consumer, h.Provider); err != nil {
|
||||
// Partly rotated, and said so plainly. What is gone is remade on the next push, so
|
||||
// the remedy is to run this again rather than to repair anything — but a machine
|
||||
// whose secret was discarded and not resent is holding a credential the provider is
|
||||
// about to stop honouring, and that is worth knowing now.
|
||||
return fmt.Errorf(
|
||||
"rotating %s for %s from %s: %w\n\nSome credentials were discarded and not yet "+
|
||||
"sent. Run this again once the cause is fixed",
|
||||
h.Provision, h.Consumer, h.Provider, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **Both ends, in one send.** There is a window either way — a role's password changes on the
|
||||
// provider and the file changes on the consumer, and they cannot be simultaneous — so the
|
||||
// honest thing is to make it as short as the broker allows and to never leave it open across
|
||||
// a command boundary, where it depends on somebody's memory.
|
||||
fmt.Printf("\nsending to both ends:\n")
|
||||
if err := sendTo(ctx, inv, machines); err != nil {
|
||||
return fmt.Errorf(
|
||||
"%w\n\nThe old credentials are gone from the mesh and the new ones have not arrived. "+
|
||||
"Nothing on those machines has changed yet, so what is running keeps working "+
|
||||
"until the provider next applies. Fix the cause and run `push --behind`", err)
|
||||
}
|
||||
|
||||
fmt.Printf("\n%d machine(s) told. Until both ends have applied, a consumer whose password "+
|
||||
"changed cannot authenticate — `status` says who is still behind\n", len(machines))
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user