mesh-controller/cmd/mesh-controller/rotate.go

package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"sort"
)

// rotateCommand replaces a credential and moves both ends together.
//
// **This is the invariant novox/hq ADR 0001 records as unowned, and it was measurably false.** On
// 2026-08-22 `provision_ensure` — documented as never rotating an existing secret — minted a new
// password on every adoption and updated only the provider's row. Consumers on three nodes held
// dead credentials for two days; two rows for one provision were written 216 ms apart, so at most
// one could match the live role. Nothing enumerated who held the old one, and nothing said so.
//
// Three things make that impossible here, and all three are deliberate:
//
// **The holders are a set the mesh can name.** Each pair has its own credential, so rotating one
// consumer's password touches one role and leaves every other consumer alone — and the list of who
// is affected is a query rather than an assumption.
//
// **Both ends are pushed by this command, not by a later one.** A rotation that changed the record
// and left the sending to whoever remembered is the fault above, exactly.
//
// **It is all-or-nothing.** If any affected machine cannot be resolved, nothing is sent and the old
// credential keeps working — which is a mesh that has not rotated, and is far better than one that
// has half-rotated.
func rotateCommand(ctx context.Context, args []string) error {
	set := flag.NewFlagSet("rotate", flag.ContinueOnError)
	// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
	// and rotating the other nine would be a great deal of disruption for one suspicion.
	only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
	positionals, err := parseAround(set, args)
	if err != nil {
		return err
	}
	if len(positionals) != 1 {
		return errors.New("rotate <provision> [--consumer <machine>]")
	}
	provision := positionals[0]

	open, err := openStores(ctx)
	if err != nil {
		return err
	}
	defer open.Close()
	inv := open.inventory

	holders, err := inv.HoldersOf(ctx, provision, *only)
	if err != nil {
		return err
	}
	if len(holders) == 0 {
		// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
		// and a rotation somebody believes happened is worse than one they know did not.
		if *only != "" {
			return fmt.Errorf(
				"%s holds no credential for %q, so there is nothing to rotate. `plan %s` says "+
					"what it does hold", *only, provision, *only)
		}
		return fmt.Errorf(
			"nothing in this mesh holds a credential for %q, so there is nothing to rotate",
			provision)
	}

	// Every machine at both ends, named before anything changes. A person about to rotate a
	// production credential is entitled to know the blast radius before it is the past tense.
	affected := map[string]bool{}
	for _, h := range holders {
		affected[h.Consumer] = true
		affected[h.Provider] = true
	}
	machines := make([]string, 0, len(affected))
	for name := range affected {
		machines = append(machines, name)
	}
	sort.Strings(machines)

	fmt.Printf("rotating %s for %d holder(s):\n", provision, len(holders))
	for _, h := range holders {
		// The module, because a machine may hold several credentials for one provision and
		// rotating "anchor's database password" now means rotating three of them.
		fmt.Printf("  %s on %s, from %s\n", h.ConsumerModule, h.Consumer, h.Provider)
	}

	for _, h := range holders {
		if err := inv.RotateSecret(ctx, h.Provision, h.Consumer, h.ConsumerModule, h.Provider); err != nil {
			// Partly rotated, and said so plainly. What is gone is remade on the next push, so
			// the remedy is to run this again rather than to repair anything — but a machine
			// whose secret was discarded and not resent is holding a credential the provider is
			// about to stop honouring, and that is worth knowing now.
			return fmt.Errorf(
				"rotating %s for %s on %s from %s: %w\n\nSome credentials were discarded and "+
					"not yet sent. Run this again once the cause is fixed",
				h.Provision, h.ConsumerModule, h.Consumer, h.Provider, err)
		}
	}

	// **Both ends, in one send.** There is a window either way — a role's password changes on the
	// provider and the file changes on the consumer, and they cannot be simultaneous — so the
	// honest thing is to make it as short as the broker allows and to never leave it open across
	// a command boundary, where it depends on somebody's memory.
	fmt.Printf("\nsending to both ends:\n")
	if err := sendTo(ctx, open, machines); err != nil {
		return fmt.Errorf(
			"%w\n\nThe old credentials are gone from the mesh and the new ones have not arrived. "+
				"Nothing on those machines has changed yet, so what is running keeps working "+
				"until the provider next applies. Fix the cause and run `push --behind`", err)
	}

	fmt.Printf("\n%d machine(s) told. Until both ends have applied, a consumer whose password "+
		"changed cannot authenticate — `status` says who is still behind\n", len(machines))
	return nil
}