298 lines
12 KiB
Go
298 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// rotateCommand replaces a credential and moves both ends together.
|
|
//
|
|
// **This is the invariant novox/hq ADR 0001 records as unowned, and it was measurably false.** On
|
|
// 2026-08-22 `provision_ensure` — documented as never rotating an existing secret — minted a new
|
|
// password on every adoption and updated only the provider's row. Consumers on three nodes held
|
|
// dead credentials for two days; two rows for one provision were written 216 ms apart, so at most
|
|
// one could match the live role. Nothing enumerated who held the old one, and nothing said so.
|
|
//
|
|
// Three things make that impossible here, and all three are deliberate:
|
|
//
|
|
// **The holders are a set the mesh can name.** Each pair has its own credential, so rotating one
|
|
// consumer's password touches one role and leaves every other consumer alone — and the list of who
|
|
// is affected is a query rather than an assumption.
|
|
//
|
|
// **Both ends are pushed by this command, not by a later one.** A rotation that changed the record
|
|
// and left the sending to whoever remembered is the fault above, exactly.
|
|
//
|
|
// **It is all-or-nothing.** If any affected machine cannot be resolved, nothing is sent and the old
|
|
// credential keeps working — which is a mesh that has not rotated, and is far better than one that
|
|
// has half-rotated.
|
|
func rotateCommand(ctx context.Context, args []string) error {
|
|
set := flag.NewFlagSet("rotate", flag.ContinueOnError)
|
|
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
|
|
// and rotating the other nine would be a great deal of disruption for one suspicion.
|
|
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
|
|
// One consuming module rather than every module on the machine. A machine runs many consumers
|
|
// of one provision, each with its own credential; one module that leaked its credential (novox/hq
|
|
// issue 268) is no reason to restart every other one on the machine.
|
|
module := set.String("module", "", "only this consuming module's credential")
|
|
// Why, which the vault asks of every rotation (issue 767) and the hand-act log records; required only where a
|
|
// provider's credential is the vault's, so nothing the operator typed before changes meaning.
|
|
why := addHandActFlags(set)
|
|
positionals, err := parseAround(set, args)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(positionals) != 1 {
|
|
return errors.New("rotate <provision> [--consumer <machine>] [--module <module>]")
|
|
}
|
|
provision := positionals[0]
|
|
|
|
open, err := openStores(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer open.Close()
|
|
inv := open.inventory
|
|
|
|
// A credential the provider's software issued is not the mesh's to remake (novox/hq ADR 0305 §9).
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := issuedRefusesRotation(shelf, provision); err != nil {
|
|
return err
|
|
}
|
|
|
|
holders, err := inv.HoldersOf(ctx, provision, *only)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
holders = ofModule(holders, *module)
|
|
if len(holders) == 0 && *module != "" {
|
|
return fmt.Errorf(
|
|
"no module %s%s holds a credential for %q, so there is nothing to rotate. `plan <machine>` "+
|
|
"says what a machine holds", *module, onMachine(*only), provision)
|
|
}
|
|
if len(holders) == 0 {
|
|
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
|
|
// and a rotation somebody believes happened is worse than one they know did not.
|
|
if *only != "" {
|
|
return fmt.Errorf(
|
|
"%s holds no credential for %q, so there is nothing to rotate. `plan %s` says "+
|
|
"what it does hold", *only, provision, *only)
|
|
}
|
|
return fmt.Errorf(
|
|
"nothing in this mesh holds a credential for %q, so there is nothing to rotate",
|
|
provision)
|
|
}
|
|
|
|
// Every machine at both ends, named before anything changes. A person about to rotate a
|
|
// production credential is entitled to know the blast radius before it is the past tense.
|
|
fmt.Printf("rotating %s for %d holder(s):\n", provision, len(holders))
|
|
for _, h := range holders {
|
|
// The module, because a machine may hold several credentials for one provision and
|
|
// rotating "anchor's database password" now means rotating three of them.
|
|
fmt.Printf(" %s on %s, from %s%s\n", h.ConsumerModule, h.Consumer, h.Provider, asLocal(h.Local))
|
|
}
|
|
|
|
// **The vault's, where the provider is filled the new way** (issue 767, ADR 0349 clause 8): its credential is
|
|
// the vault's, so the vault remakes it and its holders take it from there; the controller writes no value of its
|
|
// own and sends nothing for it. Which holders go where is read before anything changes.
|
|
toVault, today, err := rotateRoutes(holders,
|
|
func(provider, provision string) (string, error) { return offeringModuleOn(ctx, provider, provision) },
|
|
filledTheNewWay)
|
|
if err != nil {
|
|
return fmt.Errorf("nothing was rotated: %w", err)
|
|
}
|
|
if len(toVault) > 0 && !why.given() {
|
|
return errors.New("not rotated: the vault holds this credential and rotates it only with a reason, " +
|
|
"`--why \"…\"`; nothing was asked")
|
|
}
|
|
if why.given() {
|
|
why.record(ctx, "rotate", append([]string{provision}, rotateNarrowArgs(*only, *module)...))
|
|
}
|
|
for _, g := range toVault {
|
|
words, err := rotateProvisionInVault(theVault(ctx), g, *only, *module, actingAs(), *why.why)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Println(words)
|
|
}
|
|
if len(today) == 0 {
|
|
return nil
|
|
}
|
|
holders = today
|
|
machines := machinesOf(holders)
|
|
|
|
for _, h := range holders {
|
|
if err := inv.RotateSecret(ctx, h.Provision, h.Consumer, h.ConsumerModule, h.Provider, h.Local); err != nil {
|
|
// Partly rotated, and said so plainly. What is gone is remade on the next push, so
|
|
// the remedy is to run this again rather than to repair anything — but a machine
|
|
// whose secret was discarded and not resent is holding a credential the provider is
|
|
// about to stop honouring, and that is worth knowing now.
|
|
return fmt.Errorf(
|
|
"rotating %s for %s on %s from %s: %w\n\nSome credentials were discarded and "+
|
|
"not yet sent. Run this again once the cause is fixed",
|
|
h.Provision, h.ConsumerModule, h.Consumer, h.Provider, err)
|
|
}
|
|
}
|
|
|
|
// **Both ends, in one send.** There is a window either way — a role's password changes on the
|
|
// provider and the file changes on the consumer, and they cannot be simultaneous — so the
|
|
// honest thing is to make it as short as the broker allows and to never leave it open across
|
|
// a command boundary, where it depends on somebody's memory.
|
|
fmt.Printf("\nsending to both ends:\n")
|
|
if err := sendTo(ctx, open, machines); err != nil {
|
|
return fmt.Errorf(
|
|
"%w\n\nThe old credentials are gone from the mesh and the new ones have not arrived. "+
|
|
"Nothing on those machines has changed yet, so what is running keeps working "+
|
|
"until the provider next applies. Fix the cause and run `push --behind`", err)
|
|
}
|
|
|
|
fmt.Printf("\n%d machine(s) told. Until both ends have applied, a consumer whose password "+
|
|
"changed cannot authenticate — `status` says who is still behind\n", len(machines))
|
|
return nil
|
|
}
|
|
|
|
// machinesOf is every machine at both ends of these holders, in order.
|
|
func machinesOf(holders []inventory.Holder) []string {
|
|
affected := map[string]bool{}
|
|
for _, h := range holders {
|
|
affected[h.Consumer] = true
|
|
affected[h.Provider] = true
|
|
}
|
|
machines := make([]string, 0, len(affected))
|
|
for name := range affected {
|
|
machines = append(machines, name)
|
|
}
|
|
sort.Strings(machines)
|
|
return machines
|
|
}
|
|
|
|
// ofModule is the holders whose consuming module is this one; all of them when none is named.
|
|
func ofModule(holders []inventory.Holder, module string) []inventory.Holder {
|
|
if module == "" {
|
|
return holders
|
|
}
|
|
var out []inventory.Holder
|
|
for _, h := range holders {
|
|
if h.ConsumerModule == module {
|
|
out = append(out, h)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func onMachine(machine string) string {
|
|
if machine == "" {
|
|
return ""
|
|
}
|
|
return " on " + machine
|
|
}
|
|
|
|
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
|
|
func asLocal(local string) string {
|
|
if local == "" {
|
|
return ""
|
|
}
|
|
return " (as " + local + ")"
|
|
}
|
|
|
|
// rotateGroup is the holders of one provider's credential for one provision whose provider is filled the new way:
|
|
// the vault's to rotate (issue 767).
|
|
type rotateGroup struct {
|
|
Provider, Provision, Module string
|
|
Holders []inventory.Holder
|
|
}
|
|
|
|
// rotateRoutes says which holders the vault rotates, grouped by provider and provision, and which today's path
|
|
// does: the vault's are those whose provider's offering module on its machine is filled the new way (the switch
|
|
// `accept --provider` reads too). Anything that cannot be read is an error, never today's path by default: a value
|
|
// remade where its holders no longer look would be reported rotated and change nothing.
|
|
func rotateRoutes(holders []inventory.Holder, offering func(provider, provision string) (string, error),
|
|
ofTheVault vaultSwitch) ([]rotateGroup, []inventory.Holder, error) {
|
|
type key struct{ provider, provision string }
|
|
decided := map[key]int{} // index into groups, or -1 for today's path
|
|
var groups []rotateGroup
|
|
var today []inventory.Holder
|
|
for _, h := range holders {
|
|
k := key{h.Provider, h.Provision}
|
|
i, seen := decided[k]
|
|
if !seen {
|
|
module, err := offering(h.Provider, h.Provision)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("which module offers %s on %s cannot be read: %w", h.Provision, h.Provider, err)
|
|
}
|
|
ours := false
|
|
if module != "" {
|
|
if ours, err = ofTheVault(module); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
}
|
|
i = -1
|
|
if ours {
|
|
i = len(groups)
|
|
groups = append(groups, rotateGroup{Provider: h.Provider, Provision: h.Provision, Module: module})
|
|
}
|
|
decided[k] = i
|
|
}
|
|
if i < 0 {
|
|
today = append(today, h)
|
|
continue
|
|
}
|
|
groups[i].Holders = append(groups[i].Holders, h)
|
|
}
|
|
return groups, today, nil
|
|
}
|
|
|
|
// rotateNarrowArgs is --consumer and --module as typed, for the hand-act log.
|
|
func rotateNarrowArgs(consumer, module string) []string {
|
|
var out []string
|
|
if consumer != "" {
|
|
out = append(out, "--consumer", consumer)
|
|
}
|
|
if module != "" {
|
|
out = append(out, "--module", module)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// rotateProvisionInVault asks the vault to rotate one provider's credential for a provision (issue 767): a shared
|
|
// value is made anew for every holder, and the vault refuses it for one consumer alone; for a provider whose
|
|
// consumers keep a login each (ADR 0158 §11), consumer and module name whose own login is remade, and only theirs.
|
|
// The vault says secret-changed and each holder takes the new value from it; nothing is sealed or sent here.
|
|
func rotateProvisionInVault(ask vaultAsker, g rotateGroup, consumer, module, by, why string) (string, error) {
|
|
if strings.TrimSpace(why) == "" {
|
|
return "", errors.New("not rotated: the vault rotates a credential only with a reason, `--why \"…\"`; nothing was asked")
|
|
}
|
|
args := provisionArgs(g.Provider, g.Provision)
|
|
args["by"], args["why"] = by, strings.TrimSpace(why)
|
|
if consumer != "" {
|
|
args["consumer"] = consumer
|
|
}
|
|
if module != "" {
|
|
args["consumer_module"] = module
|
|
}
|
|
changed, err := endVaultCall(ask, "rotate", args)
|
|
if err != nil {
|
|
return "", fmt.Errorf("%s from %s not rotated, and nothing else was: %w", g.Provision, g.Provider, err)
|
|
}
|
|
if len(changed.Each) > 0 {
|
|
each := make([]string, 0, len(changed.Each))
|
|
for _, c := range changed.Each {
|
|
each = append(each, fmt.Sprintf("%s (generation %d)", c.Name, c.Generation))
|
|
}
|
|
return fmt.Sprintf("rotated in the vault, each consumer's own login of %s from %s, asked by %s: %s; the values "+
|
|
"are sealed and not shown, and each holder takes its own from the vault", g.Provision, g.Provider,
|
|
changed.By, strings.Join(each, ", ")), nil
|
|
}
|
|
return fmt.Sprintf("rotated %s in the vault (generation %d), one value for every holder of %s from %s, asked by "+
|
|
"%s; the value is sealed and not shown, and each holder takes it from the vault", changed.Name,
|
|
changed.Generation, g.Provision, g.Provider, changed.By), nil
|
|
}
|