Merge pull request 'Secrets vault: operator key, a second seal on every secret, recovery and export' (#34) from feat/secrets-vault into main

cmd/mesh-controller/main.go

@@ -100,6 +100,8 @@ func run() error {
 		return settingsCommand(ctx, args[1:])
 	case "secret":
 		return secretCommand(ctx, args[1:])
+	case "operator":
+		return operatorCommand(ctx, args[1:])
 	case "plan":
 		return planCommand(ctx, args[1:])
 	case "push":
@@ -155,6 +157,12 @@ func usage() {
   settings clear <module> [--node <n>]      take a layer away
   secret accept <node> <module> <name>     carry a value the mesh did not make and cannot invent
   secret accept ... --from <file>           ...read it from a file rather than being asked
+  secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>]
+                                            break-glass: open the operator-sealed copy, to a 0600 file
+  secret export [--out <file>]              every operator-sealed copy, ciphertext — keep it with the key
+  operator key make [--out <file>]          make the operator's sealing key, off the mesh; private half to the file only
+  operator key set <public> [--replace]     tell the mesh which operator key to seal to
+  operator key show                         the operator key, and what it can recover
   build <repository> [--ref R]   have a build machine build it, and record what came out
   build --behind                 build every module the mesh holds older than its source
   builds [<module>]              what has been built lately, and what came of it

cmd/mesh-controller/operator.go

@@ -0,0 +1,162 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/novox/mesh-controller/internal/secrets"
+)
+
+// operatorCommand is the mesh's one holder of secrets that is not a machine.
+//
+// **Every secret a module holds for itself is sealed to the node that uses it, and a node whose key
+// is gone takes its secrets with it** — the store's superuser and the broker's administrator among
+// them. novox/hq ADR 0085 (amended) gives them a second recipient: a person, with a sealing key
+// whose private half is made where the operator is and never enters the mesh. Making the key and
+// telling the mesh about it are two commands, on purpose: the first needs no mesh at all and runs
+// wherever the operator keeps things; the second gives the mesh the public half and nothing else.
+// The controller's own container is a scratch image with no writable path, which is the right
+// shape for a program that must hold no key — so the private half could not be written there
+// even by mistake.
+//
+//	operator key make [--out <file>]    make a keypair: private half to the file, public half printed
+//	operator key set <public>           tell the mesh which key to seal to
+//	operator key show                   the public key, its fingerprint, and what it can recover
+const operatorUsage = "operator key make [--out <file>] | operator key set <public> [--replace] | operator key show"
+
+func operatorCommand(ctx context.Context, args []string) error {
+	if len(args) < 2 || args[0] != "key" {
+		return errors.New(operatorUsage)
+	}
+	switch args[1] {
+	case "make":
+		return operatorKeyMake(args[2:])
+	case "set":
+		return operatorKeySet(ctx, args[2:])
+	case "show":
+		return operatorKeyShow(ctx)
+	default:
+		return errors.New(operatorUsage)
+	}
+}
+
+// operatorKeyMake needs no mesh: it is the operator's, run wherever the key will live.
+func operatorKeyMake(args []string) error {
+	set := flag.NewFlagSet("operator key make", flag.ContinueOnError)
+	out := set.String("out", "operator.key",
+		"where to write the private key (0600); keep it off the mesh, and keep it")
+	if err := set.Parse(args); err != nil {
+		return err
+	}
+	public, private, err := secrets.Keypair()
+	if err != nil {
+		return err
+	}
+	// Create-exclusive: a key somebody may still need is never overwritten, and there is no window
+	// between checking and writing in which one could appear.
+	if err := writeNew(*out, []byte(private+"\n")); err != nil {
+		return err
+	}
+	fmt.Printf("operator key %s\n", secrets.Fingerprint(public))
+	fmt.Printf("  private half written to %s (0600) — keep it off the mesh, and keep it\n", *out)
+	fmt.Printf("  public half, to give the mesh with `operator key set`:\n")
+	fmt.Printf("public %s\n", public)
+	return nil
+}
+
+func operatorKeySet(ctx context.Context, args []string) error {
+	rest, flags := split(args)
+	set := flag.NewFlagSet("operator key set", flag.ContinueOnError)
+	replace := set.Bool("replace", false,
+		"replace an existing operator key — secrets sealed to the old one stay sealed to it")
+	if err := set.Parse(flags); err != nil {
+		return err
+	}
+	if len(rest) != 1 {
+		return errors.New(operatorUsage)
+	}
+	public := strings.TrimSpace(rest[0])
+	if _, err := secrets.Seal(public, []byte("probe")); err != nil {
+		return fmt.Errorf("that is not a public sealing key: %w", err)
+	}
+
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+
+	if current, err := inv.OperatorKey(ctx); err != nil {
+		return err
+	} else if current != "" && current != public && !*replace {
+		return fmt.Errorf(
+			"the mesh already has an operator key (%s). Pass --replace to change it — "+
+				"secrets sealed to the current key stay sealed to it until each is issued again",
+			secrets.Fingerprint(current))
+	}
+	orphaned, err := inv.SetOperatorKey(ctx, public)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("operator key %s\n", secrets.Fingerprint(public))
+	fmt.Printf("  the mesh holds the public half only and cannot open what it seals to it;\n")
+	fmt.Printf("  from now on every secret a module holds for itself is sealed to it as well.\n")
+	fmt.Printf("  Secrets made before this cannot be — each is recoverable once issued again\n")
+	if orphaned > 0 {
+		fmt.Printf("  %d secret(s) are sealed to the previous key and stay so until issued again\n", orphaned)
+	}
+	return nil
+}
+
+func operatorKeyShow(ctx context.Context) error {
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+	key, err := inv.OperatorKey(ctx)
+	if err != nil {
+		return err
+	}
+	if key == "" {
+		fmt.Println("the mesh has no operator key; `operator key make` then `operator key set` gives it one")
+		return nil
+	}
+	kept, earlier, unrecoverable, err := inv.KeptForOperator(ctx)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("operator key %s\n  %s\n", secrets.Fingerprint(key), key)
+	fmt.Printf("  %d secret(s) recoverable with it\n", len(kept))
+	if len(earlier) > 0 {
+		fmt.Printf("  %d secret(s) sealed to an earlier operator key — recoverable with that key only, until issued again:\n", len(earlier))
+		for _, k := range earlier {
+			fmt.Printf("    %s  %s  %s  (%s)\n", k.Node, k.Module, k.Name, secrets.Fingerprint(k.Key))
+		}
+	}
+	if len(unrecoverable) > 0 {
+		fmt.Printf("  %d secret(s) not recoverable — made before the mesh had an operator key:\n", len(unrecoverable))
+		for _, k := range unrecoverable {
+			fmt.Printf("    %s  %s  %s\n", k.Node, k.Module, k.Name)
+		}
+	}
+	return nil
+}
+
+// readPrivateKey is the operator's key from the file `operator key make` wrote.
+func readPrivateKey(path string) (string, error) {
+	if path == "" {
+		return "", errors.New("--key <file> names the operator's private key, written by `operator key make`")
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(raw)), nil
+}

cmd/mesh-controller/plan.go

@@ -462,10 +462,26 @@ func declarationWith(ctx context.Context, open *stores, node string,
 		}
 	}
 
+	// And, for a module that keeps them, every operator-sealed secret in the mesh — the vault's
+	// copy, outside the store (novox/hq ADR 0085, amended). Read only; nothing here mints. The
+	// export changes whenever any secret in the mesh is made or rotated, so the vault's declaration
+	// changes with it and the vault node is sent again: that is what keeps its copy current, and
+	// the cost is one two-table read per composition of the vault's node, in every mode.
+	var kept *catalogue.KeptExport
+	for _, m := range plan.Modules {
+		if m.Keeps == "" {
+			continue
+		}
+		if kept, err = inv.OperatorExport(ctx); err != nil {
+			return nil, err
+		}
+		break
+	}
+
 	return plan.Declaration(catalogue.Rendering{
 		Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
 		Certificate: certificate, Authority: authority, Mesh: private, Names: names,
-		Foundation: foundation})
+		Foundation: foundation, Kept: kept})
 }
 
 // routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq

cmd/mesh-controller/secret.go

@@ -3,12 +3,17 @@ package main
 import (
 	"bufio"
 	"context"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"os"
 	"strings"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/secrets"
 )
 
 // secretCommand gives the mesh a value it must carry and could not have invented.
@@ -28,8 +33,17 @@ import (
 // The value is sealed on the way in and the plaintext discarded, exactly as a generated one is.
 // **The only difference between the two is where the value came from.**
 func secretCommand(ctx context.Context, args []string) error {
-	if len(args) == 0 || args[0] != "accept" {
-		return errors.New("secret accept <node> <module> <name> [--from <file>]")
+	if len(args) == 0 {
+		return errors.New(secretUsage)
+	}
+	switch args[0] {
+	case "accept":
+	case "recover":
+		return secretRecover(ctx, args[1:])
+	case "export":
+		return secretExport(ctx, args[1:])
+	default:
+		return errors.New(secretUsage)
 	}
 	rest, flags := split(args[1:])
 	set := flag.NewFlagSet("secret accept", flag.ContinueOnError)
@@ -39,7 +53,7 @@ func secretCommand(ctx context.Context, args []string) error {
 		return err
 	}
 	if len(rest) != 3 {
-		return errors.New("secret accept <node> <module> <name> [--from <file>]")
+		return errors.New(secretUsage)
 	}
 	node, module, name := rest[0], rest[1], rest[2]
 
@@ -69,6 +83,198 @@ func secretCommand(ctx context.Context, args []string) error {
 	return nil
 }
 
+const secretUsage = "secret accept <node> <module> <name> [--from <file>]\n" +
+	"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
+	"secret export [--out <file>]"
+
+// secretRecover is break-glass: a secret opened with the operator's key, written to a file.
+//
+// **The mesh cannot show a secret back, and this does not make it able to.** What is opened here
+// is the copy sealed to the operator key (novox/hq ADR 0085, amended); the mesh holds that blob and
+// no key for it, and this program holds the key for the length of the call and no blob until given
+// one. Recovery needs both, which is what keeps the sealing meaningful.
+//
+// The value goes to a file at 0600, never to the terminal unless asked for with `--out -` — the
+// source mesh's secret tools were written after a secret was printed into a transcript, and that
+// rule is theirs. `--from-export` reads the blob from a file `secret export` wrote, so recovery
+// works with the store gone, which is the case it exists for.
+func secretRecover(ctx context.Context, args []string) error {
+	rest, flags := split(args)
+	set := flag.NewFlagSet("secret recover", flag.ContinueOnError)
+	keyFile := set.String("key", "", "the operator's private key, from `operator key make`")
+	out := set.String("out", "", "where to write the value (0600); - for standard output. Default <node>.<module>.<name>.secret")
+	fromExport := set.String("from-export", "", "read the sealed copy from this `secret export` file instead of the store")
+	provider := set.String("provider", "", "for a pair credential held from more than one provider: which one")
+	if err := set.Parse(flags); err != nil {
+		return err
+	}
+	if len(rest) != 3 {
+		return errors.New(secretUsage)
+	}
+	node, module, name := rest[0], rest[1], rest[2]
+	private, err := readPrivateKey(*keyFile)
+	if err != nil {
+		return err
+	}
+
+	var kept inventory.Kept
+	if *fromExport != "" {
+		kept, err = keptFromExport(*fromExport, node, module, name, *provider)
+		if err != nil {
+			return err
+		}
+	} else {
+		open, err := openStores(ctx)
+		if err != nil {
+			return err
+		}
+		defer open.Close()
+		kept, err = open.inventory.KeptSecret(ctx, node, module, name, *provider)
+		if err != nil {
+			return err
+		}
+	}
+
+	value, err := secrets.Open(private, kept.Sealed)
+	if err != nil {
+		return fmt.Errorf("%s on %s: %q is sealed to operator key %s, and that key does not open it: %w",
+			module, node, name, secrets.Fingerprint(kept.Key), err)
+	}
+	if *out == "-" {
+		_, err := os.Stdout.Write(append(value, '\n'))
+		return err
+	}
+	path := *out
+	if path == "" {
+		path = node + "." + module + "." + name + ".secret"
+	}
+	if err := writeNew(path, value); err != nil {
+		return err
+	}
+	fmt.Printf("%s on %s: %q recovered to %s (0600) — %d bytes, origin %s\n",
+		module, node, name, path, len(value), kept.Origin)
+	return nil
+}
+
+// An export is what a person keeps beside the operator key — the catalogue's shape, so the vault
+// keeps the same document on its disk (Manifest.Keeps).
+type export = catalogue.KeptExport
+
+func secretExport(ctx context.Context, args []string) error {
+	set := flag.NewFlagSet("secret export", flag.ContinueOnError)
+	out := set.String("out", "", "where to write the export (0600); - or empty for standard output")
+	if err := set.Parse(args); err != nil {
+		return err
+	}
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+	key, err := inv.OperatorKey(ctx)
+	if err != nil {
+		return err
+	}
+	if key == "" {
+		return errors.New("the mesh has no operator key, so nothing is sealed to one; `operator key make` and `operator key set` first")
+	}
+	doc, err := inv.OperatorExport(ctx)
+	if err != nil {
+		return err
+	}
+	body, err := json.MarshalIndent(doc, "", "  ")
+	if err != nil {
+		return err
+	}
+	body = append(body, '\n')
+	if *out == "" || *out == "-" {
+		_, err := os.Stdout.Write(body)
+		return err
+	}
+	// Replaced whole, and made 0600 whether or not it existed: an export is ciphertext and a public
+	// key, but it is also the list of every secret the mesh has, and a file left at an earlier mode
+	// while the command says 0600 is a lie in the one place a person checks.
+	if err := writeReplacing(*out, body); err != nil {
+		return err
+	}
+	fmt.Printf("%d secret(s) exported to %s (0600), sealed to operator key %s — ciphertext, keep it with the key\n",
+		len(doc.Kept), *out, doc.Fingerprint)
+	if len(doc.EarlierKey) > 0 {
+		fmt.Printf("  %d secret(s) are sealed to an EARLIER operator key: recoverable with that key only\n", len(doc.EarlierKey))
+	}
+	if len(doc.Unrecoverable) > 0 {
+		fmt.Printf("  %d secret(s) are NOT in it: made before the mesh had an operator key\n", len(doc.Unrecoverable))
+	}
+	return nil
+}
+
+// writeNew writes a file that must not exist yet, atomically: create-exclusive, 0600. A check
+// followed by a write is a window in which a key somebody still needs can be overwritten.
+func writeNew(path string, content []byte) error {
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+	if err != nil {
+		if os.IsExist(err) {
+			return fmt.Errorf("%s already exists; not overwriting it", path)
+		}
+		return err
+	}
+	if _, err := f.Write(content); err != nil {
+		f.Close()
+		return err
+	}
+	return f.Close()
+}
+
+// writeReplacing writes a file whole, creating or truncating it, and leaves it at 0600 either way.
+func writeReplacing(path string, content []byte) error {
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
+	if err != nil {
+		return err
+	}
+	if _, err := f.Write(content); err != nil {
+		f.Close()
+		return err
+	}
+	if err := f.Chmod(0o600); err != nil {
+		f.Close()
+		return err
+	}
+	return f.Close()
+}
+
+func keptFromExport(path, node, module, name, provider string) (inventory.Kept, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return inventory.Kept{}, err
+	}
+	var e export
+	if err := json.Unmarshal(raw, &e); err != nil {
+		return inventory.Kept{}, fmt.Errorf("%s is not a secret export: %w", path, err)
+	}
+	// Sealed to the current key or to an earlier one: both are copies the given key might open,
+	// and Open says which. Not the unrecoverable list, which holds no copy at all.
+	var found []inventory.Kept
+	for _, k := range append(append([]inventory.Kept{}, e.Kept...), e.EarlierKey...) {
+		if k.Node == node && k.Module == module && k.Name == name && (provider == "" || k.Provider == provider) {
+			found = append(found, k)
+		}
+	}
+	switch len(found) {
+	case 0:
+		return inventory.Kept{}, fmt.Errorf("%s holds no copy of %s's %q on %s", path, module, name, node)
+	case 1:
+		return found[0], nil
+	default:
+		providers := make([]string, 0, len(found))
+		for _, f := range found {
+			providers = append(providers, f.Provider)
+		}
+		return inventory.Kept{}, fmt.Errorf("%s holds %s's %q on %s from more than one provider (%s); say which with --provider",
+			path, module, name, node, strings.Join(providers, ", "))
+	}
+}
+
 // split separates what this command is about from how it was asked.
 //
 // **Because the standard library stops parsing at the first non-flag argument.** With the

internal/catalogue/declaration.go

@@ -14,6 +14,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 )
 
 // SettingsBy is the layers that apply to each module, keyed by module name.
@@ -89,6 +90,10 @@ type Rendering struct {
 	// a fact about the mesh, and resolution answers questions about one machine.
 	Mesh []string
 
+	// Kept is every operator-sealed secret in the mesh, for a module that `keeps` them. Nil when
+	// nothing on this node keeps them, or the mesh has no operator key.
+	Kept *KeptExport
+
 	// Foundation is the ports the mesh itself needs reachable on every machine, which no module
 	// declares because the foundation is not a module (novox/hq 04-ISSUES/051 and 052). The broker
 	// is the one that matters: a machine dials it to enrol, and a firewall derived only from
@@ -384,6 +389,13 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
 			}
 			first = append(first, file)
 		}
+		if m.Keeps != "" && with.Kept != nil {
+			file, err := keptFile(m.Keeps, with.Kept)
+			if err != nil {
+				return nil, err
+			}
+			first = append(first, file)
+		}
 		if m.Computed != "" {
 			generator, known := with.Generators[m.Computed]
 			if !known {
@@ -725,6 +737,58 @@ func receivedFile(requirement, path string, given []Contribution) (map[string]an
 	}, nil
 }
 
+// Kept is one secret as the operator can recover it: where it belongs, and the value sealed to the
+// operator's key. Never a node's blob, and never a value.
+//
+// Two kinds, addressed the same way — by the node and module that hold it and the name they know
+// it by. An `own` secret is one a module has for itself; a `pair` secret is a credential the
+// module was granted for a provision it requires (`name` is the provision), and Provider says which
+// node grants it.
+type Kept struct {
+	Node   string `json:"node"`
+	Module string `json:"module"`
+	Name   string `json:"name"`
+	// Kind is `own` or `pair`.
+	Kind string `json:"kind"`
+	// Provider is the node granting a pair credential; empty for an own secret.
+	Provider string `json:"provider,omitempty"`
+	// Origin is `made` or `accepted` — whether the mesh minted it or a person supplied it.
+	Origin string `json:"origin"`
+	// Sealed is the value, sealed to the operator key named in Key.
+	Sealed string    `json:"sealed"`
+	Key    string    `json:"key"`
+	MadeAt time.Time `json:"made-at"`
+}
+
+// KeptExport is what a person keeps beside the operator key, and what a vault keeps on its disk:
+// every operator-sealed copy, and the honest list of what has none.
+type KeptExport struct {
+	Export      int    `json:"export"`
+	OperatorKey string `json:"operator-key"`
+	Fingerprint string `json:"fingerprint"`
+	// Kept is sealed to OperatorKey. EarlierKey is sealed to a key the mesh has since replaced —
+	// recoverable with that key, if the person still has it, and with nothing else. Unrecoverable
+	// has no operator copy at all.
+	Kept          []Kept `json:"kept"`
+	EarlierKey    []Kept `json:"sealed-to-earlier-key,omitempty"`
+	Unrecoverable []Kept `json:"unrecoverable,omitempty"`
+}
+
+// KeptID is the resource that carries the export to a module that keeps it.
+func KeptID() string { return "kept" }
+
+// keptFile is the export, written where the module said. Ciphertext throughout — see Keeps.
+func keptFile(dir string, kept *KeptExport) (map[string]any, error) {
+	body, err := json.MarshalIndent(kept, "", "  ")
+	if err != nil {
+		return nil, err
+	}
+	return map[string]any{
+		"id": KeptID(), "type": "file", "path": strings.TrimRight(dir, "/") + "/export.json",
+		"mode": "0600", "content": string(body) + "\n",
+	}, nil
+}
+
 // sortedKeys is map iteration made repeatable, which everything written to a machine needs.
 func sortedKeys[V any](m map[string]V) []string {
 	out := make([]string, 0, len(m))

internal/catalogue/kept_test.go

@@ -0,0 +1,68 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// A module that keeps the operator-sealed secrets is handed the export as a file, at 0600, and a
+// module that does not keep them is handed nothing — the export goes to the vault and nowhere else.
+func TestOnlyAModuleThatKeepsGetsTheExport(t *testing.T) {
+	vault := Manifest{Module: "mesh-vault", Version: "1", Provides: FromAnywhere("secret"),
+		Keeps: "/var/lib/mesh-vault/root"}
+	other := Manifest{Module: "zsh", Version: "1", Provides: Offers("shell")}
+	got, err := Resolve(shelf(vault, other), []string{"mesh-vault", "zsh"},
+		Node{Name: "anchor", At: "10.0.0.1", Capabilities: map[string]bool{}}, World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	kept := &KeptExport{Export: 1, OperatorKey: "OPERATOR", Fingerprint: "sha256:abcd",
+		Kept: []Kept{{Node: "anchor", Module: "postgres", Name: "superuser", Origin: "accepted", Sealed: "CIPHERTEXT", Key: "OPERATOR"}}}
+	out, err := got.Declaration(Rendering{Kept: kept})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var files int
+	for _, r := range out {
+		if r["path"] != "/var/lib/mesh-vault/root/export.json" {
+			continue
+		}
+		files++
+		if r["mode"] != "0600" {
+			t.Errorf("the export is written at mode %v, and it is the mesh's root secrets, sealed or not", r["mode"])
+		}
+		content, _ := r["content"].(string)
+		for _, want := range []string{`"operator-key": "OPERATOR"`, `"sealed": "CIPHERTEXT"`, `"module": "postgres"`} {
+			if !strings.Contains(content, want) {
+				t.Errorf("the export lacks %s:\n%s", want, content)
+			}
+		}
+		if id, _ := r["id"].(string); !strings.Contains(id, "mesh-vault") {
+			t.Errorf("the export's resource id %q does not carry its module", id)
+		}
+	}
+	if files != 1 {
+		t.Fatalf("%d export files; one module keeps them", files)
+	}
+
+	// No operator key yet: the vault is declared without the file, not with an empty one.
+	out, err = got.Declaration(Rendering{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range out {
+		if r["path"] == "/var/lib/mesh-vault/root/export.json" {
+			t.Fatal("an export was written with nothing to export")
+		}
+	}
+}
+
+func TestKeepsMustBeAnAbsolutePath(t *testing.T) {
+	_, err := ParseManifest([]byte(`{"module":"mesh-vault","version":"1","keeps":"root"}`))
+	if err == nil || !strings.Contains(err.Error(), "keeps") {
+		t.Fatalf("a relative keeps path was not refused: %v", err)
+	}
+	if _, err := ParseManifest([]byte(`{"module":"mesh-vault","version":"1","keeps":"/var/lib/mesh-vault/root"}`)); err != nil {
+		t.Fatalf("an absolute keeps path was refused: %v", err)
+	}
+}

internal/catalogue/manifest.go

@@ -297,6 +297,17 @@ type Manifest struct {
 	// module, in a file anybody can read, for ever.
 	OwnSecrets map[string]string `json:"own-secrets,omitempty"`
 
+	// Keeps is where this module wants every operator-sealed secret in the mesh written — the
+	// vault's field, and so far nobody else's (novox/hq ADR 0085, amended).
+	//
+	// A directory. The mesh writes one file into it, `export.json`: every secret a module holds for
+	// itself, sealed to the operator's key, with the key's public half and the list of what is NOT
+	// in it. Ciphertext to the machine that holds it and to everything on the bus it crossed —
+	// only the operator, holding the private half off the mesh, can open a line of it. That is
+	// what lets a vault's disk stand in for the store when the store is gone: recovery needs the
+	// export and the key, and the mesh holds neither in a form it can use.
+	Keeps string `json:"keeps,omitempty"`
+
 	// Listens is what this module accepts connections on, and from where.
 	//
 	// **A rule names its source** ([ADR 0007](novox/hq)). A port with no source is open to
@@ -918,6 +929,10 @@ func ParseManifest(raw []byte) (Manifest, error) {
 				"%s grants %q to its consumers and does not provide it", m.Module, to))
 		}
 	}
+	if m.Keeps != "" && !strings.HasPrefix(m.Keeps, "/") {
+		problems = append(problems, fmt.Sprintf(
+			"%s keeps the operator-sealed secrets at %q, which is not an absolute path", m.Module, m.Keeps))
+	}
 	for to, where := range m.Receives {
 		if !name.MatchString(to) {
 			problems = append(problems, fmt.Sprintf("%q is not a usable name to receive", to))

internal/inventory/migrations/0023-an-operator-key-and-a-second-seal.sql

@@ -0,0 +1,22 @@
+-- The operator's sealing key, and a second seal on every secret a module holds for itself.
+--
+-- Every secret here is sealed to the node that will use it and to nothing else, so a node whose key
+-- is gone takes its secrets with it -- and the mesh's own root secrets, the store's superuser and
+-- the broker's administrator among them, are exactly such secrets. novox/hq ADR 0085 (amended)
+-- gives them a second holder: a person, with a key whose private half never enters the mesh. What
+-- the mesh keeps is one more blob it cannot open.
+
+-- At most one operator key at a time. A row rather than a setting, because it is a fact about the
+-- mesh with consequences (what can be recovered), not somebody's preference about a module.
+create table operator_key (
+    public  text        not null primary key,
+    made_at timestamptz not null default now()
+);
+
+alter table module_secret
+    -- The same value, sealed to the operator key -- null for a secret minted before there was
+    -- one, which cannot be sealed after the fact: the plaintext was discarded. Such a secret is
+    -- recoverable only once it is issued again.
+    add column operator_sealed text,
+    -- Which operator key, so a replaced key can be told what it can no longer open.
+    add column operator_key    text;

internal/inventory/migrations/0024-a-pair-credential-is-sealed-to-the-operator-too.sql

@@ -0,0 +1,10 @@
+-- The same second seal for a pair credential (novox/hq ADR 0085, amended).
+--
+-- 0023 gave a module's own secrets an operator-sealed copy. A secret the vault provides to a module
+-- is not an own secret -- it is the credential of the consumer-vault pair -- and so were the
+-- credentials every provider grants. Without this, a vault-provided password could be rotated and
+-- audited but not recovered, which is a vault that keeps everything except what it was for.
+
+alter table secret
+    add column operator_sealed text,
+    add column operator_key    text;

internal/inventory/operator.go

@@ -0,0 +1,219 @@
+package inventory
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/jackc/pgx/v5"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/secrets"
+)
+
+// The operator's sealing key: the one holder of secrets that is not a node.
+//
+// Every secret a module holds for itself is sealed to the node that uses it, and a node whose key
+// is gone takes its secrets with it — the mesh's root secrets included. novox/hq ADR 0085 (amended)
+// gives them a second recipient: a person, holding a key whose private half never enters the mesh.
+// What is recorded here is the public half, which is all the mesh needs to seal to it; what it
+// yields is one more blob per secret that the mesh cannot open.
+
+// operatorColumns is the pair of nullable columns a secret row carries for its operator copy:
+// both null when the mesh has no operator key, so a row says plainly that no such copy exists.
+func operatorColumns(operator, blob string) (sealed, key *string) {
+	if operator == "" || blob == "" {
+		return nil, nil
+	}
+	return &blob, &operator
+}
+
+// operatorSeal seals a value somebody supplied to the operator key, when the mesh has one.
+func (i *Inventory) operatorSeal(ctx context.Context, value string) (sealed, key *string, err error) {
+	operator, err := i.OperatorKey(ctx)
+	if err != nil || operator == "" {
+		return nil, nil, err
+	}
+	blob, err := secrets.Seal(operator, []byte(value))
+	if err != nil {
+		return nil, nil, err
+	}
+	sealed, key = operatorColumns(operator, blob)
+	return sealed, key, nil
+}
+
+// OperatorKey is the public key secrets are also sealed to, or empty when the mesh has none.
+func (i *Inventory) OperatorKey(ctx context.Context) (string, error) {
+	var key string
+	err := i.store.Pool().QueryRow(ctx,
+		`select public from operator_key order by made_at desc limit 1`).Scan(&key)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return "", nil
+	}
+	return key, err
+}
+
+// SetOperatorKey records the operator's public key, replacing any earlier one.
+//
+// **Replacing is said, not silent.** Secrets sealed to the earlier key stay sealed to it: the
+// plaintext is gone, so they cannot be sealed again to the new one until each is issued again. The
+// number of them is returned so the caller can say so — a key swapped with nothing said would look
+// like a mesh with a recovery path and be a mesh without one.
+func (i *Inventory) SetOperatorKey(ctx context.Context, public string) (orphaned int, err error) {
+	if public == "" {
+		return 0, fmt.Errorf("an operator key is a public key, and this is nothing")
+	}
+	tx, err := i.store.Pool().Begin(ctx)
+	if err != nil {
+		return 0, err
+	}
+	defer tx.Rollback(ctx)
+	// Both tables: a module's own secrets and the pair credentials. A count over one of them said
+	// "nothing orphaned" about a mesh whose every vault-provided secret had just been.
+	if err := tx.QueryRow(ctx,
+		`select (select count(*) from module_secret where operator_key is not null and operator_key <> $1)
+		      + (select count(*) from secret        where operator_key is not null and operator_key <> $1)`,
+		public).Scan(&orphaned); err != nil {
+		return 0, err
+	}
+	if _, err := tx.Exec(ctx, `delete from operator_key where public <> $1`, public); err != nil {
+		return 0, err
+	}
+	if _, err := tx.Exec(ctx,
+		`insert into operator_key (public) values ($1) on conflict (public) do nothing`, public); err != nil {
+		return 0, err
+	}
+	return orphaned, tx.Commit(ctx)
+}
+
+// Kept is the catalogue's: one secret as the operator can recover it.
+type Kept = catalogue.Kept
+
+// OperatorExport is the export as the operator and the vault both keep it: every secret sealed to
+// the mesh's current operator key, every one sealed to an earlier key (recoverable with that key,
+// if the person still has it), and every one with no operator copy at all. Nil when the mesh has
+// no operator key. One constructor, so the file `secret export` writes and the file the mesh puts
+// on the vault's disk cannot drift apart.
+func (i *Inventory) OperatorExport(ctx context.Context) (*catalogue.KeptExport, error) {
+	operator, err := i.OperatorKey(ctx)
+	if err != nil || operator == "" {
+		return nil, err
+	}
+	kept, earlier, unrecoverable, err := i.KeptForOperator(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return &catalogue.KeptExport{
+		Export: 1, OperatorKey: operator, Fingerprint: secrets.Fingerprint(operator),
+		Kept: kept, EarlierKey: earlier, Unrecoverable: unrecoverable,
+	}, nil
+}
+
+// KeptForOperator is every secret by what can open it: the mesh's current operator key, an
+// earlier operator key, or nothing.
+//
+// The last two are the honest half. A secret minted before the mesh had an operator key has no
+// operator-sealed copy and cannot get one — the plaintext was discarded; one sealed to a key the
+// mesh has since replaced is not opened by the current key, however the export is labelled. Naming
+// both is what lets an export say what it does not cover, rather than being taken for complete.
+func (i *Inventory) KeptForOperator(ctx context.Context) (kept, earlier, unrecoverable []Kept, err error) {
+	current, err := i.OperatorKey(ctx)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	rows, err := i.store.Pool().Query(ctx,
+		`select 'own', n.name, s.module, s.name, '', s.origin, coalesce(s.operator_sealed, ''),
+		        coalesce(s.operator_key, ''), s.made_at
+		   from module_secret s join node n on n.id = s.node
+		 union all
+		 select 'pair', c.name, s.consumer_module, s.name, p.name, 'made', coalesce(s.operator_sealed, ''),
+		        coalesce(s.operator_key, ''), s.created_at
+		   from secret s join node c on c.id = s.consumer join node p on p.id = s.provider
+		  order by 1, 2, 3, 4`)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var k Kept
+		if err := rows.Scan(&k.Kind, &k.Node, &k.Module, &k.Name, &k.Provider, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt); err != nil {
+			return nil, nil, nil, err
+		}
+		switch {
+		case k.Sealed == "":
+			unrecoverable = append(unrecoverable, k)
+		case k.Key != current:
+			earlier = append(earlier, k)
+		default:
+			kept = append(kept, k)
+		}
+	}
+	return kept, earlier, unrecoverable, rows.Err()
+}
+
+// KeptSecret is one secret's operator-sealed copy, for recovery.
+//
+// An own secret first, then a pair credential by the provision's name — a module whose own secret
+// and requirement share a name is refused at resolution, so the two cannot both answer. A pair
+// credential is keyed by provider as well, and a consumer whose provision moved leaves the old
+// provider's row behind: two rows is refused with both providers named, never answered with
+// whichever came first, unless `provider` says which.
+func (i *Inventory) KeptSecret(ctx context.Context, node, module, name, provider string) (Kept, error) {
+	var k Kept
+	err := i.store.Pool().QueryRow(ctx,
+		`select 'own', n.name, s.module, s.name, '', s.origin, coalesce(s.operator_sealed, ''),
+		        coalesce(s.operator_key, ''), s.made_at
+		   from module_secret s join node n on n.id = s.node
+		  where n.name = $1 and s.module = $2 and s.name = $3`, node, module, name).
+		Scan(&k.Kind, &k.Node, &k.Module, &k.Name, &k.Provider, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt)
+	if errors.Is(err, pgx.ErrNoRows) {
+		rows, qerr := i.store.Pool().Query(ctx,
+			`select 'pair', c.name, s.consumer_module, s.name, p.name, 'made', coalesce(s.operator_sealed, ''),
+			        coalesce(s.operator_key, ''), s.created_at
+			   from secret s join node c on c.id = s.consumer join node p on p.id = s.provider
+			  where c.name = $1 and s.consumer_module = $2 and s.name = $3 and ($4 = '' or p.name = $4)
+			  order by p.name`, node, module, name, provider)
+		if qerr != nil {
+			return Kept{}, qerr
+		}
+		defer rows.Close()
+		var found []Kept
+		for rows.Next() {
+			var row Kept
+			if err := rows.Scan(&row.Kind, &row.Node, &row.Module, &row.Name, &row.Provider, &row.Origin, &row.Sealed, &row.Key, &row.MadeAt); err != nil {
+				return Kept{}, err
+			}
+			found = append(found, row)
+		}
+		if err := rows.Err(); err != nil {
+			return Kept{}, err
+		}
+		switch len(found) {
+		case 0:
+			err = pgx.ErrNoRows
+		case 1:
+			k, err = found[0], nil
+		default:
+			providers := make([]string, 0, len(found))
+			for _, f := range found {
+				providers = append(providers, f.Provider)
+			}
+			return Kept{}, fmt.Errorf("%s on %s holds a %q credential from more than one provider (%s); say which with --provider",
+				module, node, name, strings.Join(providers, ", "))
+		}
+	}
+	if errors.Is(err, pgx.ErrNoRows) {
+		return Kept{}, fmt.Errorf("%s on %s holds nothing called %q — neither a secret of its own nor a credential for a provision", module, node, name)
+	}
+	if err != nil {
+		return Kept{}, err
+	}
+	if k.Sealed == "" {
+		return Kept{}, fmt.Errorf(
+			"%s on %s holds %q, but it was made before the mesh had an operator key and so has no "+
+				"copy a person can open. Issue it again (secret accept, or let the mesh remake it) "+
+				"and it will", module, node, name)
+	}
+	return k, nil
+}

internal/inventory/operator_test.go

@@ -0,0 +1,219 @@
+package inventory
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/secrets"
+)
+
+// With an operator key, a module's own secret is sealed to the operator as well — and the operator
+// opens exactly the value the node was given.
+func TestAnOwnSecretIsSealedToTheOperatorToo(t *testing.T) {
+	inv, ctx := twoNodesWithKeys(t)
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"}, Source{}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Before there is a key: minted, and honestly unrecoverable.
+	if _, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.KeptSecret(ctx, "consumer", "postgres", "superuser", ""); err == nil {
+		t.Fatal("a secret made before the operator key was reported recoverable")
+	}
+	kept, _, unrecoverable, err := inv.KeptForOperator(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(kept) != 0 || len(unrecoverable) != 1 {
+		t.Fatalf("before a key: %d kept, %d unrecoverable", len(kept), len(unrecoverable))
+	}
+
+	pub, priv, err := secrets.Keypair()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if orphaned, err := inv.SetOperatorKey(ctx, pub); err != nil || orphaned != 0 {
+		t.Fatalf("set: orphaned %d, %v", orphaned, err)
+	}
+
+	// A new secret is sealed to both; an accepted one too.
+	if _, err := inv.SecretForModule(ctx, "provider", "postgres", "superuser"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.AcceptSecretForModule(ctx, "provider", "postgres", "replication", "given-by-a-person"); err != nil {
+		t.Fatal(err)
+	}
+	kept, _, unrecoverable, err = inv.KeptForOperator(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(kept) != 2 || len(unrecoverable) != 1 {
+		t.Fatalf("after a key: %d kept, %d unrecoverable", len(kept), len(unrecoverable))
+	}
+	got, err := inv.KeptSecret(ctx, "provider", "postgres", "replication", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	value, err := secrets.Open(priv, got.Sealed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(value) != "given-by-a-person" {
+		t.Fatalf("recovered %q", value)
+	}
+	if got.Origin != "accepted" || got.Key != pub {
+		t.Fatalf("kept as %+v", got)
+	}
+	minted, err := inv.KeptSecret(ctx, "provider", "postgres", "superuser", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v, err := secrets.Open(priv, minted.Sealed); err != nil || len(v) != 40 {
+		t.Fatalf("the minted secret did not open to a 40-character value: %v", err)
+	}
+
+	// The old secret is kept, not resealed: asking again is a read, the plaintext is gone, and it
+	// stays honestly unrecoverable.
+	if _, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.KeptSecret(ctx, "consumer", "postgres", "superuser", ""); err == nil {
+		t.Fatal("asking again did not remake, yet it became recoverable")
+	}
+	// Until the node rejoins with a new sealing key: then the secret is remade, and the remake is
+	// sealed to the operator — the one scenario the vault exists for.
+	rejoined, err := inv.NodeByName(ctx, "consumer")
+	if err != nil {
+		t.Fatal(err)
+	}
+	newKey, _ := aSealingKey(t)
+	if err := inv.RecordSealingKey(ctx, rejoined.ID, newKey); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser"); err != nil {
+		t.Fatal(err)
+	}
+	remade, err := inv.KeptSecret(ctx, "consumer", "postgres", "superuser", "")
+	if err != nil {
+		t.Fatalf("the remade secret is not recoverable: %v", err)
+	}
+	if _, err := secrets.Open(priv, remade.Sealed); err != nil {
+		t.Fatal(err)
+	}
+
+	// Replacing the key says how many secrets stay sealed to the old one — and those move out of
+	// the recoverable list, whatever the export is labelled with.
+	pub2, _, _ := secrets.Keypair()
+	orphaned, err := inv.SetOperatorKey(ctx, pub2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if orphaned != 3 {
+		t.Fatalf("replacing the key orphaned %d, and three were sealed to it", orphaned)
+	}
+	if now, _ := inv.OperatorKey(ctx); now != pub2 {
+		t.Fatal("the new key is not the mesh's key")
+	}
+	kept, earlier, _, err := inv.KeptForOperator(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(kept) != 0 || len(earlier) != 3 {
+		t.Fatalf("after replacing the key: %d recoverable with it, %d sealed to the earlier key", len(kept), len(earlier))
+	}
+	doc, err := inv.OperatorExport(ctx)
+	if err != nil || doc == nil || len(doc.EarlierKey) != 3 || len(doc.Kept) != 0 {
+		t.Fatalf("the export does not say what the current key cannot open: %+v %v", doc, err)
+	}
+}
+
+// A pair credential — what the vault provides a module — is sealed to the operator too, and the
+// operator's copy is the very value the consumer's node unseals.
+func TestAPairCredentialIsSealedToTheOperatorToo(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	// Two nodes with keys, keeping the consumer's opener: the test made the key, so it can play the
+	// consumer's host for one assertion.
+	var openAsConsumer func(string) ([]byte, bool)
+	for _, n := range []string{"consumer", "provider"} {
+		node, err := inv.AddNode(ctx, n)
+		if err != nil {
+			t.Fatal(err)
+		}
+		key, open := aSealingKey(t)
+		if n == "consumer" {
+			openAsConsumer = open
+		}
+		if err := inv.RecordSealingKey(ctx, node.ID, key); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, m := range []string{"gitea", "mesh-vault"} {
+		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	pub, priv, err := secrets.Keypair()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.SetOperatorKey(ctx, pub); err != nil {
+		t.Fatal(err)
+	}
+	made, err := inv.SecretFor(ctx, "secret", "consumer", "gitea", "provider")
+	if err != nil {
+		t.Fatal(err)
+	}
+	kept, err := inv.KeptSecret(ctx, "consumer", "gitea", "secret", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if kept.Kind != "pair" || kept.Provider != "provider" {
+		t.Fatalf("kept as %+v", kept)
+	}
+	all, _, _, err := inv.KeptForOperator(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var pairs int
+	for _, k := range all {
+		if k.Kind == "pair" {
+			pairs++
+		}
+	}
+	if pairs != 1 {
+		t.Fatalf("%d pair credential(s) recoverable, expected 1", pairs)
+	}
+
+	// A second provider of the same provision: two rows, refused rather than the first one taken,
+	// unless the provider is named. And replacing the key counts pair credentials as orphaned.
+	if _, err := inv.SecretFor(ctx, "secret", "consumer", "gitea", "consumer"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.KeptSecret(ctx, "consumer", "gitea", "secret", ""); err == nil || !strings.Contains(err.Error(), "--provider") {
+		t.Fatalf("two providers were not refused: %v", err)
+	}
+	if byName, err := inv.KeptSecret(ctx, "consumer", "gitea", "secret", "provider"); err != nil || byName.Provider != "provider" {
+		t.Fatalf("naming the provider did not select it: %+v %v", byName, err)
+	}
+	pub2, _, _ := secrets.Keypair()
+	if orphaned, err := inv.SetOperatorKey(ctx, pub2); err != nil || orphaned != 2 {
+		t.Fatalf("replacing the key orphaned %d pair credential(s), and two were sealed to it (%v)", orphaned, err)
+	}
+	fromOperator, err := secrets.Open(priv, kept.Sealed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The consumer's blob is sealed to the consumer node. Same value, two recipients.
+	fromNode, ok := openAsConsumer(made.ForConsumer)
+	if !ok {
+		t.Fatal("the consumer cannot open its own blob")
+	}
+	if string(fromOperator) != string(fromNode) {
+		t.Fatal("the operator's copy of the pair credential differs from the consumer's")
+	}
+}

internal/inventory/secrets.go

@@ -74,20 +74,28 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModul
 		return held, nil
 	}
 
-	made, err := secrets.Make(consumerKey, providerKey)
+	// And to the operator, when the mesh has one (novox/hq ADR 0085, amended): the third copy that
+	// makes a vault-provided secret recoverable, and nothing the mesh can open.
+	operator, err := i.OperatorKey(ctx)
 	if err != nil {
 		return Secret{}, err
 	}
+	made, blob, err := secrets.MakeWithOperator(consumerKey, providerKey, operator)
+	if err != nil {
+		return Secret{}, err
+	}
+	forOperator, operatorKey := operatorColumns(operator, blob)
 	_, err = i.store.Pool().Exec(ctx,
 		`insert into secret (name, consumer, consumer_module, provider, for_consumer, for_provider,
-		                     consumer_key, provider_key)
-		 values ($1, $2, $3, $4, $5, $6, $7, $8)
+		                     consumer_key, provider_key, operator_sealed, operator_key)
+		 values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
 		 on conflict (name, consumer, consumer_module, provider) do update set
 		   for_consumer = excluded.for_consumer, for_provider = excluded.for_provider,
 		   consumer_key = excluded.consumer_key, provider_key = excluded.provider_key,
-		   created_at = now()`,
+		   created_at = now(),
+		   operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
 		name, consumerNode.ID, consumerModule, providerNode.ID,
-		made.ForConsumer, made.ForProvider, made.ConsumerKey, made.ProviderKey)
+		made.ForConsumer, made.ForProvider, made.ConsumerKey, made.ProviderKey, forOperator, operatorKey)
 	if err != nil {
 		return Secret{}, err
 	}
@@ -227,19 +235,26 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri
 			module, node, name, node)
 	}
 
-	made, err := secrets.Make(key, key)
+	operator, err := i.OperatorKey(ctx)
 	if err != nil {
 		return "", err
 	}
-	// Sealed once, to one recipient. Make seals to two ends because a provision has two; here
-	// both are the same machine, and only one copy is kept.
+	// Sealed once to the machine — Make seals to two ends because a provision has two; here both
+	// are the same machine, and only one copy is kept — and once more to the operator when the
+	// mesh has one (novox/hq ADR 0085, amended), which is the copy a person can recover from.
+	made, blob, err := secrets.MakeWithOperator(key, key, operator)
+	if err != nil {
+		return "", err
+	}
+	forOperator, operatorKey := operatorColumns(operator, blob)
 	if _, err := i.store.Pool().Exec(ctx,
-		`insert into module_secret (node, module, name, sealed, node_key, origin)
-		 values ($1, $2, $3, $4, $5, 'made')
+		`insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key)
+		 values ($1, $2, $3, $4, $5, 'made', $6, $7)
 		 on conflict (node, module, name) do update set
 		   sealed = excluded.sealed, node_key = excluded.node_key,
-		   origin = excluded.origin, made_at = now()`,
-		record.ID, module, name, made.ForConsumer, key); err != nil {
+		   origin = excluded.origin, made_at = now(),
+		   operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
+		record.ID, module, name, made.ForConsumer, key, forOperator, operatorKey); err != nil {
 		return "", err
 	}
 	return made.ForConsumer, nil
@@ -273,13 +288,20 @@ func (i *Inventory) AcceptSecretForModule(ctx context.Context, node, module, nam
 	if err != nil {
 		return err
 	}
+	// And to the operator, when the mesh has one: a value a person supplied is the one a person
+	// most needs to get back, since the mesh cannot make another (novox/hq ADR 0085, amended).
+	forOperator, operatorKey, err := i.operatorSeal(ctx, value)
+	if err != nil {
+		return err
+	}
 	_, err = i.store.Pool().Exec(ctx,
-		`insert into module_secret (node, module, name, sealed, node_key, origin)
-		 values ($1, $2, $3, $4, $5, 'accepted')
+		`insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key)
+		 values ($1, $2, $3, $4, $5, 'accepted', $6, $7)
 		 on conflict (node, module, name) do update set
 		   sealed = excluded.sealed, node_key = excluded.node_key,
-		   origin = excluded.origin, made_at = now()`,
-		record.ID, module, name, sealed.ForConsumer, key)
+		   origin = excluded.origin, made_at = now(),
+		   operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
+		record.ID, module, name, sealed.ForConsumer, key, forOperator, operatorKey)
 	return err
 }
 

internal/secrets/operator_test.go

@@ -0,0 +1,75 @@
+package secrets
+
+import "testing"
+
+// A secret sealed to the operator as well is opened by the operator's key and by nothing else.
+func TestAThirdRecipientOpensWithItsOwnKeyOnly(t *testing.T) {
+	nodePub, nodePriv, err := Keypair()
+	if err != nil {
+		t.Fatal(err)
+	}
+	opPub, opPriv, err := Keypair()
+	if err != nil {
+		t.Fatal(err)
+	}
+	sealed, forOperator, err := MakeWithOperator(nodePub, nodePub, opPub)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if forOperator == "" {
+		t.Fatal("no blob for the operator")
+	}
+	if _, none, err := MakeWithOperator(nodePub, nodePub, ""); err != nil || none != "" {
+		t.Fatalf("no operator key, yet a blob %q (%v)", none, err)
+	}
+	fromNode, err := Open(nodePriv, sealed.ForConsumer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fromOperator, err := Open(opPriv, forOperator)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(fromNode) != string(fromOperator) {
+		t.Fatal("the operator's copy is a different value from the node's")
+	}
+	if len(fromNode) != 40 {
+		t.Fatalf("a minted value is %d characters, not 40", len(fromNode))
+	}
+	if _, err := Open(nodePriv, forOperator); err == nil {
+		t.Fatal("the node's key opened the operator's blob")
+	}
+	if _, err := Open(opPriv, sealed.ForConsumer); err == nil {
+		t.Fatal("the operator's key opened the node's blob")
+	}
+}
+
+// An accepted value, sealed to the operator, comes back byte for byte.
+func TestAnAcceptedValueRoundTripsThroughTheOperatorKey(t *testing.T) {
+	opPub, opPriv, err := Keypair()
+	if err != nil {
+		t.Fatal(err)
+	}
+	blob, err := Seal(opPub, []byte("  the-superuser's password, spaces and all  "))
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := Open(opPriv, blob)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != "  the-superuser's password, spaces and all  " {
+		t.Fatalf("got %q", got)
+	}
+}
+
+func TestAFingerprintNamesAKeyAndIsNotOne(t *testing.T) {
+	pub, _, _ := Keypair()
+	fp := Fingerprint(pub)
+	if len(fp) != len("sha256:")+16 || fp[:7] != "sha256:" {
+		t.Fatalf("fingerprint %q", fp)
+	}
+	if fp == Fingerprint(pub+"x") {
+		t.Fatal("two keys, one fingerprint")
+	}
+}

internal/secrets/seal.go

@@ -1,8 +1,11 @@
 package secrets
 
 import (
+	"crypto/ecdh"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/base64"
+	"encoding/hex"
 	"fmt"
 	"strings"
 
@@ -48,10 +51,22 @@ type Sealed struct {
 // rather than reading the old one back — the only version of rotation that is honest about what
 // the mesh knows.
 func Make(consumerKey, providerKey string) (Sealed, error) {
+	sealed, _, err := MakeWithOperator(consumerKey, providerKey, "")
+	return sealed, err
+}
+
+// MakeWithOperator is Make with a third recipient: the same fresh value, sealed once more to the
+// operator's key, returned beside the two node blobs — or "" when the mesh has no operator key.
+//
+// The operator is the one holder that is not a node (novox/hq ADR 0085, amended): a person with a
+// key that never entered the mesh, who can recover a secret when the node cannot. The plaintext
+// still exists only inside this call; a third blob is one more thing the mesh cannot open, not one
+// more copy it can.
+func MakeWithOperator(consumerKey, providerKey, operatorKey string) (Sealed, string, error) {
 	if consumerKey == "" || providerKey == "" {
 		// Sealing to an empty key would produce a blob nobody can open, stored as though it were
 		// a working credential. The caller knows which node is which and says so.
-		return Sealed{}, fmt.Errorf("both ends need a sealing key before a secret can be made")
+		return Sealed{}, "", fmt.Errorf("both ends need a sealing key before a secret can be made")
 	}
 
 	// 30 bytes, not 32: base64url of 30 is exactly 40 characters, and 40 is the longest secret an
@@ -59,7 +74,7 @@ func Make(consumerKey, providerKey string) (Sealed, error) {
 	// "fit the tightest backend" rule ADR 0049 sets for the login, on the secret. 240 bits is ample.
 	value := make([]byte, 30)
 	if _, err := rand.Read(value); err != nil {
-		return Sealed{}, err
+		return Sealed{}, "", err
 	}
 	// Base64 without padding, because it lands in a configuration file something else parses and
 	// a password containing a newline or a quote is a support call.
@@ -67,16 +82,22 @@ func Make(consumerKey, providerKey string) (Sealed, error) {
 
 	forConsumer, err := Seal(consumerKey, []byte(password))
 	if err != nil {
-		return Sealed{}, err
+		return Sealed{}, "", err
 	}
 	forProvider, err := Seal(providerKey, []byte(password))
 	if err != nil {
-		return Sealed{}, err
+		return Sealed{}, "", err
+	}
+	var forOperator string
+	if operatorKey != "" {
+		if forOperator, err = Seal(operatorKey, []byte(password)); err != nil {
+			return Sealed{}, "", err
+		}
 	}
 	return Sealed{
 		ForConsumer: forConsumer, ForProvider: forProvider,
 		ConsumerKey: consumerKey, ProviderKey: providerKey,
-	}, nil
+	}, forOperator, nil
 }
 
 // Accept seals a value somebody supplied, rather than one the mesh made.
@@ -115,6 +136,52 @@ func Accept(value string, consumerKey, providerKey string) (Sealed, error) {
 	}, nil
 }
 
+// Open is the other half of Seal, for the one holder of a private key this program ever acts for:
+// the operator, recovering a secret with the key that never entered the mesh (novox/hq ADR 0085,
+// amended). A node opens its own blobs in the host; the controller opens nothing of a node's, and
+// cannot — it has no node's private key, which is the whole point of sealing.
+func Open(privateKey string, sealed string) ([]byte, error) {
+	private, err := base64.StdEncoding.DecodeString(strings.TrimSpace(privateKey))
+	if err != nil || len(private) != 32 {
+		return nil, fmt.Errorf("that is not a sealing key")
+	}
+	key, err := ecdh.X25519().NewPrivateKey(private)
+	if err != nil {
+		return nil, fmt.Errorf("that is not a usable sealing key: %w", err)
+	}
+	blob, err := base64.StdEncoding.DecodeString(sealed)
+	if err != nil {
+		return nil, fmt.Errorf("this is not a sealed value: %w", err)
+	}
+	var pub, priv [32]byte
+	copy(pub[:], key.PublicKey().Bytes())
+	copy(priv[:], private)
+	out, ok := box.OpenAnonymous(nil, blob, &pub, &priv)
+	if !ok {
+		return nil, fmt.Errorf("this was not sealed to that key")
+	}
+	return out, nil
+}
+
+// Keypair makes a sealing keypair for a holder outside the mesh — the operator. The private half is
+// returned to be written where the caller says and nowhere else; the public half is what the mesh
+// records and seals to.
+func Keypair() (public, private string, err error) {
+	key, err := ecdh.X25519().GenerateKey(rand.Reader)
+	if err != nil {
+		return "", "", err
+	}
+	return base64.StdEncoding.EncodeToString(key.PublicKey().Bytes()),
+		base64.StdEncoding.EncodeToString(key.Bytes()), nil
+}
+
+// Fingerprint names a public key without being one: the first bytes of its hash, so two people can
+// agree which key they mean out loud.
+func Fingerprint(publicKey string) string {
+	sum := sha256.Sum256([]byte(strings.TrimSpace(publicKey)))
+	return "sha256:" + hex.EncodeToString(sum[:8])
+}
+
 // Seal closes a value to a node's public sealing key.
 func Seal(publicKey string, value []byte) (string, error) {
 	public, err := base64.StdEncoding.DecodeString(publicKey)