An operator key, a second seal on every own secret, and the vault keeps the export

novox/hq ADR 0085, amended: the mesh's root secrets — the store's superuser,
the broker's administrator, every secret a module holds for itself — were
sealed to a node key and nothing else, so a lost node took them with it.
Now the mesh records an operator's public sealing key and seals every own
secret to it as well, minted or accepted. The private half is written once
by `operator key new` to a file the operator keeps off the mesh; the mesh
holds one more blob per secret that it cannot open.

`secret recover` opens a secret with that key, to a 0600 file, from the
store or from an export; `secret export` writes every operator-sealed copy
as ciphertext. A module that `keeps` (the vault) is handed that export as a
declared file on its own disk, so recovery survives the store.

Secrets made before the key exists have no operator copy and are said so —
the plaintext was discarded — until each is issued again.

cmd/mesh-controller/main.go

@@ -100,6 +100,8 @@ func run() error {
 		return settingsCommand(ctx, args[1:])
 	case "secret":
 		return secretCommand(ctx, args[1:])
+	case "operator":
+		return operatorCommand(ctx, args[1:])
 	case "plan":
 		return planCommand(ctx, args[1:])
 	case "push":
@@ -155,6 +157,12 @@ func usage() {
   settings clear <module> [--node <n>]      take a layer away
   secret accept <node> <module> <name>     carry a value the mesh did not make and cannot invent
   secret accept ... --from <file>           ...read it from a file rather than being asked
+  secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>]
+                                            break-glass: open the operator-sealed copy, to a 0600 file
+  secret export [--out <file>]              every operator-sealed copy, ciphertext — keep it with the key
+  operator key make [--out <file>]          make the operator's sealing key, off the mesh; private half to the file only
+  operator key set <public> [--replace]     tell the mesh which operator key to seal to
+  operator key show                         the operator key, and what it can recover
   build <repository> [--ref R]   have a build machine build it, and record what came out
   build --behind                 build every module the mesh holds older than its source
   builds [<module>]              what has been built lately, and what came of it

cmd/mesh-controller/operator.go

@@ -0,0 +1,157 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/novox/mesh-controller/internal/secrets"
+)
+
+// operatorCommand is the mesh's one holder of secrets that is not a machine.
+//
+// **Every secret a module holds for itself is sealed to the node that uses it, and a node whose key
+// is gone takes its secrets with it** — the store's superuser and the broker's administrator among
+// them. novox/hq ADR 0085 (amended) gives them a second recipient: a person, with a sealing key
+// whose private half is made where the operator is and never enters the mesh. Making the key and
+// telling the mesh about it are two commands, on purpose: the first needs no mesh at all and runs
+// wherever the operator keeps things; the second gives the mesh the public half and nothing else.
+// The controller's own container is a scratch image with no writable path, which is the right
+// shape for a program that must hold no key — so the private half could not be written there
+// even by mistake.
+//
+//	operator key make [--out <file>]    make a keypair: private half to the file, public half printed
+//	operator key set <public>           tell the mesh which key to seal to
+//	operator key show                   the public key, its fingerprint, and what it can recover
+const operatorUsage = "operator key make [--out <file>] | operator key set <public> [--replace] | operator key show"
+
+func operatorCommand(ctx context.Context, args []string) error {
+	if len(args) < 2 || args[0] != "key" {
+		return errors.New(operatorUsage)
+	}
+	switch args[1] {
+	case "make":
+		return operatorKeyMake(args[2:])
+	case "set":
+		return operatorKeySet(ctx, args[2:])
+	case "show":
+		return operatorKeyShow(ctx)
+	default:
+		return errors.New(operatorUsage)
+	}
+}
+
+// operatorKeyMake needs no mesh: it is the operator's, run wherever the key will live.
+func operatorKeyMake(args []string) error {
+	set := flag.NewFlagSet("operator key make", flag.ContinueOnError)
+	out := set.String("out", "operator.key",
+		"where to write the private key (0600); keep it off the mesh, and keep it")
+	if err := set.Parse(args); err != nil {
+		return err
+	}
+	if _, err := os.Stat(*out); err == nil {
+		return fmt.Errorf("%s already exists; this will not overwrite a key somebody may still need", *out)
+	}
+	public, private, err := secrets.Keypair()
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(*out, []byte(private+"\n"), 0o600); err != nil {
+		return err
+	}
+	fmt.Printf("operator key %s\n", secrets.Fingerprint(public))
+	fmt.Printf("  private half written to %s (0600) — keep it off the mesh, and keep it\n", *out)
+	fmt.Printf("  public half, to give the mesh with `operator key set`:\n")
+	fmt.Printf("public %s\n", public)
+	return nil
+}
+
+func operatorKeySet(ctx context.Context, args []string) error {
+	rest, flags := split(args)
+	set := flag.NewFlagSet("operator key set", flag.ContinueOnError)
+	replace := set.Bool("replace", false,
+		"replace an existing operator key — secrets sealed to the old one stay sealed to it")
+	if err := set.Parse(flags); err != nil {
+		return err
+	}
+	if len(rest) != 1 {
+		return errors.New(operatorUsage)
+	}
+	public := strings.TrimSpace(rest[0])
+	if _, err := secrets.Seal(public, []byte("probe")); err != nil {
+		return fmt.Errorf("that is not a public sealing key: %w", err)
+	}
+
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+
+	if current, err := inv.OperatorKey(ctx); err != nil {
+		return err
+	} else if current != "" && current != public && !*replace {
+		return fmt.Errorf(
+			"the mesh already has an operator key (%s). Pass --replace to change it — "+
+				"secrets sealed to the current key stay sealed to it until each is issued again",
+			secrets.Fingerprint(current))
+	}
+	orphaned, err := inv.SetOperatorKey(ctx, public)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("operator key %s\n", secrets.Fingerprint(public))
+	fmt.Printf("  the mesh holds the public half only and cannot open what it seals to it;\n")
+	fmt.Printf("  from now on every secret a module holds for itself is sealed to it as well.\n")
+	fmt.Printf("  Secrets made before this cannot be — each is recoverable once issued again\n")
+	if orphaned > 0 {
+		fmt.Printf("  %d secret(s) are sealed to the previous key and stay so until issued again\n", orphaned)
+	}
+	return nil
+}
+
+func operatorKeyShow(ctx context.Context) error {
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+	key, err := inv.OperatorKey(ctx)
+	if err != nil {
+		return err
+	}
+	if key == "" {
+		fmt.Println("the mesh has no operator key; `operator key make` then `operator key set` gives it one")
+		return nil
+	}
+	kept, unrecoverable, err := inv.KeptForOperator(ctx)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("operator key %s\n  %s\n", secrets.Fingerprint(key), key)
+	fmt.Printf("  %d secret(s) recoverable with it\n", len(kept))
+	if len(unrecoverable) > 0 {
+		fmt.Printf("  %d secret(s) not recoverable — made before it, or sealed to an earlier key:\n", len(unrecoverable))
+		for _, k := range unrecoverable {
+			fmt.Printf("    %s  %s  %s\n", k.Node, k.Module, k.Name)
+		}
+	}
+	return nil
+}
+
+// readPrivateKey is the operator's key from the file `operator key make` wrote.
+func readPrivateKey(path string) (string, error) {
+	if path == "" {
+		return "", errors.New("--key <file> names the operator's private key, written by `operator key make`")
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(raw)), nil
+}

cmd/mesh-controller/plan.go

@@ -13,6 +13,7 @@ import (
 	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/inventory"
 	"github.com/novox/mesh-controller/internal/licences"
+	"github.com/novox/mesh-controller/internal/secrets"
 	"net"
 	"strconv"
 )
@@ -462,10 +463,35 @@ func declarationWith(ctx context.Context, open *stores, node string,
 		}
 	}
 
+	// And, for a module that keeps them, every operator-sealed secret in the mesh — the vault's
+	// copy, outside the store (novox/hq ADR 0085, amended). Read only; nothing here mints.
+	var kept *catalogue.KeptExport
+	for _, m := range plan.Modules {
+		if m.Keeps == "" {
+			continue
+		}
+		operator, err := inv.OperatorKey(ctx)
+		if err != nil {
+			return nil, err
+		}
+		if operator == "" {
+			break // nothing is sealed to an operator, so there is nothing to keep yet
+		}
+		recoverable, unrecoverable, err := inv.KeptForOperator(ctx)
+		if err != nil {
+			return nil, err
+		}
+		kept = &catalogue.KeptExport{
+			Export: 1, OperatorKey: operator, Fingerprint: secrets.Fingerprint(operator),
+			Kept: recoverable, Unrecoverable: unrecoverable,
+		}
+		break
+	}
+
 	return plan.Declaration(catalogue.Rendering{
 		Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
 		Certificate: certificate, Authority: authority, Mesh: private, Names: names,
-		Foundation: foundation})
+		Foundation: foundation, Kept: kept})
 }
 
 // routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq

cmd/mesh-controller/secret.go

@@ -3,12 +3,17 @@ package main
 import (
 	"bufio"
 	"context"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"os"
 	"strings"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/secrets"
 )
 
 // secretCommand gives the mesh a value it must carry and could not have invented.
@@ -28,8 +33,17 @@ import (
 // The value is sealed on the way in and the plaintext discarded, exactly as a generated one is.
 // **The only difference between the two is where the value came from.**
 func secretCommand(ctx context.Context, args []string) error {
-	if len(args) == 0 || args[0] != "accept" {
+	if len(args) == 0 {
-		return errors.New("secret accept <node> <module> <name> [--from <file>]")
+		return errors.New(secretUsage)
+	}
+	switch args[0] {
+	case "accept":
+	case "recover":
+		return secretRecover(ctx, args[1:])
+	case "export":
+		return secretExport(ctx, args[1:])
+	default:
+		return errors.New(secretUsage)
 	}
 	rest, flags := split(args[1:])
 	set := flag.NewFlagSet("secret accept", flag.ContinueOnError)
@@ -39,7 +53,7 @@ func secretCommand(ctx context.Context, args []string) error {
 		return err
 	}
 	if len(rest) != 3 {
-		return errors.New("secret accept <node> <module> <name> [--from <file>]")
+		return errors.New(secretUsage)
 	}
 	node, module, name := rest[0], rest[1], rest[2]
 
@@ -69,6 +83,148 @@ func secretCommand(ctx context.Context, args []string) error {
 	return nil
 }
 
+const secretUsage = "secret accept <node> <module> <name> [--from <file>]\n" +
+	"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>]\n" +
+	"secret export [--out <file>]"
+
+// secretRecover is break-glass: a secret opened with the operator's key, written to a file.
+//
+// **The mesh cannot show a secret back, and this does not make it able to.** What is opened here
+// is the copy sealed to the operator key (novox/hq ADR 0085, amended); the mesh holds that blob and
+// no key for it, and this program holds the key for the length of the call and no blob until given
+// one. Recovery needs both, which is what keeps the sealing meaningful.
+//
+// The value goes to a file at 0600, never to the terminal unless asked for with `--out -` — the
+// source mesh's secret tools were written after a secret was printed into a transcript, and that
+// rule is theirs. `--from-export` reads the blob from a file `secret export` wrote, so recovery
+// works with the store gone, which is the case it exists for.
+func secretRecover(ctx context.Context, args []string) error {
+	rest, flags := split(args)
+	set := flag.NewFlagSet("secret recover", flag.ContinueOnError)
+	keyFile := set.String("key", "", "the operator's private key, from `operator key make`")
+	out := set.String("out", "", "where to write the value (0600); - for standard output. Default <node>.<module>.<name>.secret")
+	fromExport := set.String("from-export", "", "read the sealed copy from this `secret export` file instead of the store")
+	if err := set.Parse(flags); err != nil {
+		return err
+	}
+	if len(rest) != 3 {
+		return errors.New(secretUsage)
+	}
+	node, module, name := rest[0], rest[1], rest[2]
+	private, err := readPrivateKey(*keyFile)
+	if err != nil {
+		return err
+	}
+
+	var kept inventory.Kept
+	if *fromExport != "" {
+		kept, err = keptFromExport(*fromExport, node, module, name)
+		if err != nil {
+			return err
+		}
+	} else {
+		open, err := openStores(ctx)
+		if err != nil {
+			return err
+		}
+		defer open.Close()
+		kept, err = open.inventory.KeptSecret(ctx, node, module, name)
+		if err != nil {
+			return err
+		}
+	}
+
+	value, err := secrets.Open(private, kept.Sealed)
+	if err != nil {
+		return fmt.Errorf("%s on %s: %q is sealed to operator key %s, and that key does not open it: %w",
+			module, node, name, secrets.Fingerprint(kept.Key), err)
+	}
+	if *out == "-" {
+		_, err := os.Stdout.Write(append(value, '\n'))
+		return err
+	}
+	path := *out
+	if path == "" {
+		path = node + "." + module + "." + name + ".secret"
+	}
+	if _, err := os.Stat(path); err == nil {
+		return fmt.Errorf("%s already exists; not overwriting it", path)
+	}
+	if err := os.WriteFile(path, value, 0o600); err != nil {
+		return err
+	}
+	fmt.Printf("%s on %s: %q recovered to %s (0600) — %d bytes, origin %s\n",
+		module, node, name, path, len(value), kept.Origin)
+	return nil
+}
+
+// An export is what a person keeps beside the operator key — the catalogue's shape, so the vault
+// keeps the same document on its disk (Manifest.Keeps).
+type export = catalogue.KeptExport
+
+func secretExport(ctx context.Context, args []string) error {
+	set := flag.NewFlagSet("secret export", flag.ContinueOnError)
+	out := set.String("out", "", "where to write the export (0600); - or empty for standard output")
+	if err := set.Parse(args); err != nil {
+		return err
+	}
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+	key, err := inv.OperatorKey(ctx)
+	if err != nil {
+		return err
+	}
+	if key == "" {
+		return errors.New("the mesh has no operator key, so nothing is sealed to one; `operator key make` and `operator key set` first")
+	}
+	kept, unrecoverable, err := inv.KeptForOperator(ctx)
+	if err != nil {
+		return err
+	}
+	body, err := json.MarshalIndent(export{
+		Export: 1, OperatorKey: key, Fingerprint: secrets.Fingerprint(key),
+		Kept: kept, Unrecoverable: unrecoverable,
+	}, "", "  ")
+	if err != nil {
+		return err
+	}
+	body = append(body, '\n')
+	if *out == "" || *out == "-" {
+		_, err := os.Stdout.Write(body)
+		return err
+	}
+	if err := os.WriteFile(*out, body, 0o600); err != nil {
+		return err
+	}
+	fmt.Printf("%d secret(s) exported to %s (0600), sealed to operator key %s — ciphertext, keep it with the key\n",
+		len(kept), *out, secrets.Fingerprint(key))
+	if len(unrecoverable) > 0 {
+		fmt.Printf("  %d secret(s) are NOT in it: made before the mesh had an operator key\n", len(unrecoverable))
+	}
+	return nil
+}
+
+func keptFromExport(path, node, module, name string) (inventory.Kept, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return inventory.Kept{}, err
+	}
+	var e export
+	if err := json.Unmarshal(raw, &e); err != nil {
+		return inventory.Kept{}, fmt.Errorf("%s is not a secret export: %w", path, err)
+	}
+	for _, k := range e.Kept {
+		if k.Node == node && k.Module == module && k.Name == name {
+			return k, nil
+		}
+	}
+	return inventory.Kept{}, fmt.Errorf("%s holds no copy of %s's %q on %s", path, module, name, node)
+}
+
 // split separates what this command is about from how it was asked.
 //
 // **Because the standard library stops parsing at the first non-flag argument.** With the

internal/catalogue/declaration.go

@@ -14,6 +14,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 )
 
 // SettingsBy is the layers that apply to each module, keyed by module name.
@@ -89,6 +90,10 @@ type Rendering struct {
 	// a fact about the mesh, and resolution answers questions about one machine.
 	Mesh []string
 
+	// Kept is every operator-sealed secret in the mesh, for a module that `keeps` them. Nil when
+	// nothing on this node keeps them, or the mesh has no operator key.
+	Kept *KeptExport
+
 	// Foundation is the ports the mesh itself needs reachable on every machine, which no module
 	// declares because the foundation is not a module (novox/hq 04-ISSUES/051 and 052). The broker
 	// is the one that matters: a machine dials it to enrol, and a firewall derived only from
@@ -384,6 +389,13 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) {
 			}
 			first = append(first, file)
 		}
+		if m.Keeps != "" && with.Kept != nil {
+			file, err := keptFile(m.Keeps, with.Kept)
+			if err != nil {
+				return nil, err
+			}
+			first = append(first, file)
+		}
 		if m.Computed != "" {
 			generator, known := with.Generators[m.Computed]
 			if !known {
@@ -725,6 +737,45 @@ func receivedFile(requirement, path string, given []Contribution) (map[string]an
 	}, nil
 }
 
+// Kept is one secret as the operator can recover it: where it belongs, and the value sealed to the
+// operator's key. Never a node's blob, and never a value.
+type Kept struct {
+	Node   string `json:"node"`
+	Module string `json:"module"`
+	Name   string `json:"name"`
+	// Origin is `made` or `accepted` — whether the mesh minted it or a person supplied it.
+	Origin string `json:"origin"`
+	// Sealed is the value, sealed to the operator key named in Key.
+	Sealed string    `json:"sealed"`
+	Key    string    `json:"key"`
+	MadeAt time.Time `json:"made-at"`
+}
+
+// KeptExport is what a person keeps beside the operator key, and what a vault keeps on its disk:
+// every operator-sealed copy, and the honest list of what has none.
+type KeptExport struct {
+	Export        int    `json:"export"`
+	OperatorKey   string `json:"operator-key"`
+	Fingerprint   string `json:"fingerprint"`
+	Kept          []Kept `json:"kept"`
+	Unrecoverable []Kept `json:"unrecoverable,omitempty"`
+}
+
+// KeptID is the resource that carries the export to a module that keeps it.
+func KeptID() string { return "kept" }
+
+// keptFile is the export, written where the module said. Ciphertext throughout — see Keeps.
+func keptFile(dir string, kept *KeptExport) (map[string]any, error) {
+	body, err := json.MarshalIndent(kept, "", "  ")
+	if err != nil {
+		return nil, err
+	}
+	return map[string]any{
+		"id": KeptID(), "type": "file", "path": strings.TrimRight(dir, "/") + "/export.json",
+		"mode": "0600", "content": string(body) + "\n",
+	}, nil
+}
+
 // sortedKeys is map iteration made repeatable, which everything written to a machine needs.
 func sortedKeys[V any](m map[string]V) []string {
 	out := make([]string, 0, len(m))

internal/catalogue/kept_test.go

@@ -0,0 +1,68 @@
+package catalogue
+
+import (
+	"strings"
+	"testing"
+)
+
+// A module that keeps the operator-sealed secrets is handed the export as a file, at 0600, and a
+// module that does not keep them is handed nothing — the export goes to the vault and nowhere else.
+func TestOnlyAModuleThatKeepsGetsTheExport(t *testing.T) {
+	vault := Manifest{Module: "mesh-vault", Version: "1", Provides: FromAnywhere("secret"),
+		Keeps: "/var/lib/mesh-vault/root"}
+	other := Manifest{Module: "zsh", Version: "1", Provides: Offers("shell")}
+	got, err := Resolve(shelf(vault, other), []string{"mesh-vault", "zsh"},
+		Node{Name: "anchor", At: "10.0.0.1", Capabilities: map[string]bool{}}, World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	kept := &KeptExport{Export: 1, OperatorKey: "OPERATOR", Fingerprint: "sha256:abcd",
+		Kept: []Kept{{Node: "anchor", Module: "postgres", Name: "superuser", Origin: "accepted", Sealed: "CIPHERTEXT", Key: "OPERATOR"}}}
+	out, err := got.Declaration(Rendering{Kept: kept})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var files int
+	for _, r := range out {
+		if r["path"] != "/var/lib/mesh-vault/root/export.json" {
+			continue
+		}
+		files++
+		if r["mode"] != "0600" {
+			t.Errorf("the export is written at mode %v, and it is the mesh's root secrets, sealed or not", r["mode"])
+		}
+		content, _ := r["content"].(string)
+		for _, want := range []string{`"operator-key": "OPERATOR"`, `"sealed": "CIPHERTEXT"`, `"module": "postgres"`} {
+			if !strings.Contains(content, want) {
+				t.Errorf("the export lacks %s:\n%s", want, content)
+			}
+		}
+		if id, _ := r["id"].(string); !strings.Contains(id, "mesh-vault") {
+			t.Errorf("the export's resource id %q does not carry its module", id)
+		}
+	}
+	if files != 1 {
+		t.Fatalf("%d export files; one module keeps them", files)
+	}
+
+	// No operator key yet: the vault is declared without the file, not with an empty one.
+	out, err = got.Declaration(Rendering{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, r := range out {
+		if r["path"] == "/var/lib/mesh-vault/root/export.json" {
+			t.Fatal("an export was written with nothing to export")
+		}
+	}
+}
+
+func TestKeepsMustBeAnAbsolutePath(t *testing.T) {
+	_, err := ParseManifest([]byte(`{"module":"mesh-vault","version":"1","keeps":"root"}`))
+	if err == nil || !strings.Contains(err.Error(), "keeps") {
+		t.Fatalf("a relative keeps path was not refused: %v", err)
+	}
+	if _, err := ParseManifest([]byte(`{"module":"mesh-vault","version":"1","keeps":"/var/lib/mesh-vault/root"}`)); err != nil {
+		t.Fatalf("an absolute keeps path was refused: %v", err)
+	}
+}

internal/catalogue/manifest.go

@@ -297,6 +297,17 @@ type Manifest struct {
 	// module, in a file anybody can read, for ever.
 	OwnSecrets map[string]string `json:"own-secrets,omitempty"`
 
+	// Keeps is where this module wants every operator-sealed secret in the mesh written — the
+	// vault's field, and so far nobody else's (novox/hq ADR 0085, amended).
+	//
+	// A directory. The mesh writes one file into it, `export.json`: every secret a module holds for
+	// itself, sealed to the operator's key, with the key's public half and the list of what is NOT
+	// in it. Ciphertext to the machine that holds it and to everything on the bus it crossed —
+	// only the operator, holding the private half off the mesh, can open a line of it. That is
+	// what lets a vault's disk stand in for the store when the store is gone: recovery needs the
+	// export and the key, and the mesh holds neither in a form it can use.
+	Keeps string `json:"keeps,omitempty"`
+
 	// Listens is what this module accepts connections on, and from where.
 	//
 	// **A rule names its source** ([ADR 0007](novox/hq)). A port with no source is open to
@@ -918,6 +929,10 @@ func ParseManifest(raw []byte) (Manifest, error) {
 				"%s grants %q to its consumers and does not provide it", m.Module, to))
 		}
 	}
+	if m.Keeps != "" && !strings.HasPrefix(m.Keeps, "/") {
+		problems = append(problems, fmt.Sprintf(
+			"%s keeps the operator-sealed secrets at %q, which is not an absolute path", m.Module, m.Keeps))
+	}
 	for to, where := range m.Receives {
 		if !name.MatchString(to) {
 			problems = append(problems, fmt.Sprintf("%q is not a usable name to receive", to))

internal/inventory/migrations/0023-an-operator-key-and-a-second-seal.sql

@@ -0,0 +1,22 @@
+-- The operator's sealing key, and a second seal on every secret a module holds for itself.
+--
+-- Every secret here is sealed to the node that will use it and to nothing else, so a node whose key
+-- is gone takes its secrets with it -- and the mesh's own root secrets, the store's superuser and
+-- the broker's administrator among them, are exactly such secrets. novox/hq ADR 0085 (amended)
+-- gives them a second holder: a person, with a key whose private half never enters the mesh. What
+-- the mesh keeps is one more blob it cannot open.
+
+-- At most one operator key at a time. A row rather than a setting, because it is a fact about the
+-- mesh with consequences (what can be recovered), not somebody's preference about a module.
+create table operator_key (
+    public  text        not null primary key,
+    made_at timestamptz not null default now()
+);
+
+alter table module_secret
+    -- The same value, sealed to the operator key -- null for a secret minted before there was
+    -- one, which cannot be sealed after the fact: the plaintext was discarded. Such a secret is
+    -- recoverable only once it is issued again.
+    add column operator_sealed text,
+    -- Which operator key, so a replaced key can be told what it can no longer open.
+    add column operator_key    text;

internal/inventory/operator.go

@@ -0,0 +1,116 @@
+package inventory
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/jackc/pgx/v5"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// The operator's sealing key: the one holder of secrets that is not a node.
+//
+// Every secret a module holds for itself is sealed to the node that uses it, and a node whose key
+// is gone takes its secrets with it — the mesh's root secrets included. novox/hq ADR 0085 (amended)
+// gives them a second recipient: a person, holding a key whose private half never enters the mesh.
+// What is recorded here is the public half, which is all the mesh needs to seal to it; what it
+// yields is one more blob per secret that the mesh cannot open.
+
+// OperatorKey is the public key secrets are also sealed to, or empty when the mesh has none.
+func (i *Inventory) OperatorKey(ctx context.Context) (string, error) {
+	var key string
+	err := i.store.Pool().QueryRow(ctx,
+		`select public from operator_key order by made_at desc limit 1`).Scan(&key)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return "", nil
+	}
+	return key, err
+}
+
+// SetOperatorKey records the operator's public key, replacing any earlier one.
+//
+// **Replacing is said, not silent.** Secrets sealed to the earlier key stay sealed to it: the
+// plaintext is gone, so they cannot be sealed again to the new one until each is issued again. The
+// number of them is returned so the caller can say so — a key swapped with nothing said would look
+// like a mesh with a recovery path and be a mesh without one.
+func (i *Inventory) SetOperatorKey(ctx context.Context, public string) (orphaned int, err error) {
+	if public == "" {
+		return 0, fmt.Errorf("an operator key is a public key, and this is nothing")
+	}
+	tx, err := i.store.Pool().Begin(ctx)
+	if err != nil {
+		return 0, err
+	}
+	defer tx.Rollback(ctx)
+	if err := tx.QueryRow(ctx,
+		`select count(*) from module_secret
+		  where operator_key is not null and operator_key <> $1`, public).Scan(&orphaned); err != nil {
+		return 0, err
+	}
+	if _, err := tx.Exec(ctx, `delete from operator_key where public <> $1`, public); err != nil {
+		return 0, err
+	}
+	if _, err := tx.Exec(ctx,
+		`insert into operator_key (public) values ($1) on conflict (public) do nothing`, public); err != nil {
+		return 0, err
+	}
+	return orphaned, tx.Commit(ctx)
+}
+
+// Kept is the catalogue's: one secret as the operator can recover it.
+type Kept = catalogue.Kept
+
+// KeptForOperator is every secret the operator can recover, and which cannot.
+//
+// The second list is the honest half: a secret minted before the mesh had an operator key has no
+// operator-sealed copy and cannot get one — the plaintext was discarded. Naming those is what lets
+// an export say what it does not cover, rather than being taken for complete.
+func (i *Inventory) KeptForOperator(ctx context.Context) (kept []Kept, unrecoverable []Kept, err error) {
+	rows, err := i.store.Pool().Query(ctx,
+		`select n.name, s.module, s.name, s.origin, coalesce(s.operator_sealed, ''),
+		        coalesce(s.operator_key, ''), s.made_at
+		   from module_secret s join node n on n.id = s.node
+		  order by n.name, s.module, s.name`)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var k Kept
+		if err := rows.Scan(&k.Node, &k.Module, &k.Name, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt); err != nil {
+			return nil, nil, err
+		}
+		if k.Sealed == "" {
+			unrecoverable = append(unrecoverable, k)
+			continue
+		}
+		kept = append(kept, k)
+	}
+	return kept, unrecoverable, rows.Err()
+}
+
+// KeptSecret is one secret's operator-sealed copy, for recovery.
+func (i *Inventory) KeptSecret(ctx context.Context, node, module, name string) (Kept, error) {
+	var k Kept
+	err := i.store.Pool().QueryRow(ctx,
+		`select n.name, s.module, s.name, s.origin, coalesce(s.operator_sealed, ''),
+		        coalesce(s.operator_key, ''), s.made_at
+		   from module_secret s join node n on n.id = s.node
+		  where n.name = $1 and s.module = $2 and s.name = $3`, node, module, name).
+		Scan(&k.Node, &k.Module, &k.Name, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return Kept{}, fmt.Errorf("%s on %s holds nothing called %q", module, node, name)
+	}
+	if err != nil {
+		return Kept{}, err
+	}
+	if k.Sealed == "" {
+		return Kept{}, fmt.Errorf(
+			"%s on %s holds %q, but it was made before the mesh had an operator key and so has no "+
+				"copy a person can open. Issue it again (secret accept, or let the mesh remake it) "+
+				"and it will", module, node, name)
+	}
+	return k, nil
+}

internal/inventory/operator_test.go

@@ -0,0 +1,97 @@
+package inventory
+
+import (
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/secrets"
+)
+
+// With an operator key, a module's own secret is sealed to the operator as well — and the operator
+// opens exactly the value the node was given.
+func TestAnOwnSecretIsSealedToTheOperatorToo(t *testing.T) {
+	inv, ctx := twoNodesWithKeys(t)
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"}, Source{}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Before there is a key: minted, and honestly unrecoverable.
+	if _, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.KeptSecret(ctx, "consumer", "postgres", "superuser"); err == nil {
+		t.Fatal("a secret made before the operator key was reported recoverable")
+	}
+	kept, unrecoverable, err := inv.KeptForOperator(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(kept) != 0 || len(unrecoverable) != 1 {
+		t.Fatalf("before a key: %d kept, %d unrecoverable", len(kept), len(unrecoverable))
+	}
+
+	pub, priv, err := secrets.Keypair()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if orphaned, err := inv.SetOperatorKey(ctx, pub); err != nil || orphaned != 0 {
+		t.Fatalf("set: orphaned %d, %v", orphaned, err)
+	}
+
+	// A new secret is sealed to both; an accepted one too.
+	if _, err := inv.SecretForModule(ctx, "provider", "postgres", "superuser"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.AcceptSecretForModule(ctx, "provider", "postgres", "replication", "given-by-a-person"); err != nil {
+		t.Fatal(err)
+	}
+	kept, unrecoverable, err = inv.KeptForOperator(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(kept) != 2 || len(unrecoverable) != 1 {
+		t.Fatalf("after a key: %d kept, %d unrecoverable", len(kept), len(unrecoverable))
+	}
+	got, err := inv.KeptSecret(ctx, "provider", "postgres", "replication")
+	if err != nil {
+		t.Fatal(err)
+	}
+	value, err := secrets.Open(priv, got.Sealed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(value) != "given-by-a-person" {
+		t.Fatalf("recovered %q", value)
+	}
+	if got.Origin != "accepted" || got.Key != pub {
+		t.Fatalf("kept as %+v", got)
+	}
+	minted, err := inv.KeptSecret(ctx, "provider", "postgres", "superuser")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v, err := secrets.Open(priv, minted.Sealed); err != nil || len(v) != 40 {
+		t.Fatalf("the minted secret did not open to a 40-character value: %v", err)
+	}
+
+	// The old secret, remade for a rejoined node, becomes recoverable — it was issued again.
+	if _, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.KeptSecret(ctx, "consumer", "postgres", "superuser"); err == nil {
+		t.Fatal("asking again did not remake, yet it became recoverable")
+	}
+
+	// Replacing the key says how many secrets stay sealed to the old one.
+	pub2, _, _ := secrets.Keypair()
+	orphaned, err := inv.SetOperatorKey(ctx, pub2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if orphaned != 2 {
+		t.Fatalf("replacing the key orphaned %d, and two were sealed to it", orphaned)
+	}
+	if now, _ := inv.OperatorKey(ctx); now != pub2 {
+		t.Fatal("the new key is not the mesh's key")
+	}
+}

internal/inventory/secrets.go

@@ -227,19 +227,33 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri
 			module, node, name, node)
 	}
 
-	made, err := secrets.Make(key, key)
+	operator, err := i.OperatorKey(ctx)
 	if err != nil {
 		return "", err
 	}
-	// Sealed once, to one recipient. Make seals to two ends because a provision has two; here
+	var also []string
-	// both are the same machine, and only one copy is kept.
+	if operator != "" {
+		also = append(also, operator)
+	}
+	made, more, err := secrets.MakeAlso(key, key, also...)
+	if err != nil {
+		return "", err
+	}
+	// Sealed once to the machine — Make seals to two ends because a provision has two; here both
+	// are the same machine, and only one copy is kept — and once more to the operator when the
+	// mesh has one (novox/hq ADR 0085, amended), which is the copy a person can recover from.
+	var forOperator, operatorKey *string
+	if operator != "" {
+		forOperator, operatorKey = &more[0], &operator
+	}
 	if _, err := i.store.Pool().Exec(ctx,
-		`insert into module_secret (node, module, name, sealed, node_key, origin)
+		`insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key)
-		 values ($1, $2, $3, $4, $5, 'made')
+		 values ($1, $2, $3, $4, $5, 'made', $6, $7)
 		 on conflict (node, module, name) do update set
 		   sealed = excluded.sealed, node_key = excluded.node_key,
-		   origin = excluded.origin, made_at = now()`,
+		   origin = excluded.origin, made_at = now(),
-		record.ID, module, name, made.ForConsumer, key); err != nil {
+		   operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
+		record.ID, module, name, made.ForConsumer, key, forOperator, operatorKey); err != nil {
 		return "", err
 	}
 	return made.ForConsumer, nil
@@ -273,13 +287,28 @@ func (i *Inventory) AcceptSecretForModule(ctx context.Context, node, module, nam
 	if err != nil {
 		return err
 	}
+	// And to the operator, when the mesh has one: a value a person supplied is the one a person
+	// most needs to get back, since the mesh cannot make another (novox/hq ADR 0085, amended).
+	operator, err := i.OperatorKey(ctx)
+	if err != nil {
+		return err
+	}
+	var forOperator, operatorKey *string
+	if operator != "" {
+		blob, err := secrets.Seal(operator, []byte(value))
+		if err != nil {
+			return err
+		}
+		forOperator, operatorKey = &blob, &operator
+	}
 	_, err = i.store.Pool().Exec(ctx,
-		`insert into module_secret (node, module, name, sealed, node_key, origin)
+		`insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key)
-		 values ($1, $2, $3, $4, $5, 'accepted')
+		 values ($1, $2, $3, $4, $5, 'accepted', $6, $7)
 		 on conflict (node, module, name) do update set
 		   sealed = excluded.sealed, node_key = excluded.node_key,
-		   origin = excluded.origin, made_at = now()`,
+		   origin = excluded.origin, made_at = now(),
-		record.ID, module, name, sealed.ForConsumer, key)
+		   operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
+		record.ID, module, name, sealed.ForConsumer, key, forOperator, operatorKey)
 	return err
 }
 

internal/secrets/operator_test.go

@@ -0,0 +1,72 @@
+package secrets
+
+import "testing"
+
+// A secret sealed to the operator as well is opened by the operator's key and by nothing else.
+func TestAThirdRecipientOpensWithItsOwnKeyOnly(t *testing.T) {
+	nodePub, nodePriv, err := Keypair()
+	if err != nil {
+		t.Fatal(err)
+	}
+	opPub, opPriv, err := Keypair()
+	if err != nil {
+		t.Fatal(err)
+	}
+	sealed, more, err := MakeAlso(nodePub, nodePub, opPub)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(more) != 1 {
+		t.Fatalf("%d extra blobs for one extra key", len(more))
+	}
+	fromNode, err := Open(nodePriv, sealed.ForConsumer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fromOperator, err := Open(opPriv, more[0])
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(fromNode) != string(fromOperator) {
+		t.Fatal("the operator's copy is a different value from the node's")
+	}
+	if len(fromNode) != 40 {
+		t.Fatalf("a minted value is %d characters, not 40", len(fromNode))
+	}
+	if _, err := Open(nodePriv, more[0]); err == nil {
+		t.Fatal("the node's key opened the operator's blob")
+	}
+	if _, err := Open(opPriv, sealed.ForConsumer); err == nil {
+		t.Fatal("the operator's key opened the node's blob")
+	}
+}
+
+// An accepted value, sealed to the operator, comes back byte for byte.
+func TestAnAcceptedValueRoundTripsThroughTheOperatorKey(t *testing.T) {
+	opPub, opPriv, err := Keypair()
+	if err != nil {
+		t.Fatal(err)
+	}
+	blob, err := Seal(opPub, []byte("  the-superuser's password, spaces and all  "))
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := Open(opPriv, blob)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != "  the-superuser's password, spaces and all  " {
+		t.Fatalf("got %q", got)
+	}
+}
+
+func TestAFingerprintNamesAKeyAndIsNotOne(t *testing.T) {
+	pub, _, _ := Keypair()
+	fp := Fingerprint(pub)
+	if len(fp) != len("sha256:")+16 || fp[:7] != "sha256:" {
+		t.Fatalf("fingerprint %q", fp)
+	}
+	if fp == Fingerprint(pub+"x") {
+		t.Fatal("two keys, one fingerprint")
+	}
+}

internal/secrets/seal.go

@@ -1,8 +1,11 @@
 package secrets
 
 import (
+	"crypto/ecdh"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/base64"
+	"encoding/hex"
 	"fmt"
 	"strings"
 
@@ -48,10 +51,22 @@ type Sealed struct {
 // rather than reading the old one back — the only version of rotation that is honest about what
 // the mesh knows.
 func Make(consumerKey, providerKey string) (Sealed, error) {
+	sealed, _, err := MakeAlso(consumerKey, providerKey)
+	return sealed, err
+}
+
+// MakeAlso is Make with further recipients: the same fresh value, sealed once more to each key in
+// `also`, returned in that order.
+//
+// **For the operator key, and nothing else so far** (novox/hq ADR 0085, amended). A secret a module
+// holds for itself is sealed to its node and, when the mesh has an operator key, to that as well —
+// so a person holding the key can recover it when the node cannot. The plaintext still exists only
+// inside this call; a third blob is one more thing the mesh cannot open, not one more copy it can.
+func MakeAlso(consumerKey, providerKey string, also ...string) (Sealed, []string, error) {
 	if consumerKey == "" || providerKey == "" {
 		// Sealing to an empty key would produce a blob nobody can open, stored as though it were
 		// a working credential. The caller knows which node is which and says so.
-		return Sealed{}, fmt.Errorf("both ends need a sealing key before a secret can be made")
+		return Sealed{}, nil, fmt.Errorf("both ends need a sealing key before a secret can be made")
 	}
 
 	// 30 bytes, not 32: base64url of 30 is exactly 40 characters, and 40 is the longest secret an
@@ -59,7 +74,7 @@ func Make(consumerKey, providerKey string) (Sealed, error) {
 	// "fit the tightest backend" rule ADR 0049 sets for the login, on the secret. 240 bits is ample.
 	value := make([]byte, 30)
 	if _, err := rand.Read(value); err != nil {
-		return Sealed{}, err
+		return Sealed{}, nil, err
 	}
 	// Base64 without padding, because it lands in a configuration file something else parses and
 	// a password containing a newline or a quote is a support call.
@@ -67,16 +82,24 @@ func Make(consumerKey, providerKey string) (Sealed, error) {
 
 	forConsumer, err := Seal(consumerKey, []byte(password))
 	if err != nil {
-		return Sealed{}, err
+		return Sealed{}, nil, err
 	}
 	forProvider, err := Seal(providerKey, []byte(password))
 	if err != nil {
-		return Sealed{}, err
+		return Sealed{}, nil, err
+	}
+	more := make([]string, 0, len(also))
+	for _, key := range also {
+		blob, err := Seal(key, []byte(password))
+		if err != nil {
+			return Sealed{}, nil, err
+		}
+		more = append(more, blob)
 	}
 	return Sealed{
 		ForConsumer: forConsumer, ForProvider: forProvider,
 		ConsumerKey: consumerKey, ProviderKey: providerKey,
-	}, nil
+	}, more, nil
 }
 
 // Accept seals a value somebody supplied, rather than one the mesh made.
@@ -115,6 +138,52 @@ func Accept(value string, consumerKey, providerKey string) (Sealed, error) {
 	}, nil
 }
 
+// Open is the other half of Seal, for the one holder of a private key this program ever acts for:
+// the operator, recovering a secret with the key that never entered the mesh (novox/hq ADR 0085,
+// amended). A node opens its own blobs in the host; the controller opens nothing of a node's, and
+// cannot — it has no node's private key, which is the whole point of sealing.
+func Open(privateKey string, sealed string) ([]byte, error) {
+	private, err := base64.StdEncoding.DecodeString(strings.TrimSpace(privateKey))
+	if err != nil || len(private) != 32 {
+		return nil, fmt.Errorf("that is not a sealing key")
+	}
+	key, err := ecdh.X25519().NewPrivateKey(private)
+	if err != nil {
+		return nil, fmt.Errorf("that is not a usable sealing key: %w", err)
+	}
+	blob, err := base64.StdEncoding.DecodeString(sealed)
+	if err != nil {
+		return nil, fmt.Errorf("this is not a sealed value: %w", err)
+	}
+	var pub, priv [32]byte
+	copy(pub[:], key.PublicKey().Bytes())
+	copy(priv[:], private)
+	out, ok := box.OpenAnonymous(nil, blob, &pub, &priv)
+	if !ok {
+		return nil, fmt.Errorf("this was not sealed to that key")
+	}
+	return out, nil
+}
+
+// Keypair makes a sealing keypair for a holder outside the mesh — the operator. The private half is
+// returned to be written where the caller says and nowhere else; the public half is what the mesh
+// records and seals to.
+func Keypair() (public, private string, err error) {
+	key, err := ecdh.X25519().GenerateKey(rand.Reader)
+	if err != nil {
+		return "", "", err
+	}
+	return base64.StdEncoding.EncodeToString(key.PublicKey().Bytes()),
+		base64.StdEncoding.EncodeToString(key.Bytes()), nil
+}
+
+// Fingerprint names a public key without being one: the first bytes of its hash, so two people can
+// agree which key they mean out loud.
+func Fingerprint(publicKey string) string {
+	sum := sha256.Sum256([]byte(strings.TrimSpace(publicKey)))
+	return "sha256:" + hex.EncodeToString(sum[:8])
+}
+
 // Seal closes a value to a node's public sealing key.
 func Seal(publicKey string, value []byte) (string, error) {
 	public, err := base64.StdEncoding.DecodeString(publicKey)