An operator key, a second seal on every own secret, and the vault keeps the export

novox/hq ADR 0085, amended: the mesh's root secrets — the store's superuser,
the broker's administrator, every secret a module holds for itself — were
sealed to a node key and nothing else, so a lost node took them with it.
Now the mesh records an operator's public sealing key and seals every own
secret to it as well, minted or accepted. The private half is written once
by `operator key new` to a file the operator keeps off the mesh; the mesh
holds one more blob per secret that it cannot open.

`secret recover` opens a secret with that key, to a 0600 file, from the
store or from an export; `secret export` writes every operator-sealed copy
as ciphertext. A module that `keeps` (the vault) is handed that export as a
declared file on its own disk, so recovery survives the store.

Secrets made before the key exists have no operator copy and are said so —
the plaintext was discarded — until each is issued again.

cmd/mesh-controller/main.go

@@ -100,6 +100,8 @@ func run() error {
 		return settingsCommand(ctx, args[1:])
 	case "secret":
 		return secretCommand(ctx, args[1:])
+	case "operator":
+		return operatorCommand(ctx, args[1:])
 	case "plan":
 		return planCommand(ctx, args[1:])
 	case "push":
@@ -155,6 +157,12 @@ func usage() {
   settings clear <module> [--node <n>]      take a layer away
   secret accept <node> <module> <name>     carry a value the mesh did not make and cannot invent
   secret accept ... --from <file>           ...read it from a file rather than being asked
+  secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>]
+                                            break-glass: open the operator-sealed copy, to a 0600 file
+  secret export [--out <file>]              every operator-sealed copy, ciphertext — keep it with the key
+  operator key make [--out <file>]          make the operator's sealing key, off the mesh; private half to the file only
+  operator key set <public> [--replace]     tell the mesh which operator key to seal to
+  operator key show                         the operator key, and what it can recover
   build <repository> [--ref R]   have a build machine build it, and record what came out
   build --behind                 build every module the mesh holds older than its source
   builds [<module>]              what has been built lately, and what came of it

cmd/mesh-controller/operator.go

@@ -0,0 +1,157 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/novox/mesh-controller/internal/secrets"
+)
+
+// operatorCommand is the mesh's one holder of secrets that is not a machine.
+//
+// **Every secret a module holds for itself is sealed to the node that uses it, and a node whose key
+// is gone takes its secrets with it** — the store's superuser and the broker's administrator among
+// them. novox/hq ADR 0085 (amended) gives them a second recipient: a person, with a sealing key
+// whose private half is made where the operator is and never enters the mesh. Making the key and
+// telling the mesh about it are two commands, on purpose: the first needs no mesh at all and runs
+// wherever the operator keeps things; the second gives the mesh the public half and nothing else.
+// The controller's own container is a scratch image with no writable path, which is the right
+// shape for a program that must hold no key — so the private half could not be written there
+// even by mistake.
+//
+//	operator key make [--out <file>]    make a keypair: private half to the file, public half printed
+//	operator key set <public>           tell the mesh which key to seal to
+//	operator key show                   the public key, its fingerprint, and what it can recover
+const operatorUsage = "operator key make [--out <file>] | operator key set <public> [--replace] | operator key show"
+
+func operatorCommand(ctx context.Context, args []string) error {
+	if len(args) < 2 || args[0] != "key" {
+		return errors.New(operatorUsage)
+	}
+	switch args[1] {
+	case "make":
+		return operatorKeyMake(args[2:])
+	case "set":
+		return operatorKeySet(ctx, args[2:])
+	case "show":
+		return operatorKeyShow(ctx)
+	default:
+		return errors.New(operatorUsage)
+	}
+}
+
+// operatorKeyMake needs no mesh: it is the operator's, run wherever the key will live.
+func operatorKeyMake(args []string) error {
+	set := flag.NewFlagSet("operator key make", flag.ContinueOnError)
+	out := set.String("out", "operator.key",
+		"where to write the private key (0600); keep it off the mesh, and keep it")
+	if err := set.Parse(args); err != nil {
+		return err
+	}
+	if _, err := os.Stat(*out); err == nil {
+		return fmt.Errorf("%s already exists; this will not overwrite a key somebody may still need", *out)
+	}
+	public, private, err := secrets.Keypair()
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(*out, []byte(private+"\n"), 0o600); err != nil {
+		return err
+	}
+	fmt.Printf("operator key %s\n", secrets.Fingerprint(public))
+	fmt.Printf("  private half written to %s (0600) — keep it off the mesh, and keep it\n", *out)
+	fmt.Printf("  public half, to give the mesh with `operator key set`:\n")
+	fmt.Printf("public %s\n", public)
+	return nil
+}
+
+func operatorKeySet(ctx context.Context, args []string) error {
+	rest, flags := split(args)
+	set := flag.NewFlagSet("operator key set", flag.ContinueOnError)
+	replace := set.Bool("replace", false,
+		"replace an existing operator key — secrets sealed to the old one stay sealed to it")
+	if err := set.Parse(flags); err != nil {
+		return err
+	}
+	if len(rest) != 1 {
+		return errors.New(operatorUsage)
+	}
+	public := strings.TrimSpace(rest[0])
+	if _, err := secrets.Seal(public, []byte("probe")); err != nil {
+		return fmt.Errorf("that is not a public sealing key: %w", err)
+	}
+
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+
+	if current, err := inv.OperatorKey(ctx); err != nil {
+		return err
+	} else if current != "" && current != public && !*replace {
+		return fmt.Errorf(
+			"the mesh already has an operator key (%s). Pass --replace to change it — "+
+				"secrets sealed to the current key stay sealed to it until each is issued again",
+			secrets.Fingerprint(current))
+	}
+	orphaned, err := inv.SetOperatorKey(ctx, public)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("operator key %s\n", secrets.Fingerprint(public))
+	fmt.Printf("  the mesh holds the public half only and cannot open what it seals to it;\n")
+	fmt.Printf("  from now on every secret a module holds for itself is sealed to it as well.\n")
+	fmt.Printf("  Secrets made before this cannot be — each is recoverable once issued again\n")
+	if orphaned > 0 {
+		fmt.Printf("  %d secret(s) are sealed to the previous key and stay so until issued again\n", orphaned)
+	}
+	return nil
+}
+
+func operatorKeyShow(ctx context.Context) error {
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+	key, err := inv.OperatorKey(ctx)
+	if err != nil {
+		return err
+	}
+	if key == "" {
+		fmt.Println("the mesh has no operator key; `operator key make` then `operator key set` gives it one")
+		return nil
+	}
+	kept, unrecoverable, err := inv.KeptForOperator(ctx)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("operator key %s\n  %s\n", secrets.Fingerprint(key), key)
+	fmt.Printf("  %d secret(s) recoverable with it\n", len(kept))
+	if len(unrecoverable) > 0 {
+		fmt.Printf("  %d secret(s) not recoverable — made before it, or sealed to an earlier key:\n", len(unrecoverable))
+		for _, k := range unrecoverable {
+			fmt.Printf("    %s  %s  %s\n", k.Node, k.Module, k.Name)
+		}
+	}
+	return nil
+}
+
+// readPrivateKey is the operator's key from the file `operator key make` wrote.
+func readPrivateKey(path string) (string, error) {
+	if path == "" {
+		return "", errors.New("--key <file> names the operator's private key, written by `operator key make`")
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(raw)), nil
+}

cmd/mesh-controller/plan.go

@@ -13,6 +13,7 @@ import (
 	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/inventory"
 	"github.com/novox/mesh-controller/internal/licences"
+	"github.com/novox/mesh-controller/internal/secrets"
 	"net"
 	"strconv"
 )
@@ -462,10 +463,35 @@ func declarationWith(ctx context.Context, open *stores, node string,
 		}
 	}
 
+	// And, for a module that keeps them, every operator-sealed secret in the mesh — the vault's
+	// copy, outside the store (novox/hq ADR 0085, amended). Read only; nothing here mints.
+	var kept *catalogue.KeptExport
+	for _, m := range plan.Modules {
+		if m.Keeps == "" {
+			continue
+		}
+		operator, err := inv.OperatorKey(ctx)
+		if err != nil {
+			return nil, err
+		}
+		if operator == "" {
+			break // nothing is sealed to an operator, so there is nothing to keep yet
+		}
+		recoverable, unrecoverable, err := inv.KeptForOperator(ctx)
+		if err != nil {
+			return nil, err
+		}
+		kept = &catalogue.KeptExport{
+			Export: 1, OperatorKey: operator, Fingerprint: secrets.Fingerprint(operator),
+			Kept: recoverable, Unrecoverable: unrecoverable,
+		}
+		break
+	}
+
 	return plan.Declaration(catalogue.Rendering{
 		Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports,
 		Certificate: certificate, Authority: authority, Mesh: private, Names: names,
-		Foundation: foundation})
+		Foundation: foundation, Kept: kept})
 }
 
 // routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq

cmd/mesh-controller/secret.go

@@ -3,12 +3,17 @@ package main
 import (
 	"bufio"
 	"context"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"os"
 	"strings"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/secrets"
 )
 
 // secretCommand gives the mesh a value it must carry and could not have invented.
@@ -28,8 +33,17 @@ import (
 // The value is sealed on the way in and the plaintext discarded, exactly as a generated one is.
 // **The only difference between the two is where the value came from.**
 func secretCommand(ctx context.Context, args []string) error {
-	if len(args) == 0 || args[0] != "accept" {
-		return errors.New("secret accept <node> <module> <name> [--from <file>]")
+	if len(args) == 0 {
+		return errors.New(secretUsage)
+	}
+	switch args[0] {
+	case "accept":
+	case "recover":
+		return secretRecover(ctx, args[1:])
+	case "export":
+		return secretExport(ctx, args[1:])
+	default:
+		return errors.New(secretUsage)
 	}
 	rest, flags := split(args[1:])
 	set := flag.NewFlagSet("secret accept", flag.ContinueOnError)
@@ -39,7 +53,7 @@ func secretCommand(ctx context.Context, args []string) error {
 		return err
 	}
 	if len(rest) != 3 {
-		return errors.New("secret accept <node> <module> <name> [--from <file>]")
+		return errors.New(secretUsage)
 	}
 	node, module, name := rest[0], rest[1], rest[2]
 
@@ -69,6 +83,148 @@ func secretCommand(ctx context.Context, args []string) error {
 	return nil
 }
 
+const secretUsage = "secret accept <node> <module> <name> [--from <file>]\n" +
+	"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>]\n" +
+	"secret export [--out <file>]"
+
+// secretRecover is break-glass: a secret opened with the operator's key, written to a file.
+//
+// **The mesh cannot show a secret back, and this does not make it able to.** What is opened here
+// is the copy sealed to the operator key (novox/hq ADR 0085, amended); the mesh holds that blob and
+// no key for it, and this program holds the key for the length of the call and no blob until given
+// one. Recovery needs both, which is what keeps the sealing meaningful.
+//
+// The value goes to a file at 0600, never to the terminal unless asked for with `--out -` — the
+// source mesh's secret tools were written after a secret was printed into a transcript, and that
+// rule is theirs. `--from-export` reads the blob from a file `secret export` wrote, so recovery
+// works with the store gone, which is the case it exists for.
+func secretRecover(ctx context.Context, args []string) error {
+	rest, flags := split(args)
+	set := flag.NewFlagSet("secret recover", flag.ContinueOnError)
+	keyFile := set.String("key", "", "the operator's private key, from `operator key make`")
+	out := set.String("out", "", "where to write the value (0600); - for standard output. Default <node>.<module>.<name>.secret")
+	fromExport := set.String("from-export", "", "read the sealed copy from this `secret export` file instead of the store")
+	if err := set.Parse(flags); err != nil {
+		return err
+	}
+	if len(rest) != 3 {
+		return errors.New(secretUsage)
+	}
+	node, module, name := rest[0], rest[1], rest[2]
+	private, err := readPrivateKey(*keyFile)
+	if err != nil {
+		return err
+	}
+
+	var kept inventory.Kept
+	if *fromExport != "" {
+		kept, err = keptFromExport(*fromExport, node, module, name)
+		if err != nil {
+			return err
+		}
+	} else {
+		open, err := openStores(ctx)
+		if err != nil {
+			return err
+		}
+		defer open.Close()
+		kept, err = open.inventory.KeptSecret(ctx, node, module, name)
+		if err != nil {
+			return err
+		}
+	}
+
+	value, err := secrets.Open(private, kept.Sealed)
+	if err != nil {
+		return fmt.Errorf("%s on %s: %q is sealed to operator key %s, and that key does not open it: %w",
+			module, node, name, secrets.Fingerprint(kept.Key), err)
+	}
+	if *out == "-" {
+		_, err := os.Stdout.Write(append(value, '\n'))
+		return err
+	}
+	path := *out
+	if path == "" {
+		path = node + "." + module + "." + name + ".secret"
+	}
+	if _, err := os.Stat(path); err == nil {
+		return fmt.Errorf("%s already exists; not overwriting it", path)
+	}
+	if err := os.WriteFile(path, value, 0o600); err != nil {
+		return err
+	}
+	fmt.Printf("%s on %s: %q recovered to %s (0600) — %d bytes, origin %s\n",
+		module, node, name, path, len(value), kept.Origin)
+	return nil
+}
+
+// An export is what a person keeps beside the operator key — the catalogue's shape, so the vault
+// keeps the same document on its disk (Manifest.Keeps).
+type export = catalogue.KeptExport
+
+func secretExport(ctx context.Context, args []string) error {
+	set := flag.NewFlagSet("secret export", flag.ContinueOnError)
+	out := set.String("out", "", "where to write the export (0600); - or empty for standard output")
+	if err := set.Parse(args); err != nil {
+		return err
+	}
+	open, err := openStores(ctx)
+	if err != nil {
+		return err
+	}
+	defer open.Close()
+	inv := open.inventory
+	key, err := inv.OperatorKey(ctx)
+	if err != nil {
+		return err
+	}
+	if key == "" {
+		return errors.New("the mesh has no operator key, so nothing is sealed to one; `operator key make` and `operator key set` first")
+	}
+	kept, unrecoverable, err := inv.KeptForOperator(ctx)
+	if err != nil {
+		return err
+	}
+	body, err := json.MarshalIndent(export{
+		Export: 1, OperatorKey: key, Fingerprint: secrets.Fingerprint(key),
+		Kept: kept, Unrecoverable: unrecoverable,
+	}, "", "  ")
+	if err != nil {
+		return err
+	}
+	body = append(body, '\n')
+	if *out == "" || *out == "-" {
+		_, err := os.Stdout.Write(body)
+		return err
+	}
+	if err := os.WriteFile(*out, body, 0o600); err != nil {
+		return err
+	}
+	fmt.Printf("%d secret(s) exported to %s (0600), sealed to operator key %s — ciphertext, keep it with the key\n",
+		len(kept), *out, secrets.Fingerprint(key))
+	if len(unrecoverable) > 0 {
+		fmt.Printf("  %d secret(s) are NOT in it: made before the mesh had an operator key\n", len(unrecoverable))
+	}
+	return nil
+}
+
+func keptFromExport(path, node, module, name string) (inventory.Kept, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return inventory.Kept{}, err
+	}
+	var e export
+	if err := json.Unmarshal(raw, &e); err != nil {
+		return inventory.Kept{}, fmt.Errorf("%s is not a secret export: %w", path, err)
+	}
+	for _, k := range e.Kept {
+		if k.Node == node && k.Module == module && k.Name == name {
+			return k, nil
+		}
+	}
+	return inventory.Kept{}, fmt.Errorf("%s holds no copy of %s's %q on %s", path, module, name, node)
+}
+
 // split separates what this command is about from how it was asked.
 //
 // **Because the standard library stops parsing at the first non-flag argument.** With the