An operator key, a second seal on every own secret, and the vault keeps the export

novox/hq ADR 0085, amended: the mesh's root secrets — the store's superuser,
the broker's administrator, every secret a module holds for itself — were
sealed to a node key and nothing else, so a lost node took them with it.
Now the mesh records an operator's public sealing key and seals every own
secret to it as well, minted or accepted. The private half is written once
by `operator key new` to a file the operator keeps off the mesh; the mesh
holds one more blob per secret that it cannot open.

`secret recover` opens a secret with that key, to a 0600 file, from the
store or from an export; `secret export` writes every operator-sealed copy
as ciphertext. A module that `keeps` (the vault) is handed that export as a
declared file on its own disk, so recovery survives the store.

Secrets made before the key exists have no operator copy and are said so —
the plaintext was discarded — until each is issued again.

internal/inventory/migrations/0023-an-operator-key-and-a-second-seal.sql

@@ -0,0 +1,22 @@
+-- The operator's sealing key, and a second seal on every secret a module holds for itself.
+--
+-- Every secret here is sealed to the node that will use it and to nothing else, so a node whose key
+-- is gone takes its secrets with it -- and the mesh's own root secrets, the store's superuser and
+-- the broker's administrator among them, are exactly such secrets. novox/hq ADR 0085 (amended)
+-- gives them a second holder: a person, with a key whose private half never enters the mesh. What
+-- the mesh keeps is one more blob it cannot open.
+
+-- At most one operator key at a time. A row rather than a setting, because it is a fact about the
+-- mesh with consequences (what can be recovered), not somebody's preference about a module.
+create table operator_key (
+    public  text        not null primary key,
+    made_at timestamptz not null default now()
+);
+
+alter table module_secret
+    -- The same value, sealed to the operator key -- null for a secret minted before there was
+    -- one, which cannot be sealed after the fact: the plaintext was discarded. Such a secret is
+    -- recoverable only once it is issued again.
+    add column operator_sealed text,
+    -- Which operator key, so a replaced key can be told what it can no longer open.
+    add column operator_key    text;

internal/inventory/operator.go

@@ -0,0 +1,116 @@
+package inventory
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/jackc/pgx/v5"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// The operator's sealing key: the one holder of secrets that is not a node.
+//
+// Every secret a module holds for itself is sealed to the node that uses it, and a node whose key
+// is gone takes its secrets with it — the mesh's root secrets included. novox/hq ADR 0085 (amended)
+// gives them a second recipient: a person, holding a key whose private half never enters the mesh.
+// What is recorded here is the public half, which is all the mesh needs to seal to it; what it
+// yields is one more blob per secret that the mesh cannot open.
+
+// OperatorKey is the public key secrets are also sealed to, or empty when the mesh has none.
+func (i *Inventory) OperatorKey(ctx context.Context) (string, error) {
+	var key string
+	err := i.store.Pool().QueryRow(ctx,
+		`select public from operator_key order by made_at desc limit 1`).Scan(&key)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return "", nil
+	}
+	return key, err
+}
+
+// SetOperatorKey records the operator's public key, replacing any earlier one.
+//
+// **Replacing is said, not silent.** Secrets sealed to the earlier key stay sealed to it: the
+// plaintext is gone, so they cannot be sealed again to the new one until each is issued again. The
+// number of them is returned so the caller can say so — a key swapped with nothing said would look
+// like a mesh with a recovery path and be a mesh without one.
+func (i *Inventory) SetOperatorKey(ctx context.Context, public string) (orphaned int, err error) {
+	if public == "" {
+		return 0, fmt.Errorf("an operator key is a public key, and this is nothing")
+	}
+	tx, err := i.store.Pool().Begin(ctx)
+	if err != nil {
+		return 0, err
+	}
+	defer tx.Rollback(ctx)
+	if err := tx.QueryRow(ctx,
+		`select count(*) from module_secret
+		  where operator_key is not null and operator_key <> $1`, public).Scan(&orphaned); err != nil {
+		return 0, err
+	}
+	if _, err := tx.Exec(ctx, `delete from operator_key where public <> $1`, public); err != nil {
+		return 0, err
+	}
+	if _, err := tx.Exec(ctx,
+		`insert into operator_key (public) values ($1) on conflict (public) do nothing`, public); err != nil {
+		return 0, err
+	}
+	return orphaned, tx.Commit(ctx)
+}
+
+// Kept is the catalogue's: one secret as the operator can recover it.
+type Kept = catalogue.Kept
+
+// KeptForOperator is every secret the operator can recover, and which cannot.
+//
+// The second list is the honest half: a secret minted before the mesh had an operator key has no
+// operator-sealed copy and cannot get one — the plaintext was discarded. Naming those is what lets
+// an export say what it does not cover, rather than being taken for complete.
+func (i *Inventory) KeptForOperator(ctx context.Context) (kept []Kept, unrecoverable []Kept, err error) {
+	rows, err := i.store.Pool().Query(ctx,
+		`select n.name, s.module, s.name, s.origin, coalesce(s.operator_sealed, ''),
+		        coalesce(s.operator_key, ''), s.made_at
+		   from module_secret s join node n on n.id = s.node
+		  order by n.name, s.module, s.name`)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var k Kept
+		if err := rows.Scan(&k.Node, &k.Module, &k.Name, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt); err != nil {
+			return nil, nil, err
+		}
+		if k.Sealed == "" {
+			unrecoverable = append(unrecoverable, k)
+			continue
+		}
+		kept = append(kept, k)
+	}
+	return kept, unrecoverable, rows.Err()
+}
+
+// KeptSecret is one secret's operator-sealed copy, for recovery.
+func (i *Inventory) KeptSecret(ctx context.Context, node, module, name string) (Kept, error) {
+	var k Kept
+	err := i.store.Pool().QueryRow(ctx,
+		`select n.name, s.module, s.name, s.origin, coalesce(s.operator_sealed, ''),
+		        coalesce(s.operator_key, ''), s.made_at
+		   from module_secret s join node n on n.id = s.node
+		  where n.name = $1 and s.module = $2 and s.name = $3`, node, module, name).
+		Scan(&k.Node, &k.Module, &k.Name, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return Kept{}, fmt.Errorf("%s on %s holds nothing called %q", module, node, name)
+	}
+	if err != nil {
+		return Kept{}, err
+	}
+	if k.Sealed == "" {
+		return Kept{}, fmt.Errorf(
+			"%s on %s holds %q, but it was made before the mesh had an operator key and so has no "+
+				"copy a person can open. Issue it again (secret accept, or let the mesh remake it) "+
+				"and it will", module, node, name)
+	}
+	return k, nil
+}

internal/inventory/operator_test.go

@@ -0,0 +1,97 @@
+package inventory
+
+import (
+	"testing"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/secrets"
+)
+
+// With an operator key, a module's own secret is sealed to the operator as well — and the operator
+// opens exactly the value the node was given.
+func TestAnOwnSecretIsSealedToTheOperatorToo(t *testing.T) {
+	inv, ctx := twoNodesWithKeys(t)
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"}, Source{}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Before there is a key: minted, and honestly unrecoverable.
+	if _, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.KeptSecret(ctx, "consumer", "postgres", "superuser"); err == nil {
+		t.Fatal("a secret made before the operator key was reported recoverable")
+	}
+	kept, unrecoverable, err := inv.KeptForOperator(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(kept) != 0 || len(unrecoverable) != 1 {
+		t.Fatalf("before a key: %d kept, %d unrecoverable", len(kept), len(unrecoverable))
+	}
+
+	pub, priv, err := secrets.Keypair()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if orphaned, err := inv.SetOperatorKey(ctx, pub); err != nil || orphaned != 0 {
+		t.Fatalf("set: orphaned %d, %v", orphaned, err)
+	}
+
+	// A new secret is sealed to both; an accepted one too.
+	if _, err := inv.SecretForModule(ctx, "provider", "postgres", "superuser"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.AcceptSecretForModule(ctx, "provider", "postgres", "replication", "given-by-a-person"); err != nil {
+		t.Fatal(err)
+	}
+	kept, unrecoverable, err = inv.KeptForOperator(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(kept) != 2 || len(unrecoverable) != 1 {
+		t.Fatalf("after a key: %d kept, %d unrecoverable", len(kept), len(unrecoverable))
+	}
+	got, err := inv.KeptSecret(ctx, "provider", "postgres", "replication")
+	if err != nil {
+		t.Fatal(err)
+	}
+	value, err := secrets.Open(priv, got.Sealed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(value) != "given-by-a-person" {
+		t.Fatalf("recovered %q", value)
+	}
+	if got.Origin != "accepted" || got.Key != pub {
+		t.Fatalf("kept as %+v", got)
+	}
+	minted, err := inv.KeptSecret(ctx, "provider", "postgres", "superuser")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v, err := secrets.Open(priv, minted.Sealed); err != nil || len(v) != 40 {
+		t.Fatalf("the minted secret did not open to a 40-character value: %v", err)
+	}
+
+	// The old secret, remade for a rejoined node, becomes recoverable — it was issued again.
+	if _, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.KeptSecret(ctx, "consumer", "postgres", "superuser"); err == nil {
+		t.Fatal("asking again did not remake, yet it became recoverable")
+	}
+
+	// Replacing the key says how many secrets stay sealed to the old one.
+	pub2, _, _ := secrets.Keypair()
+	orphaned, err := inv.SetOperatorKey(ctx, pub2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if orphaned != 2 {
+		t.Fatalf("replacing the key orphaned %d, and two were sealed to it", orphaned)
+	}
+	if now, _ := inv.OperatorKey(ctx); now != pub2 {
+		t.Fatal("the new key is not the mesh's key")
+	}
+}

internal/inventory/secrets.go

@@ -227,19 +227,33 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri
 			module, node, name, node)
 	}
 
-	made, err := secrets.Make(key, key)
+	operator, err := i.OperatorKey(ctx)
 	if err != nil {
 		return "", err
 	}
-	// Sealed once, to one recipient. Make seals to two ends because a provision has two; here
-	// both are the same machine, and only one copy is kept.
+	var also []string
+	if operator != "" {
+		also = append(also, operator)
+	}
+	made, more, err := secrets.MakeAlso(key, key, also...)
+	if err != nil {
+		return "", err
+	}
+	// Sealed once to the machine — Make seals to two ends because a provision has two; here both
+	// are the same machine, and only one copy is kept — and once more to the operator when the
+	// mesh has one (novox/hq ADR 0085, amended), which is the copy a person can recover from.
+	var forOperator, operatorKey *string
+	if operator != "" {
+		forOperator, operatorKey = &more[0], &operator
+	}
 	if _, err := i.store.Pool().Exec(ctx,
-		`insert into module_secret (node, module, name, sealed, node_key, origin)
-		 values ($1, $2, $3, $4, $5, 'made')
+		`insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key)
+		 values ($1, $2, $3, $4, $5, 'made', $6, $7)
 		 on conflict (node, module, name) do update set
 		   sealed = excluded.sealed, node_key = excluded.node_key,
-		   origin = excluded.origin, made_at = now()`,
-		record.ID, module, name, made.ForConsumer, key); err != nil {
+		   origin = excluded.origin, made_at = now(),
+		   operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
+		record.ID, module, name, made.ForConsumer, key, forOperator, operatorKey); err != nil {
 		return "", err
 	}
 	return made.ForConsumer, nil
@@ -273,13 +287,28 @@ func (i *Inventory) AcceptSecretForModule(ctx context.Context, node, module, nam
 	if err != nil {
 		return err
 	}
+	// And to the operator, when the mesh has one: a value a person supplied is the one a person
+	// most needs to get back, since the mesh cannot make another (novox/hq ADR 0085, amended).
+	operator, err := i.OperatorKey(ctx)
+	if err != nil {
+		return err
+	}
+	var forOperator, operatorKey *string
+	if operator != "" {
+		blob, err := secrets.Seal(operator, []byte(value))
+		if err != nil {
+			return err
+		}
+		forOperator, operatorKey = &blob, &operator
+	}
 	_, err = i.store.Pool().Exec(ctx,
-		`insert into module_secret (node, module, name, sealed, node_key, origin)
-		 values ($1, $2, $3, $4, $5, 'accepted')
+		`insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key)
+		 values ($1, $2, $3, $4, $5, 'accepted', $6, $7)
 		 on conflict (node, module, name) do update set
 		   sealed = excluded.sealed, node_key = excluded.node_key,
-		   origin = excluded.origin, made_at = now()`,
-		record.ID, module, name, sealed.ForConsumer, key)
+		   origin = excluded.origin, made_at = now(),
+		   operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`,
+		record.ID, module, name, sealed.ForConsumer, key, forOperator, operatorKey)
 	return err
 }