mesh-vault: finish a new operator key after a failure partway, record every hand-out, reach the vault's copies from secret recover (review 250)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

modules/mesh-vault/cmd/mesh-vault/tools.go

@@ -1,8 +1,10 @@
 package main
 
 // What the vault serves: the seat's verb `get`, and the three tools the TypeScript vault offered —
-// secret_export, secret_holders and secret_verify — answering as they did. None returns a value: `get`
-// returns one sealed to the asker's key, the others fingerprints and ciphertext only.
+// secret_export, secret_holders and secret_verify — answering as they did; beside them secret_handed and
+// secret_vault_export over the vault's own store, and ADR 0230's four retirement tools over the custody
+// ledger. None returns a value: `get` returns one sealed to the asker's key, the others fingerprints and
+// ciphertext only.
 
 import (
 	"encoding/json"
@@ -21,6 +23,7 @@ type Env struct {
 	Ledger   *Ledger
 	Receives string
 	Root     string
+	Custody  *Custody
 	// Broken is why the vault could not be built, said by `get`.
 	Broken error
 }
@@ -111,9 +114,107 @@ func definitions(env *Env) []toolDef {
 				given, _ := a["fingerprint"].(string)
 				return verifyAnswer(env, as, given), nil
 			}},
+		{Name: "secret_handed",
+			Description: "Who was handed which secret by `get`, and when, and who was refused, newest first, as the " +
+				"vault's own store records it. Never a value.",
+			Input: map[string]any{"limit": map[string]any{"type": "number", "description": "how many (default 50)"}},
+			run: func(_ string, a map[string]any) (any, error) {
+				if env.Vault == nil {
+					return nil, errors.New("the vault is not ready")
+				}
+				return env.Vault.Store.Handed(limitOf(a))
+			}},
+		{Name: "secret_vault_export",
+			Description: "Every secret the vault itself holds, as its copy sealed to the operator key, in the shape of " +
+				"`secret export`: saved to a file, `nox secret recover vault mesh-vault <name> --key <operator key file> " +
+				"--from-export <file> --out <file>` opens one with the operator's private key. Ciphertext only.",
+			run: func(string, map[string]any) (any, error) {
+				if env.Vault == nil {
+					return nil, errors.New("the vault is not ready")
+				}
+				return vaultExport(env.Vault)
+			}},
+		{Name: "provisioner_retirement",
+			Description: "The custody ledger's holders: active, retired (with when and why), the set counted toward " +
+				"retirement, and any set waiting for a person or rejected (novox/hq ADR 0230).",
+			run: func(string, map[string]any) (any, error) {
+				if env.Custody == nil {
+					return nil, errors.New("the vault has no custody ledger")
+				}
+				return env.Custody.Retirement(), nil
+			}},
+		{Name: "provisioner_retire_approve",
+			Description: "Retire exactly the set of holders waiting for a person (novox/hq ADR 0230). Marks them retired; deletes nothing.",
+			Input:       map[string]any{"why": str("why they are retired")},
+			run: func(caller string, a map[string]any) (any, error) {
+				if env.Custody == nil {
+					return nil, errors.New("the vault has no custody ledger")
+				}
+				why, _ := a["why"].(string)
+				set, err := env.Custody.Approve(caller, why)
+				return map[string]any{"retired": set}, err
+			}},
+		{Name: "provisioner_retire_reject",
+			Description: "Keep the set of holders waiting for a person active; not asked about again while it stays the same set (novox/hq ADR 0230).",
+			Input:       map[string]any{"why": str("why they are kept")},
+			run: func(caller string, a map[string]any) (any, error) {
+				if env.Custody == nil {
+					return nil, errors.New("the vault has no custody ledger")
+				}
+				why, _ := a["why"].(string)
+				set, err := env.Custody.Reject(caller, why)
+				return map[string]any{"kept": set}, err
+			}},
+		{Name: "provisioner_delete",
+			Description: "Delete one retired holder from the custody ledger, its rotation history with it — a person's act " +
+				"(novox/hq ADR 0230). Refused for a holder that is active or asked for, and without a why.",
+			Input: map[string]any{"consumer": str("the holder's login"), "why": str("why it is deleted")},
+			run: func(caller string, a map[string]any) (any, error) {
+				if env.Custody == nil {
+					return nil, errors.New("the vault has no custody ledger")
+				}
+				as, _ := a["consumer"].(string)
+				why, _ := a["why"].(string)
+				if err := env.Custody.Delete(as, caller, why); err != nil {
+					return nil, err
+				}
+				return map[string]any{"deleted": as}, nil
+			}},
 	}
 }
 
+func limitOf(a map[string]any) int {
+	if v, ok := a["limit"].(float64); ok && v > 0 {
+		return int(v)
+	}
+	return 50
+}
+
+// vaultExport is the vault's operator copies in the shape `secret export` writes and `secret recover
+// --from-export` reads: each as node "vault", module "mesh-vault" and the secret's name.
+func vaultExport(v *Vault) (map[string]any, error) {
+	key, err := v.Store.OperatorKey()
+	if err != nil {
+		return nil, err
+	}
+	all, err := v.Store.List()
+	if err != nil {
+		return nil, err
+	}
+	kept, earlier := []map[string]any{}, []map[string]any{}
+	for _, rec := range all {
+		k := map[string]any{"node": "vault", "module": Seat, "name": rec.Name, "kind": "own", "origin": "made",
+			"sealed": rec.Operator.Sealed, "key": rec.Operator.Key, "made-at": rec.MadeAt}
+		if rec.Operator.Key == key {
+			kept = append(kept, k)
+		} else {
+			earlier = append(earlier, k)
+		}
+	}
+	return map[string]any{"export": 1, "operator-key": key, "fingerprint": "", "kept": kept,
+		"sealed-to-earlier-key": earlier}, nil
+}
+
 func exportAnswer(root string, withBlobs bool) map[string]any {
 	if root == "" {
 		return map[string]any{"available": false, "error": "this vault keeps no root secrets (MESH_VAULT_ROOT is not set)"}

modules/mesh-vault/cmd/mesh-vault/vault.go

@@ -138,6 +138,16 @@ func (v *Vault) announce(c Changed) {
 	}
 }
 
+// recordHanding keeps who asked for what, and what they got, in the vault's own store.
+func (v *Vault) recordHanding(h Handing) error {
+	h.At = v.Now().UTC()
+	err := v.Store.RecordHanding(h)
+	if err != nil {
+		v.log("the record of %s asking for %s could not be written: %v", h.To, h.Name, err)
+	}
+	return err
+}
+
 // Get answers caller with the secret name sealed to publicKey, if caller may ask for it. The caller is the
 // bus user the runtime proved, never an argument; an unproven call is refused.
 func (v *Vault) Get(caller, name, publicKey string) (Answer, error) {
@@ -150,6 +160,7 @@ func (v *Vault) Get(caller, name, publicKey string) (Answer, error) {
 	}
 	if ok, why := askers.May(caller, name); !ok {
 		v.log("refused %s to %s: %s", name, caller, why)
+		v.recordHanding(Handing{To: caller, Name: name, Outcome: "refused", Why: why})
 		return Answer{}, fmt.Errorf("%s may not ask for %s: %s", caller, name, why)
 	}
 	key, err := askerKey(publicKey)
@@ -171,25 +182,91 @@ func (v *Vault) Get(caller, name, publicKey string) (Answer, error) {
 	if err != nil {
 		return Answer{}, err
 	}
+	if err := v.recordHanding(Handing{To: caller, Name: name, Generation: rec.Generation, Outcome: "handed"}); err != nil {
+		return Answer{}, fmt.Errorf("%s is not handed out: who was handed it could not be recorded: %w", name, err)
+	}
 	v.log("handed %s generation %d to %s, sealed to its key", name, rec.Generation, caller)
 	return Answer{Name: name, Generation: rec.Generation, ChangedAt: rec.ChangedAt, Sealed: sealed}, nil
 }
 
-// SetOperatorKey makes key the operator key and seals every secret's operator copy to it again, so none
-// stays sealed to a key the operator may no longer hold. It returns how many copies were sealed again.
+// SetOperatorKey makes key (SPKI PEM) the operator key, then seals again to it every operator copy that is
+// not yet — see Reseal. It returns how many copies it sealed again.
+//
+// The key is recorded first and each record says which key its copy is sealed to, so that record is the
+// progress: a failure partway leaves the rest for the next call, and the same key given again (every minute
+// from the mesh's export) finishes them. Nothing returns early while a copy is sealed to another key.
 func (v *Vault) SetOperatorKey(key, by string) (int, error) {
-	key, err := checkOperatorKey(key)
+	raw, err := checkOperatorKey(key)
 	if err != nil {
 		return 0, err
 	}
 	v.mu.Lock()
 	defer v.mu.Unlock()
-	if current, err := v.Store.OperatorKey(); err != nil {
+	current, err := v.Store.OperatorKey()
+	if err != nil {
 		return 0, err
-	} else if current == key {
-		return 0, nil
 	}
-	if err := v.Store.setOperatorKey(key, by, v.Now().UTC()); err != nil {
+	if current != raw {
+		if err := v.Store.setOperatorKey(raw, by, v.Now().UTC()); err != nil {
+			return 0, err
+		}
+		v.log("the operator key is now %s (by %s); every operator copy is sealed again to it", raw, by)
+	}
+	return v.reseal(raw)
+}
+
+// Reseal seals again to the current operator key every copy sealed to another one, and says how many it
+// did. Called every minute, so a copy a failure left behind is not left for ever.
+func (v *Vault) Reseal() (int, error) {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	key, err := v.Store.OperatorKey()
+	if err != nil || key == "" {
+		return 0, err
+	}
+	return v.reseal(key)
+}
+
+func (v *Vault) reseal(key string) (int, error) {
+	all, err := v.Store.List()
+	if err != nil {
+		return 0, err
+	}
+	n := 0
+	var failed []error
+	for _, rec := range all {
+		if rec.Operator.Key == key && rec.Operator.Box == OperatorBox {
+			continue
+		}
+		value, err := seal.Open(rec.AtRest, v.Own.PrivateKey)
+		if err != nil {
+			failed = append(failed, fmt.Errorf("%s does not open with the vault's key: %w", rec.Name, err))
+			continue
+		}
+		sealed, err := operatorSeal(key, []byte(value))
+		if err == nil {
+			rec.Operator = OperatorCopy{Box: OperatorBox, Key: key, Sealed: sealed}
+			err = v.Store.Write(rec)
+		}
+		if err != nil {
+			failed = append(failed, fmt.Errorf("%s: %w", rec.Name, err))
+			continue
+		}
+		n++
+	}
+	if len(failed) > 0 {
+		err := fmt.Errorf("%d operator copies are still sealed to an earlier key, tried again on the next pass: %w",
+			len(failed), errors.Join(failed...))
+		v.log("%v", err)
+		return n, err
+	}
+	return n, nil
+}
+
+// StillSealedToAnother is how many operator copies are not yet sealed to the current operator key.
+func (v *Vault) StillSealedToAnother() (int, error) {
+	key, err := v.Store.OperatorKey()
+	if err != nil {
 		return 0, err
 	}
 	all, err := v.Store.List()
@@ -198,23 +275,9 @@ func (v *Vault) SetOperatorKey(key, by string) (int, error) {
 	}
 	n := 0
 	for _, rec := range all {
-		if rec.Operator.Key == key {
-			continue
+		if rec.Operator.Key != key {
+			n++
 		}
-		value, err := seal.Open(rec.AtRest, v.Own.PrivateKey)
-		if err != nil {
-			return n, fmt.Errorf("%s does not open with the vault's key, so its operator copy is still sealed to the earlier key: %w", rec.Name, err)
-		}
-		sealed, err := operatorSeal(key, []byte(value))
-		if err != nil {
-			return n, err
-		}
-		rec.Operator = OperatorCopy{Box: OperatorBox, Key: key, Sealed: sealed}
-		if err := v.Store.Write(rec); err != nil {
-			return n, err
-		}
-		n++
 	}
-	v.log("the operator key is now %s (by %s); %d operator copies sealed to it again", key, by, n)
 	return n, nil
 }