From e140ed5d0b1c1744af60a2ed248910aea0bd5f0d Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 20 Sep 2026 23:53:50 +0200 Subject: [PATCH 1/3] An operator key, a second seal on every own secret, and the vault keeps the export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit novox/hq ADR 0085, amended: the mesh's root secrets — the store's superuser, the broker's administrator, every secret a module holds for itself — were sealed to a node key and nothing else, so a lost node took them with it. Now the mesh records an operator's public sealing key and seals every own secret to it as well, minted or accepted. The private half is written once by `operator key new` to a file the operator keeps off the mesh; the mesh holds one more blob per secret that it cannot open. `secret recover` opens a secret with that key, to a 0600 file, from the store or from an export; `secret export` writes every operator-sealed copy as ciphertext. A module that `keeps` (the vault) is handed that export as a declared file on its own disk, so recovery survives the store. Secrets made before the key exists have no operator copy and are said so — the plaintext was discarded — until each is issued again. --- cmd/mesh-controller/main.go | 8 + cmd/mesh-controller/operator.go | 157 +++++++++++++++++ cmd/mesh-controller/plan.go | 28 ++- cmd/mesh-controller/secret.go | 162 +++++++++++++++++- internal/catalogue/declaration.go | 51 ++++++ internal/catalogue/kept_test.go | 68 ++++++++ internal/catalogue/manifest.go | 15 ++ ...0023-an-operator-key-and-a-second-seal.sql | 22 +++ internal/inventory/operator.go | 116 +++++++++++++ internal/inventory/operator_test.go | 97 +++++++++++ internal/inventory/secrets.go | 51 ++++-- internal/secrets/operator_test.go | 72 ++++++++ internal/secrets/seal.go | 79 ++++++++- 13 files changed, 906 insertions(+), 20 deletions(-) create mode 100644 cmd/mesh-controller/operator.go create mode 100644 internal/catalogue/kept_test.go create mode 100644 internal/inventory/migrations/0023-an-operator-key-and-a-second-seal.sql create mode 100644 internal/inventory/operator.go create mode 100644 internal/inventory/operator_test.go create mode 100644 internal/secrets/operator_test.go diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index bcfb385..08bf58e 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -100,6 +100,8 @@ func run() error { return settingsCommand(ctx, args[1:]) case "secret": return secretCommand(ctx, args[1:]) + case "operator": + return operatorCommand(ctx, args[1:]) case "plan": return planCommand(ctx, args[1:]) case "push": @@ -155,6 +157,12 @@ func usage() { settings clear [--node ] take a layer away secret accept carry a value the mesh did not make and cannot invent secret accept ... --from ...read it from a file rather than being asked + secret recover --key [--out ] [--from-export ] + break-glass: open the operator-sealed copy, to a 0600 file + secret export [--out ] every operator-sealed copy, ciphertext — keep it with the key + operator key make [--out ] make the operator's sealing key, off the mesh; private half to the file only + operator key set [--replace] tell the mesh which operator key to seal to + operator key show the operator key, and what it can recover build [--ref R] have a build machine build it, and record what came out build --behind build every module the mesh holds older than its source builds [] what has been built lately, and what came of it diff --git a/cmd/mesh-controller/operator.go b/cmd/mesh-controller/operator.go new file mode 100644 index 0000000..d46d60b --- /dev/null +++ b/cmd/mesh-controller/operator.go @@ -0,0 +1,157 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "strings" + + "github.com/novox/mesh-controller/internal/secrets" +) + +// operatorCommand is the mesh's one holder of secrets that is not a machine. +// +// **Every secret a module holds for itself is sealed to the node that uses it, and a node whose key +// is gone takes its secrets with it** — the store's superuser and the broker's administrator among +// them. novox/hq ADR 0085 (amended) gives them a second recipient: a person, with a sealing key +// whose private half is made where the operator is and never enters the mesh. Making the key and +// telling the mesh about it are two commands, on purpose: the first needs no mesh at all and runs +// wherever the operator keeps things; the second gives the mesh the public half and nothing else. +// The controller's own container is a scratch image with no writable path, which is the right +// shape for a program that must hold no key — so the private half could not be written there +// even by mistake. +// +// operator key make [--out ] make a keypair: private half to the file, public half printed +// operator key set tell the mesh which key to seal to +// operator key show the public key, its fingerprint, and what it can recover +const operatorUsage = "operator key make [--out ] | operator key set [--replace] | operator key show" + +func operatorCommand(ctx context.Context, args []string) error { + if len(args) < 2 || args[0] != "key" { + return errors.New(operatorUsage) + } + switch args[1] { + case "make": + return operatorKeyMake(args[2:]) + case "set": + return operatorKeySet(ctx, args[2:]) + case "show": + return operatorKeyShow(ctx) + default: + return errors.New(operatorUsage) + } +} + +// operatorKeyMake needs no mesh: it is the operator's, run wherever the key will live. +func operatorKeyMake(args []string) error { + set := flag.NewFlagSet("operator key make", flag.ContinueOnError) + out := set.String("out", "operator.key", + "where to write the private key (0600); keep it off the mesh, and keep it") + if err := set.Parse(args); err != nil { + return err + } + if _, err := os.Stat(*out); err == nil { + return fmt.Errorf("%s already exists; this will not overwrite a key somebody may still need", *out) + } + public, private, err := secrets.Keypair() + if err != nil { + return err + } + if err := os.WriteFile(*out, []byte(private+"\n"), 0o600); err != nil { + return err + } + fmt.Printf("operator key %s\n", secrets.Fingerprint(public)) + fmt.Printf(" private half written to %s (0600) — keep it off the mesh, and keep it\n", *out) + fmt.Printf(" public half, to give the mesh with `operator key set`:\n") + fmt.Printf("public %s\n", public) + return nil +} + +func operatorKeySet(ctx context.Context, args []string) error { + rest, flags := split(args) + set := flag.NewFlagSet("operator key set", flag.ContinueOnError) + replace := set.Bool("replace", false, + "replace an existing operator key — secrets sealed to the old one stay sealed to it") + if err := set.Parse(flags); err != nil { + return err + } + if len(rest) != 1 { + return errors.New(operatorUsage) + } + public := strings.TrimSpace(rest[0]) + if _, err := secrets.Seal(public, []byte("probe")); err != nil { + return fmt.Errorf("that is not a public sealing key: %w", err) + } + + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + + if current, err := inv.OperatorKey(ctx); err != nil { + return err + } else if current != "" && current != public && !*replace { + return fmt.Errorf( + "the mesh already has an operator key (%s). Pass --replace to change it — "+ + "secrets sealed to the current key stay sealed to it until each is issued again", + secrets.Fingerprint(current)) + } + orphaned, err := inv.SetOperatorKey(ctx, public) + if err != nil { + return err + } + fmt.Printf("operator key %s\n", secrets.Fingerprint(public)) + fmt.Printf(" the mesh holds the public half only and cannot open what it seals to it;\n") + fmt.Printf(" from now on every secret a module holds for itself is sealed to it as well.\n") + fmt.Printf(" Secrets made before this cannot be — each is recoverable once issued again\n") + if orphaned > 0 { + fmt.Printf(" %d secret(s) are sealed to the previous key and stay so until issued again\n", orphaned) + } + return nil +} + +func operatorKeyShow(ctx context.Context) error { + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + key, err := inv.OperatorKey(ctx) + if err != nil { + return err + } + if key == "" { + fmt.Println("the mesh has no operator key; `operator key make` then `operator key set` gives it one") + return nil + } + kept, unrecoverable, err := inv.KeptForOperator(ctx) + if err != nil { + return err + } + fmt.Printf("operator key %s\n %s\n", secrets.Fingerprint(key), key) + fmt.Printf(" %d secret(s) recoverable with it\n", len(kept)) + if len(unrecoverable) > 0 { + fmt.Printf(" %d secret(s) not recoverable — made before it, or sealed to an earlier key:\n", len(unrecoverable)) + for _, k := range unrecoverable { + fmt.Printf(" %s %s %s\n", k.Node, k.Module, k.Name) + } + } + return nil +} + +// readPrivateKey is the operator's key from the file `operator key make` wrote. +func readPrivateKey(path string) (string, error) { + if path == "" { + return "", errors.New("--key names the operator's private key, written by `operator key make`") + } + raw, err := os.ReadFile(path) + if err != nil { + return "", err + } + return strings.TrimSpace(string(raw)), nil +} diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 5aa92b6..a5867b7 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -13,6 +13,7 @@ import ( "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/licences" + "github.com/novox/mesh-controller/internal/secrets" "net" "strconv" ) @@ -462,10 +463,35 @@ func declarationWith(ctx context.Context, open *stores, node string, } } + // And, for a module that keeps them, every operator-sealed secret in the mesh — the vault's + // copy, outside the store (novox/hq ADR 0085, amended). Read only; nothing here mints. + var kept *catalogue.KeptExport + for _, m := range plan.Modules { + if m.Keeps == "" { + continue + } + operator, err := inv.OperatorKey(ctx) + if err != nil { + return nil, err + } + if operator == "" { + break // nothing is sealed to an operator, so there is nothing to keep yet + } + recoverable, unrecoverable, err := inv.KeptForOperator(ctx) + if err != nil { + return nil, err + } + kept = &catalogue.KeptExport{ + Export: 1, OperatorKey: operator, Fingerprint: secrets.Fingerprint(operator), + Kept: recoverable, Unrecoverable: unrecoverable, + } + break + } + return plan.Declaration(catalogue.Rendering{ Settings: settings, Generators: gens, Grants: grants, Needed: needed, Ports: ports, Certificate: certificate, Authority: authority, Mesh: private, Names: names, - Foundation: foundation}) + Foundation: foundation, Kept: kept}) } // routeNamesInTheMesh is every routed name and the address of the node that serves it (novox/hq diff --git a/cmd/mesh-controller/secret.go b/cmd/mesh-controller/secret.go index a271502..74a5ece 100644 --- a/cmd/mesh-controller/secret.go +++ b/cmd/mesh-controller/secret.go @@ -3,12 +3,17 @@ package main import ( "bufio" "context" + "encoding/json" "errors" "flag" "fmt" "io" "os" "strings" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/secrets" ) // secretCommand gives the mesh a value it must carry and could not have invented. @@ -28,8 +33,17 @@ import ( // The value is sealed on the way in and the plaintext discarded, exactly as a generated one is. // **The only difference between the two is where the value came from.** func secretCommand(ctx context.Context, args []string) error { - if len(args) == 0 || args[0] != "accept" { - return errors.New("secret accept [--from ]") + if len(args) == 0 { + return errors.New(secretUsage) + } + switch args[0] { + case "accept": + case "recover": + return secretRecover(ctx, args[1:]) + case "export": + return secretExport(ctx, args[1:]) + default: + return errors.New(secretUsage) } rest, flags := split(args[1:]) set := flag.NewFlagSet("secret accept", flag.ContinueOnError) @@ -39,7 +53,7 @@ func secretCommand(ctx context.Context, args []string) error { return err } if len(rest) != 3 { - return errors.New("secret accept [--from ]") + return errors.New(secretUsage) } node, module, name := rest[0], rest[1], rest[2] @@ -69,6 +83,148 @@ func secretCommand(ctx context.Context, args []string) error { return nil } +const secretUsage = "secret accept [--from ]\n" + + "secret recover --key [--out ] [--from-export ]\n" + + "secret export [--out ]" + +// secretRecover is break-glass: a secret opened with the operator's key, written to a file. +// +// **The mesh cannot show a secret back, and this does not make it able to.** What is opened here +// is the copy sealed to the operator key (novox/hq ADR 0085, amended); the mesh holds that blob and +// no key for it, and this program holds the key for the length of the call and no blob until given +// one. Recovery needs both, which is what keeps the sealing meaningful. +// +// The value goes to a file at 0600, never to the terminal unless asked for with `--out -` — the +// source mesh's secret tools were written after a secret was printed into a transcript, and that +// rule is theirs. `--from-export` reads the blob from a file `secret export` wrote, so recovery +// works with the store gone, which is the case it exists for. +func secretRecover(ctx context.Context, args []string) error { + rest, flags := split(args) + set := flag.NewFlagSet("secret recover", flag.ContinueOnError) + keyFile := set.String("key", "", "the operator's private key, from `operator key make`") + out := set.String("out", "", "where to write the value (0600); - for standard output. Default ...secret") + fromExport := set.String("from-export", "", "read the sealed copy from this `secret export` file instead of the store") + if err := set.Parse(flags); err != nil { + return err + } + if len(rest) != 3 { + return errors.New(secretUsage) + } + node, module, name := rest[0], rest[1], rest[2] + private, err := readPrivateKey(*keyFile) + if err != nil { + return err + } + + var kept inventory.Kept + if *fromExport != "" { + kept, err = keptFromExport(*fromExport, node, module, name) + if err != nil { + return err + } + } else { + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + kept, err = open.inventory.KeptSecret(ctx, node, module, name) + if err != nil { + return err + } + } + + value, err := secrets.Open(private, kept.Sealed) + if err != nil { + return fmt.Errorf("%s on %s: %q is sealed to operator key %s, and that key does not open it: %w", + module, node, name, secrets.Fingerprint(kept.Key), err) + } + if *out == "-" { + _, err := os.Stdout.Write(append(value, '\n')) + return err + } + path := *out + if path == "" { + path = node + "." + module + "." + name + ".secret" + } + if _, err := os.Stat(path); err == nil { + return fmt.Errorf("%s already exists; not overwriting it", path) + } + if err := os.WriteFile(path, value, 0o600); err != nil { + return err + } + fmt.Printf("%s on %s: %q recovered to %s (0600) — %d bytes, origin %s\n", + module, node, name, path, len(value), kept.Origin) + return nil +} + +// An export is what a person keeps beside the operator key — the catalogue's shape, so the vault +// keeps the same document on its disk (Manifest.Keeps). +type export = catalogue.KeptExport + +func secretExport(ctx context.Context, args []string) error { + set := flag.NewFlagSet("secret export", flag.ContinueOnError) + out := set.String("out", "", "where to write the export (0600); - or empty for standard output") + if err := set.Parse(args); err != nil { + return err + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + key, err := inv.OperatorKey(ctx) + if err != nil { + return err + } + if key == "" { + return errors.New("the mesh has no operator key, so nothing is sealed to one; `operator key make` and `operator key set` first") + } + kept, unrecoverable, err := inv.KeptForOperator(ctx) + if err != nil { + return err + } + body, err := json.MarshalIndent(export{ + Export: 1, OperatorKey: key, Fingerprint: secrets.Fingerprint(key), + Kept: kept, Unrecoverable: unrecoverable, + }, "", " ") + if err != nil { + return err + } + body = append(body, '\n') + if *out == "" || *out == "-" { + _, err := os.Stdout.Write(body) + return err + } + if err := os.WriteFile(*out, body, 0o600); err != nil { + return err + } + fmt.Printf("%d secret(s) exported to %s (0600), sealed to operator key %s — ciphertext, keep it with the key\n", + len(kept), *out, secrets.Fingerprint(key)) + if len(unrecoverable) > 0 { + fmt.Printf(" %d secret(s) are NOT in it: made before the mesh had an operator key\n", len(unrecoverable)) + } + return nil +} + +func keptFromExport(path, node, module, name string) (inventory.Kept, error) { + raw, err := os.ReadFile(path) + if err != nil { + return inventory.Kept{}, err + } + var e export + if err := json.Unmarshal(raw, &e); err != nil { + return inventory.Kept{}, fmt.Errorf("%s is not a secret export: %w", path, err) + } + for _, k := range e.Kept { + if k.Node == node && k.Module == module && k.Name == name { + return k, nil + } + } + return inventory.Kept{}, fmt.Errorf("%s holds no copy of %s's %q on %s", path, module, name, node) +} + // split separates what this command is about from how it was asked. // // **Because the standard library stops parsing at the first non-flag argument.** With the diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 76339dc..b4194db 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -14,6 +14,7 @@ import ( "sort" "strconv" "strings" + "time" ) // SettingsBy is the layers that apply to each module, keyed by module name. @@ -89,6 +90,10 @@ type Rendering struct { // a fact about the mesh, and resolution answers questions about one machine. Mesh []string + // Kept is every operator-sealed secret in the mesh, for a module that `keeps` them. Nil when + // nothing on this node keeps them, or the mesh has no operator key. + Kept *KeptExport + // Foundation is the ports the mesh itself needs reachable on every machine, which no module // declares because the foundation is not a module (novox/hq 04-ISSUES/051 and 052). The broker // is the one that matters: a machine dials it to enrol, and a firewall derived only from @@ -384,6 +389,13 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { } first = append(first, file) } + if m.Keeps != "" && with.Kept != nil { + file, err := keptFile(m.Keeps, with.Kept) + if err != nil { + return nil, err + } + first = append(first, file) + } if m.Computed != "" { generator, known := with.Generators[m.Computed] if !known { @@ -725,6 +737,45 @@ func receivedFile(requirement, path string, given []Contribution) (map[string]an }, nil } +// Kept is one secret as the operator can recover it: where it belongs, and the value sealed to the +// operator's key. Never a node's blob, and never a value. +type Kept struct { + Node string `json:"node"` + Module string `json:"module"` + Name string `json:"name"` + // Origin is `made` or `accepted` — whether the mesh minted it or a person supplied it. + Origin string `json:"origin"` + // Sealed is the value, sealed to the operator key named in Key. + Sealed string `json:"sealed"` + Key string `json:"key"` + MadeAt time.Time `json:"made-at"` +} + +// KeptExport is what a person keeps beside the operator key, and what a vault keeps on its disk: +// every operator-sealed copy, and the honest list of what has none. +type KeptExport struct { + Export int `json:"export"` + OperatorKey string `json:"operator-key"` + Fingerprint string `json:"fingerprint"` + Kept []Kept `json:"kept"` + Unrecoverable []Kept `json:"unrecoverable,omitempty"` +} + +// KeptID is the resource that carries the export to a module that keeps it. +func KeptID() string { return "kept" } + +// keptFile is the export, written where the module said. Ciphertext throughout — see Keeps. +func keptFile(dir string, kept *KeptExport) (map[string]any, error) { + body, err := json.MarshalIndent(kept, "", " ") + if err != nil { + return nil, err + } + return map[string]any{ + "id": KeptID(), "type": "file", "path": strings.TrimRight(dir, "/") + "/export.json", + "mode": "0600", "content": string(body) + "\n", + }, nil +} + // sortedKeys is map iteration made repeatable, which everything written to a machine needs. func sortedKeys[V any](m map[string]V) []string { out := make([]string, 0, len(m)) diff --git a/internal/catalogue/kept_test.go b/internal/catalogue/kept_test.go new file mode 100644 index 0000000..f60f8c3 --- /dev/null +++ b/internal/catalogue/kept_test.go @@ -0,0 +1,68 @@ +package catalogue + +import ( + "strings" + "testing" +) + +// A module that keeps the operator-sealed secrets is handed the export as a file, at 0600, and a +// module that does not keep them is handed nothing — the export goes to the vault and nowhere else. +func TestOnlyAModuleThatKeepsGetsTheExport(t *testing.T) { + vault := Manifest{Module: "mesh-vault", Version: "1", Provides: FromAnywhere("secret"), + Keeps: "/var/lib/mesh-vault/root"} + other := Manifest{Module: "zsh", Version: "1", Provides: Offers("shell")} + got, err := Resolve(shelf(vault, other), []string{"mesh-vault", "zsh"}, + Node{Name: "anchor", At: "10.0.0.1", Capabilities: map[string]bool{}}, World{}) + if err != nil { + t.Fatal(err) + } + kept := &KeptExport{Export: 1, OperatorKey: "OPERATOR", Fingerprint: "sha256:abcd", + Kept: []Kept{{Node: "anchor", Module: "postgres", Name: "superuser", Origin: "accepted", Sealed: "CIPHERTEXT", Key: "OPERATOR"}}} + out, err := got.Declaration(Rendering{Kept: kept}) + if err != nil { + t.Fatal(err) + } + var files int + for _, r := range out { + if r["path"] != "/var/lib/mesh-vault/root/export.json" { + continue + } + files++ + if r["mode"] != "0600" { + t.Errorf("the export is written at mode %v, and it is the mesh's root secrets, sealed or not", r["mode"]) + } + content, _ := r["content"].(string) + for _, want := range []string{`"operator-key": "OPERATOR"`, `"sealed": "CIPHERTEXT"`, `"module": "postgres"`} { + if !strings.Contains(content, want) { + t.Errorf("the export lacks %s:\n%s", want, content) + } + } + if id, _ := r["id"].(string); !strings.Contains(id, "mesh-vault") { + t.Errorf("the export's resource id %q does not carry its module", id) + } + } + if files != 1 { + t.Fatalf("%d export files; one module keeps them", files) + } + + // No operator key yet: the vault is declared without the file, not with an empty one. + out, err = got.Declaration(Rendering{}) + if err != nil { + t.Fatal(err) + } + for _, r := range out { + if r["path"] == "/var/lib/mesh-vault/root/export.json" { + t.Fatal("an export was written with nothing to export") + } + } +} + +func TestKeepsMustBeAnAbsolutePath(t *testing.T) { + _, err := ParseManifest([]byte(`{"module":"mesh-vault","version":"1","keeps":"root"}`)) + if err == nil || !strings.Contains(err.Error(), "keeps") { + t.Fatalf("a relative keeps path was not refused: %v", err) + } + if _, err := ParseManifest([]byte(`{"module":"mesh-vault","version":"1","keeps":"/var/lib/mesh-vault/root"}`)); err != nil { + t.Fatalf("an absolute keeps path was refused: %v", err) + } +} diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 79646e1..638fe86 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -297,6 +297,17 @@ type Manifest struct { // module, in a file anybody can read, for ever. OwnSecrets map[string]string `json:"own-secrets,omitempty"` + // Keeps is where this module wants every operator-sealed secret in the mesh written — the + // vault's field, and so far nobody else's (novox/hq ADR 0085, amended). + // + // A directory. The mesh writes one file into it, `export.json`: every secret a module holds for + // itself, sealed to the operator's key, with the key's public half and the list of what is NOT + // in it. Ciphertext to the machine that holds it and to everything on the bus it crossed — + // only the operator, holding the private half off the mesh, can open a line of it. That is + // what lets a vault's disk stand in for the store when the store is gone: recovery needs the + // export and the key, and the mesh holds neither in a form it can use. + Keeps string `json:"keeps,omitempty"` + // Listens is what this module accepts connections on, and from where. // // **A rule names its source** ([ADR 0007](novox/hq)). A port with no source is open to @@ -918,6 +929,10 @@ func ParseManifest(raw []byte) (Manifest, error) { "%s grants %q to its consumers and does not provide it", m.Module, to)) } } + if m.Keeps != "" && !strings.HasPrefix(m.Keeps, "/") { + problems = append(problems, fmt.Sprintf( + "%s keeps the operator-sealed secrets at %q, which is not an absolute path", m.Module, m.Keeps)) + } for to, where := range m.Receives { if !name.MatchString(to) { problems = append(problems, fmt.Sprintf("%q is not a usable name to receive", to)) diff --git a/internal/inventory/migrations/0023-an-operator-key-and-a-second-seal.sql b/internal/inventory/migrations/0023-an-operator-key-and-a-second-seal.sql new file mode 100644 index 0000000..c8cb69a --- /dev/null +++ b/internal/inventory/migrations/0023-an-operator-key-and-a-second-seal.sql @@ -0,0 +1,22 @@ +-- The operator's sealing key, and a second seal on every secret a module holds for itself. +-- +-- Every secret here is sealed to the node that will use it and to nothing else, so a node whose key +-- is gone takes its secrets with it -- and the mesh's own root secrets, the store's superuser and +-- the broker's administrator among them, are exactly such secrets. novox/hq ADR 0085 (amended) +-- gives them a second holder: a person, with a key whose private half never enters the mesh. What +-- the mesh keeps is one more blob it cannot open. + +-- At most one operator key at a time. A row rather than a setting, because it is a fact about the +-- mesh with consequences (what can be recovered), not somebody's preference about a module. +create table operator_key ( + public text not null primary key, + made_at timestamptz not null default now() +); + +alter table module_secret + -- The same value, sealed to the operator key -- null for a secret minted before there was + -- one, which cannot be sealed after the fact: the plaintext was discarded. Such a secret is + -- recoverable only once it is issued again. + add column operator_sealed text, + -- Which operator key, so a replaced key can be told what it can no longer open. + add column operator_key text; diff --git a/internal/inventory/operator.go b/internal/inventory/operator.go new file mode 100644 index 0000000..2092b89 --- /dev/null +++ b/internal/inventory/operator.go @@ -0,0 +1,116 @@ +package inventory + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The operator's sealing key: the one holder of secrets that is not a node. +// +// Every secret a module holds for itself is sealed to the node that uses it, and a node whose key +// is gone takes its secrets with it — the mesh's root secrets included. novox/hq ADR 0085 (amended) +// gives them a second recipient: a person, holding a key whose private half never enters the mesh. +// What is recorded here is the public half, which is all the mesh needs to seal to it; what it +// yields is one more blob per secret that the mesh cannot open. + +// OperatorKey is the public key secrets are also sealed to, or empty when the mesh has none. +func (i *Inventory) OperatorKey(ctx context.Context) (string, error) { + var key string + err := i.store.Pool().QueryRow(ctx, + `select public from operator_key order by made_at desc limit 1`).Scan(&key) + if errors.Is(err, pgx.ErrNoRows) { + return "", nil + } + return key, err +} + +// SetOperatorKey records the operator's public key, replacing any earlier one. +// +// **Replacing is said, not silent.** Secrets sealed to the earlier key stay sealed to it: the +// plaintext is gone, so they cannot be sealed again to the new one until each is issued again. The +// number of them is returned so the caller can say so — a key swapped with nothing said would look +// like a mesh with a recovery path and be a mesh without one. +func (i *Inventory) SetOperatorKey(ctx context.Context, public string) (orphaned int, err error) { + if public == "" { + return 0, fmt.Errorf("an operator key is a public key, and this is nothing") + } + tx, err := i.store.Pool().Begin(ctx) + if err != nil { + return 0, err + } + defer tx.Rollback(ctx) + if err := tx.QueryRow(ctx, + `select count(*) from module_secret + where operator_key is not null and operator_key <> $1`, public).Scan(&orphaned); err != nil { + return 0, err + } + if _, err := tx.Exec(ctx, `delete from operator_key where public <> $1`, public); err != nil { + return 0, err + } + if _, err := tx.Exec(ctx, + `insert into operator_key (public) values ($1) on conflict (public) do nothing`, public); err != nil { + return 0, err + } + return orphaned, tx.Commit(ctx) +} + +// Kept is the catalogue's: one secret as the operator can recover it. +type Kept = catalogue.Kept + +// KeptForOperator is every secret the operator can recover, and which cannot. +// +// The second list is the honest half: a secret minted before the mesh had an operator key has no +// operator-sealed copy and cannot get one — the plaintext was discarded. Naming those is what lets +// an export say what it does not cover, rather than being taken for complete. +func (i *Inventory) KeptForOperator(ctx context.Context) (kept []Kept, unrecoverable []Kept, err error) { + rows, err := i.store.Pool().Query(ctx, + `select n.name, s.module, s.name, s.origin, coalesce(s.operator_sealed, ''), + coalesce(s.operator_key, ''), s.made_at + from module_secret s join node n on n.id = s.node + order by n.name, s.module, s.name`) + if err != nil { + return nil, nil, err + } + defer rows.Close() + for rows.Next() { + var k Kept + if err := rows.Scan(&k.Node, &k.Module, &k.Name, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt); err != nil { + return nil, nil, err + } + if k.Sealed == "" { + unrecoverable = append(unrecoverable, k) + continue + } + kept = append(kept, k) + } + return kept, unrecoverable, rows.Err() +} + +// KeptSecret is one secret's operator-sealed copy, for recovery. +func (i *Inventory) KeptSecret(ctx context.Context, node, module, name string) (Kept, error) { + var k Kept + err := i.store.Pool().QueryRow(ctx, + `select n.name, s.module, s.name, s.origin, coalesce(s.operator_sealed, ''), + coalesce(s.operator_key, ''), s.made_at + from module_secret s join node n on n.id = s.node + where n.name = $1 and s.module = $2 and s.name = $3`, node, module, name). + Scan(&k.Node, &k.Module, &k.Name, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt) + if errors.Is(err, pgx.ErrNoRows) { + return Kept{}, fmt.Errorf("%s on %s holds nothing called %q", module, node, name) + } + if err != nil { + return Kept{}, err + } + if k.Sealed == "" { + return Kept{}, fmt.Errorf( + "%s on %s holds %q, but it was made before the mesh had an operator key and so has no "+ + "copy a person can open. Issue it again (secret accept, or let the mesh remake it) "+ + "and it will", module, node, name) + } + return k, nil +} diff --git a/internal/inventory/operator_test.go b/internal/inventory/operator_test.go new file mode 100644 index 0000000..600fd7c --- /dev/null +++ b/internal/inventory/operator_test.go @@ -0,0 +1,97 @@ +package inventory + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/secrets" +) + +// With an operator key, a module's own secret is sealed to the operator as well — and the operator +// opens exactly the value the node was given. +func TestAnOwnSecretIsSealedToTheOperatorToo(t *testing.T) { + inv, ctx := twoNodesWithKeys(t) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"}, Source{}); err != nil { + t.Fatal(err) + } + + // Before there is a key: minted, and honestly unrecoverable. + if _, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser"); err != nil { + t.Fatal(err) + } + if _, err := inv.KeptSecret(ctx, "consumer", "postgres", "superuser"); err == nil { + t.Fatal("a secret made before the operator key was reported recoverable") + } + kept, unrecoverable, err := inv.KeptForOperator(ctx) + if err != nil { + t.Fatal(err) + } + if len(kept) != 0 || len(unrecoverable) != 1 { + t.Fatalf("before a key: %d kept, %d unrecoverable", len(kept), len(unrecoverable)) + } + + pub, priv, err := secrets.Keypair() + if err != nil { + t.Fatal(err) + } + if orphaned, err := inv.SetOperatorKey(ctx, pub); err != nil || orphaned != 0 { + t.Fatalf("set: orphaned %d, %v", orphaned, err) + } + + // A new secret is sealed to both; an accepted one too. + if _, err := inv.SecretForModule(ctx, "provider", "postgres", "superuser"); err != nil { + t.Fatal(err) + } + if err := inv.AcceptSecretForModule(ctx, "provider", "postgres", "replication", "given-by-a-person"); err != nil { + t.Fatal(err) + } + kept, unrecoverable, err = inv.KeptForOperator(ctx) + if err != nil { + t.Fatal(err) + } + if len(kept) != 2 || len(unrecoverable) != 1 { + t.Fatalf("after a key: %d kept, %d unrecoverable", len(kept), len(unrecoverable)) + } + got, err := inv.KeptSecret(ctx, "provider", "postgres", "replication") + if err != nil { + t.Fatal(err) + } + value, err := secrets.Open(priv, got.Sealed) + if err != nil { + t.Fatal(err) + } + if string(value) != "given-by-a-person" { + t.Fatalf("recovered %q", value) + } + if got.Origin != "accepted" || got.Key != pub { + t.Fatalf("kept as %+v", got) + } + minted, err := inv.KeptSecret(ctx, "provider", "postgres", "superuser") + if err != nil { + t.Fatal(err) + } + if v, err := secrets.Open(priv, minted.Sealed); err != nil || len(v) != 40 { + t.Fatalf("the minted secret did not open to a 40-character value: %v", err) + } + + // The old secret, remade for a rejoined node, becomes recoverable — it was issued again. + if _, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser"); err != nil { + t.Fatal(err) + } + if _, err := inv.KeptSecret(ctx, "consumer", "postgres", "superuser"); err == nil { + t.Fatal("asking again did not remake, yet it became recoverable") + } + + // Replacing the key says how many secrets stay sealed to the old one. + pub2, _, _ := secrets.Keypair() + orphaned, err := inv.SetOperatorKey(ctx, pub2) + if err != nil { + t.Fatal(err) + } + if orphaned != 2 { + t.Fatalf("replacing the key orphaned %d, and two were sealed to it", orphaned) + } + if now, _ := inv.OperatorKey(ctx); now != pub2 { + t.Fatal("the new key is not the mesh's key") + } +} diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index 9756459..173b03b 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -227,19 +227,33 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri module, node, name, node) } - made, err := secrets.Make(key, key) + operator, err := i.OperatorKey(ctx) if err != nil { return "", err } - // Sealed once, to one recipient. Make seals to two ends because a provision has two; here - // both are the same machine, and only one copy is kept. + var also []string + if operator != "" { + also = append(also, operator) + } + made, more, err := secrets.MakeAlso(key, key, also...) + if err != nil { + return "", err + } + // Sealed once to the machine — Make seals to two ends because a provision has two; here both + // are the same machine, and only one copy is kept — and once more to the operator when the + // mesh has one (novox/hq ADR 0085, amended), which is the copy a person can recover from. + var forOperator, operatorKey *string + if operator != "" { + forOperator, operatorKey = &more[0], &operator + } if _, err := i.store.Pool().Exec(ctx, - `insert into module_secret (node, module, name, sealed, node_key, origin) - values ($1, $2, $3, $4, $5, 'made') + `insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key) + values ($1, $2, $3, $4, $5, 'made', $6, $7) on conflict (node, module, name) do update set sealed = excluded.sealed, node_key = excluded.node_key, - origin = excluded.origin, made_at = now()`, - record.ID, module, name, made.ForConsumer, key); err != nil { + origin = excluded.origin, made_at = now(), + operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`, + record.ID, module, name, made.ForConsumer, key, forOperator, operatorKey); err != nil { return "", err } return made.ForConsumer, nil @@ -273,13 +287,28 @@ func (i *Inventory) AcceptSecretForModule(ctx context.Context, node, module, nam if err != nil { return err } + // And to the operator, when the mesh has one: a value a person supplied is the one a person + // most needs to get back, since the mesh cannot make another (novox/hq ADR 0085, amended). + operator, err := i.OperatorKey(ctx) + if err != nil { + return err + } + var forOperator, operatorKey *string + if operator != "" { + blob, err := secrets.Seal(operator, []byte(value)) + if err != nil { + return err + } + forOperator, operatorKey = &blob, &operator + } _, err = i.store.Pool().Exec(ctx, - `insert into module_secret (node, module, name, sealed, node_key, origin) - values ($1, $2, $3, $4, $5, 'accepted') + `insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key) + values ($1, $2, $3, $4, $5, 'accepted', $6, $7) on conflict (node, module, name) do update set sealed = excluded.sealed, node_key = excluded.node_key, - origin = excluded.origin, made_at = now()`, - record.ID, module, name, sealed.ForConsumer, key) + origin = excluded.origin, made_at = now(), + operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`, + record.ID, module, name, sealed.ForConsumer, key, forOperator, operatorKey) return err } diff --git a/internal/secrets/operator_test.go b/internal/secrets/operator_test.go new file mode 100644 index 0000000..41336e3 --- /dev/null +++ b/internal/secrets/operator_test.go @@ -0,0 +1,72 @@ +package secrets + +import "testing" + +// A secret sealed to the operator as well is opened by the operator's key and by nothing else. +func TestAThirdRecipientOpensWithItsOwnKeyOnly(t *testing.T) { + nodePub, nodePriv, err := Keypair() + if err != nil { + t.Fatal(err) + } + opPub, opPriv, err := Keypair() + if err != nil { + t.Fatal(err) + } + sealed, more, err := MakeAlso(nodePub, nodePub, opPub) + if err != nil { + t.Fatal(err) + } + if len(more) != 1 { + t.Fatalf("%d extra blobs for one extra key", len(more)) + } + fromNode, err := Open(nodePriv, sealed.ForConsumer) + if err != nil { + t.Fatal(err) + } + fromOperator, err := Open(opPriv, more[0]) + if err != nil { + t.Fatal(err) + } + if string(fromNode) != string(fromOperator) { + t.Fatal("the operator's copy is a different value from the node's") + } + if len(fromNode) != 40 { + t.Fatalf("a minted value is %d characters, not 40", len(fromNode)) + } + if _, err := Open(nodePriv, more[0]); err == nil { + t.Fatal("the node's key opened the operator's blob") + } + if _, err := Open(opPriv, sealed.ForConsumer); err == nil { + t.Fatal("the operator's key opened the node's blob") + } +} + +// An accepted value, sealed to the operator, comes back byte for byte. +func TestAnAcceptedValueRoundTripsThroughTheOperatorKey(t *testing.T) { + opPub, opPriv, err := Keypair() + if err != nil { + t.Fatal(err) + } + blob, err := Seal(opPub, []byte(" the-superuser's password, spaces and all ")) + if err != nil { + t.Fatal(err) + } + got, err := Open(opPriv, blob) + if err != nil { + t.Fatal(err) + } + if string(got) != " the-superuser's password, spaces and all " { + t.Fatalf("got %q", got) + } +} + +func TestAFingerprintNamesAKeyAndIsNotOne(t *testing.T) { + pub, _, _ := Keypair() + fp := Fingerprint(pub) + if len(fp) != len("sha256:")+16 || fp[:7] != "sha256:" { + t.Fatalf("fingerprint %q", fp) + } + if fp == Fingerprint(pub+"x") { + t.Fatal("two keys, one fingerprint") + } +} diff --git a/internal/secrets/seal.go b/internal/secrets/seal.go index ed9f07a..3dff1de 100644 --- a/internal/secrets/seal.go +++ b/internal/secrets/seal.go @@ -1,8 +1,11 @@ package secrets import ( + "crypto/ecdh" "crypto/rand" + "crypto/sha256" "encoding/base64" + "encoding/hex" "fmt" "strings" @@ -48,10 +51,22 @@ type Sealed struct { // rather than reading the old one back — the only version of rotation that is honest about what // the mesh knows. func Make(consumerKey, providerKey string) (Sealed, error) { + sealed, _, err := MakeAlso(consumerKey, providerKey) + return sealed, err +} + +// MakeAlso is Make with further recipients: the same fresh value, sealed once more to each key in +// `also`, returned in that order. +// +// **For the operator key, and nothing else so far** (novox/hq ADR 0085, amended). A secret a module +// holds for itself is sealed to its node and, when the mesh has an operator key, to that as well — +// so a person holding the key can recover it when the node cannot. The plaintext still exists only +// inside this call; a third blob is one more thing the mesh cannot open, not one more copy it can. +func MakeAlso(consumerKey, providerKey string, also ...string) (Sealed, []string, error) { if consumerKey == "" || providerKey == "" { // Sealing to an empty key would produce a blob nobody can open, stored as though it were // a working credential. The caller knows which node is which and says so. - return Sealed{}, fmt.Errorf("both ends need a sealing key before a secret can be made") + return Sealed{}, nil, fmt.Errorf("both ends need a sealing key before a secret can be made") } // 30 bytes, not 32: base64url of 30 is exactly 40 characters, and 40 is the longest secret an @@ -59,7 +74,7 @@ func Make(consumerKey, providerKey string) (Sealed, error) { // "fit the tightest backend" rule ADR 0049 sets for the login, on the secret. 240 bits is ample. value := make([]byte, 30) if _, err := rand.Read(value); err != nil { - return Sealed{}, err + return Sealed{}, nil, err } // Base64 without padding, because it lands in a configuration file something else parses and // a password containing a newline or a quote is a support call. @@ -67,16 +82,24 @@ func Make(consumerKey, providerKey string) (Sealed, error) { forConsumer, err := Seal(consumerKey, []byte(password)) if err != nil { - return Sealed{}, err + return Sealed{}, nil, err } forProvider, err := Seal(providerKey, []byte(password)) if err != nil { - return Sealed{}, err + return Sealed{}, nil, err + } + more := make([]string, 0, len(also)) + for _, key := range also { + blob, err := Seal(key, []byte(password)) + if err != nil { + return Sealed{}, nil, err + } + more = append(more, blob) } return Sealed{ ForConsumer: forConsumer, ForProvider: forProvider, ConsumerKey: consumerKey, ProviderKey: providerKey, - }, nil + }, more, nil } // Accept seals a value somebody supplied, rather than one the mesh made. @@ -115,6 +138,52 @@ func Accept(value string, consumerKey, providerKey string) (Sealed, error) { }, nil } +// Open is the other half of Seal, for the one holder of a private key this program ever acts for: +// the operator, recovering a secret with the key that never entered the mesh (novox/hq ADR 0085, +// amended). A node opens its own blobs in the host; the controller opens nothing of a node's, and +// cannot — it has no node's private key, which is the whole point of sealing. +func Open(privateKey string, sealed string) ([]byte, error) { + private, err := base64.StdEncoding.DecodeString(strings.TrimSpace(privateKey)) + if err != nil || len(private) != 32 { + return nil, fmt.Errorf("that is not a sealing key") + } + key, err := ecdh.X25519().NewPrivateKey(private) + if err != nil { + return nil, fmt.Errorf("that is not a usable sealing key: %w", err) + } + blob, err := base64.StdEncoding.DecodeString(sealed) + if err != nil { + return nil, fmt.Errorf("this is not a sealed value: %w", err) + } + var pub, priv [32]byte + copy(pub[:], key.PublicKey().Bytes()) + copy(priv[:], private) + out, ok := box.OpenAnonymous(nil, blob, &pub, &priv) + if !ok { + return nil, fmt.Errorf("this was not sealed to that key") + } + return out, nil +} + +// Keypair makes a sealing keypair for a holder outside the mesh — the operator. The private half is +// returned to be written where the caller says and nowhere else; the public half is what the mesh +// records and seals to. +func Keypair() (public, private string, err error) { + key, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + return "", "", err + } + return base64.StdEncoding.EncodeToString(key.PublicKey().Bytes()), + base64.StdEncoding.EncodeToString(key.Bytes()), nil +} + +// Fingerprint names a public key without being one: the first bytes of its hash, so two people can +// agree which key they mean out loud. +func Fingerprint(publicKey string) string { + sum := sha256.Sum256([]byte(strings.TrimSpace(publicKey))) + return "sha256:" + hex.EncodeToString(sum[:8]) +} + // Seal closes a value to a node's public sealing key. func Seal(publicKey string, value []byte) (string, error) { public, err := base64.StdEncoding.DecodeString(publicKey) From 565f144a207bfd46bf1104c27bdfba1624bd65ee Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 00:36:16 +0200 Subject: [PATCH 2/3] A pair credential is sealed to the operator key too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The secret the vault provides a module is the credential of the consumer↔vault pair, and so is every credential a provider grants; sealing only own secrets to the operator left exactly those unrecoverable. Same column, same call; the export and `secret recover` address a pair by consumer node, module and the provision's name, and say which kind each entry is. --- internal/catalogue/declaration.go | 9 +++ ...edential-is-sealed-to-the-operator-too.sql | 10 +++ internal/inventory/operator.go | 26 +++++-- internal/inventory/operator_test.go | 72 +++++++++++++++++++ internal/inventory/secrets.go | 25 +++++-- 5 files changed, 131 insertions(+), 11 deletions(-) create mode 100644 internal/inventory/migrations/0024-a-pair-credential-is-sealed-to-the-operator-too.sql diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index b4194db..ed635ce 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -739,10 +739,19 @@ func receivedFile(requirement, path string, given []Contribution) (map[string]an // Kept is one secret as the operator can recover it: where it belongs, and the value sealed to the // operator's key. Never a node's blob, and never a value. +// +// Two kinds, addressed the same way — by the node and module that hold it and the name they know +// it by. An `own` secret is one a module has for itself; a `pair` secret is a credential the +// module was granted for a provision it requires (`name` is the provision), and Provider says which +// node grants it. type Kept struct { Node string `json:"node"` Module string `json:"module"` Name string `json:"name"` + // Kind is `own` or `pair`. + Kind string `json:"kind"` + // Provider is the node granting a pair credential; empty for an own secret. + Provider string `json:"provider,omitempty"` // Origin is `made` or `accepted` — whether the mesh minted it or a person supplied it. Origin string `json:"origin"` // Sealed is the value, sealed to the operator key named in Key. diff --git a/internal/inventory/migrations/0024-a-pair-credential-is-sealed-to-the-operator-too.sql b/internal/inventory/migrations/0024-a-pair-credential-is-sealed-to-the-operator-too.sql new file mode 100644 index 0000000..f4b3ab7 --- /dev/null +++ b/internal/inventory/migrations/0024-a-pair-credential-is-sealed-to-the-operator-too.sql @@ -0,0 +1,10 @@ +-- The same second seal for a pair credential (novox/hq ADR 0085, amended). +-- +-- 0023 gave a module's own secrets an operator-sealed copy. A secret the vault provides to a module +-- is not an own secret -- it is the credential of the consumer-vault pair -- and so were the +-- credentials every provider grants. Without this, a vault-provided password could be rotated and +-- audited but not recovered, which is a vault that keeps everything except what it was for. + +alter table secret + add column operator_sealed text, + add column operator_key text; diff --git a/internal/inventory/operator.go b/internal/inventory/operator.go index 2092b89..848b77a 100644 --- a/internal/inventory/operator.go +++ b/internal/inventory/operator.go @@ -69,17 +69,21 @@ type Kept = catalogue.Kept // an export say what it does not cover, rather than being taken for complete. func (i *Inventory) KeptForOperator(ctx context.Context) (kept []Kept, unrecoverable []Kept, err error) { rows, err := i.store.Pool().Query(ctx, - `select n.name, s.module, s.name, s.origin, coalesce(s.operator_sealed, ''), + `select 'own', n.name, s.module, s.name, '', s.origin, coalesce(s.operator_sealed, ''), coalesce(s.operator_key, ''), s.made_at from module_secret s join node n on n.id = s.node - order by n.name, s.module, s.name`) + union all + select 'pair', c.name, s.consumer_module, s.name, p.name, 'made', coalesce(s.operator_sealed, ''), + coalesce(s.operator_key, ''), s.created_at + from secret s join node c on c.id = s.consumer join node p on p.id = s.provider + order by 1, 2, 3, 4`) if err != nil { return nil, nil, err } defer rows.Close() for rows.Next() { var k Kept - if err := rows.Scan(&k.Node, &k.Module, &k.Name, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt); err != nil { + if err := rows.Scan(&k.Kind, &k.Node, &k.Module, &k.Name, &k.Provider, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt); err != nil { return nil, nil, err } if k.Sealed == "" { @@ -93,15 +97,25 @@ func (i *Inventory) KeptForOperator(ctx context.Context) (kept []Kept, unrecover // KeptSecret is one secret's operator-sealed copy, for recovery. func (i *Inventory) KeptSecret(ctx context.Context, node, module, name string) (Kept, error) { + // An own secret first, then a pair credential by the provision's name. A module whose own + // secret and requirement share a name is refused at resolution, so the two cannot both answer. var k Kept err := i.store.Pool().QueryRow(ctx, - `select n.name, s.module, s.name, s.origin, coalesce(s.operator_sealed, ''), + `select 'own', n.name, s.module, s.name, '', s.origin, coalesce(s.operator_sealed, ''), coalesce(s.operator_key, ''), s.made_at from module_secret s join node n on n.id = s.node where n.name = $1 and s.module = $2 and s.name = $3`, node, module, name). - Scan(&k.Node, &k.Module, &k.Name, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt) + Scan(&k.Kind, &k.Node, &k.Module, &k.Name, &k.Provider, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt) if errors.Is(err, pgx.ErrNoRows) { - return Kept{}, fmt.Errorf("%s on %s holds nothing called %q", module, node, name) + err = i.store.Pool().QueryRow(ctx, + `select 'pair', c.name, s.consumer_module, s.name, p.name, 'made', coalesce(s.operator_sealed, ''), + coalesce(s.operator_key, ''), s.created_at + from secret s join node c on c.id = s.consumer join node p on p.id = s.provider + where c.name = $1 and s.consumer_module = $2 and s.name = $3`, node, module, name). + Scan(&k.Kind, &k.Node, &k.Module, &k.Name, &k.Provider, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt) + } + if errors.Is(err, pgx.ErrNoRows) { + return Kept{}, fmt.Errorf("%s on %s holds nothing called %q — neither a secret of its own nor a credential for a provision", module, node, name) } if err != nil { return Kept{}, err diff --git a/internal/inventory/operator_test.go b/internal/inventory/operator_test.go index 600fd7c..6ead7c5 100644 --- a/internal/inventory/operator_test.go +++ b/internal/inventory/operator_test.go @@ -1,6 +1,7 @@ package inventory import ( + "context" "testing" "github.com/novox/mesh-controller/internal/catalogue" @@ -95,3 +96,74 @@ func TestAnOwnSecretIsSealedToTheOperatorToo(t *testing.T) { t.Fatal("the new key is not the mesh's key") } } + +// A pair credential — what the vault provides a module — is sealed to the operator too, and the +// operator's copy is the very value the consumer's node unseals. +func TestAPairCredentialIsSealedToTheOperatorToo(t *testing.T) { + inv := fresh(t) + ctx := context.Background() + // Two nodes with keys, keeping the consumer's opener: the test made the key, so it can play the + // consumer's host for one assertion. + var openAsConsumer func(string) ([]byte, bool) + for _, n := range []string{"consumer", "provider"} { + node, err := inv.AddNode(ctx, n) + if err != nil { + t.Fatal(err) + } + key, open := aSealingKey(t) + if n == "consumer" { + openAsConsumer = open + } + if err := inv.RecordSealingKey(ctx, node.ID, key); err != nil { + t.Fatal(err) + } + } + for _, m := range []string{"gitea", "mesh-vault"} { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil { + t.Fatal(err) + } + } + pub, priv, err := secrets.Keypair() + if err != nil { + t.Fatal(err) + } + if _, err := inv.SetOperatorKey(ctx, pub); err != nil { + t.Fatal(err) + } + made, err := inv.SecretFor(ctx, "secret", "consumer", "gitea", "provider") + if err != nil { + t.Fatal(err) + } + kept, err := inv.KeptSecret(ctx, "consumer", "gitea", "secret") + if err != nil { + t.Fatal(err) + } + if kept.Kind != "pair" || kept.Provider != "provider" { + t.Fatalf("kept as %+v", kept) + } + fromOperator, err := secrets.Open(priv, kept.Sealed) + if err != nil { + t.Fatal(err) + } + // The consumer's blob is sealed to the consumer node. Same value, two recipients. + fromNode, ok := openAsConsumer(made.ForConsumer) + if !ok { + t.Fatal("the consumer cannot open its own blob") + } + if string(fromOperator) != string(fromNode) { + t.Fatal("the operator's copy of the pair credential differs from the consumer's") + } + all, _, err := inv.KeptForOperator(ctx) + if err != nil { + t.Fatal(err) + } + var pairs int + for _, k := range all { + if k.Kind == "pair" { + pairs++ + } + } + if pairs != 1 { + t.Fatalf("%d pair credential(s) in the export, expected 1", pairs) + } +} diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index 173b03b..182f0c7 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -74,20 +74,35 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModul return held, nil } - made, err := secrets.Make(consumerKey, providerKey) + // And to the operator, when the mesh has one (novox/hq ADR 0085, amended): the third copy that + // makes a vault-provided secret recoverable, and nothing the mesh can open. + operator, err := i.OperatorKey(ctx) if err != nil { return Secret{}, err } + var also []string + if operator != "" { + also = append(also, operator) + } + made, more, err := secrets.MakeAlso(consumerKey, providerKey, also...) + if err != nil { + return Secret{}, err + } + var forOperator, operatorKey *string + if operator != "" { + forOperator, operatorKey = &more[0], &operator + } _, err = i.store.Pool().Exec(ctx, `insert into secret (name, consumer, consumer_module, provider, for_consumer, for_provider, - consumer_key, provider_key) - values ($1, $2, $3, $4, $5, $6, $7, $8) + consumer_key, provider_key, operator_sealed, operator_key) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) on conflict (name, consumer, consumer_module, provider) do update set for_consumer = excluded.for_consumer, for_provider = excluded.for_provider, consumer_key = excluded.consumer_key, provider_key = excluded.provider_key, - created_at = now()`, + created_at = now(), + operator_sealed = excluded.operator_sealed, operator_key = excluded.operator_key`, name, consumerNode.ID, consumerModule, providerNode.ID, - made.ForConsumer, made.ForProvider, made.ConsumerKey, made.ProviderKey) + made.ForConsumer, made.ForProvider, made.ConsumerKey, made.ProviderKey, forOperator, operatorKey) if err != nil { return Secret{}, err } From 77e6c1a6845f097ed71ceb37d715c47d0c3f461e Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 21 Sep 2026 01:16:32 +0200 Subject: [PATCH 3/3] Recoverable means sealed to the current operator key; recovery names the provider From review: the export counted any operator-sealed row as recoverable, so a secret sealed to a replaced key was reported as openable with the current one; replacing the key counted orphans in one table of two; and a pair credential held from two providers was recovered as whichever row came first. The export now lists what the current key opens, what an earlier key opens, and what has no copy; `secret recover` takes --provider and refuses ambiguity; files that must not exist are created exclusively; one constructor builds the export for the operator's file and the vault's disk alike. --- cmd/mesh-controller/operator.go | 17 ++-- cmd/mesh-controller/plan.go | 20 ++--- cmd/mesh-controller/secret.go | 92 +++++++++++++++----- internal/catalogue/declaration.go | 10 ++- internal/inventory/operator.go | 127 +++++++++++++++++++++++----- internal/inventory/operator_test.go | 98 +++++++++++++++------ internal/inventory/secrets.go | 36 ++------ internal/secrets/operator_test.go | 13 +-- internal/secrets/seal.go | 36 ++++---- 9 files changed, 308 insertions(+), 141 deletions(-) diff --git a/cmd/mesh-controller/operator.go b/cmd/mesh-controller/operator.go index d46d60b..af037eb 100644 --- a/cmd/mesh-controller/operator.go +++ b/cmd/mesh-controller/operator.go @@ -52,14 +52,13 @@ func operatorKeyMake(args []string) error { if err := set.Parse(args); err != nil { return err } - if _, err := os.Stat(*out); err == nil { - return fmt.Errorf("%s already exists; this will not overwrite a key somebody may still need", *out) - } public, private, err := secrets.Keypair() if err != nil { return err } - if err := os.WriteFile(*out, []byte(private+"\n"), 0o600); err != nil { + // Create-exclusive: a key somebody may still need is never overwritten, and there is no window + // between checking and writing in which one could appear. + if err := writeNew(*out, []byte(private+"\n")); err != nil { return err } fmt.Printf("operator key %s\n", secrets.Fingerprint(public)) @@ -129,14 +128,20 @@ func operatorKeyShow(ctx context.Context) error { fmt.Println("the mesh has no operator key; `operator key make` then `operator key set` gives it one") return nil } - kept, unrecoverable, err := inv.KeptForOperator(ctx) + kept, earlier, unrecoverable, err := inv.KeptForOperator(ctx) if err != nil { return err } fmt.Printf("operator key %s\n %s\n", secrets.Fingerprint(key), key) fmt.Printf(" %d secret(s) recoverable with it\n", len(kept)) + if len(earlier) > 0 { + fmt.Printf(" %d secret(s) sealed to an earlier operator key — recoverable with that key only, until issued again:\n", len(earlier)) + for _, k := range earlier { + fmt.Printf(" %s %s %s (%s)\n", k.Node, k.Module, k.Name, secrets.Fingerprint(k.Key)) + } + } if len(unrecoverable) > 0 { - fmt.Printf(" %d secret(s) not recoverable — made before it, or sealed to an earlier key:\n", len(unrecoverable)) + fmt.Printf(" %d secret(s) not recoverable — made before the mesh had an operator key:\n", len(unrecoverable)) for _, k := range unrecoverable { fmt.Printf(" %s %s %s\n", k.Node, k.Module, k.Name) } diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index a5867b7..17eaa30 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -13,7 +13,6 @@ import ( "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/licences" - "github.com/novox/mesh-controller/internal/secrets" "net" "strconv" ) @@ -464,27 +463,18 @@ func declarationWith(ctx context.Context, open *stores, node string, } // And, for a module that keeps them, every operator-sealed secret in the mesh — the vault's - // copy, outside the store (novox/hq ADR 0085, amended). Read only; nothing here mints. + // copy, outside the store (novox/hq ADR 0085, amended). Read only; nothing here mints. The + // export changes whenever any secret in the mesh is made or rotated, so the vault's declaration + // changes with it and the vault node is sent again: that is what keeps its copy current, and + // the cost is one two-table read per composition of the vault's node, in every mode. var kept *catalogue.KeptExport for _, m := range plan.Modules { if m.Keeps == "" { continue } - operator, err := inv.OperatorKey(ctx) - if err != nil { + if kept, err = inv.OperatorExport(ctx); err != nil { return nil, err } - if operator == "" { - break // nothing is sealed to an operator, so there is nothing to keep yet - } - recoverable, unrecoverable, err := inv.KeptForOperator(ctx) - if err != nil { - return nil, err - } - kept = &catalogue.KeptExport{ - Export: 1, OperatorKey: operator, Fingerprint: secrets.Fingerprint(operator), - Kept: recoverable, Unrecoverable: unrecoverable, - } break } diff --git a/cmd/mesh-controller/secret.go b/cmd/mesh-controller/secret.go index 74a5ece..a8eb726 100644 --- a/cmd/mesh-controller/secret.go +++ b/cmd/mesh-controller/secret.go @@ -84,7 +84,7 @@ func secretCommand(ctx context.Context, args []string) error { } const secretUsage = "secret accept [--from ]\n" + - "secret recover --key [--out ] [--from-export ]\n" + + "secret recover --key [--out ] [--from-export ] [--provider ]\n" + "secret export [--out ]" // secretRecover is break-glass: a secret opened with the operator's key, written to a file. @@ -104,6 +104,7 @@ func secretRecover(ctx context.Context, args []string) error { keyFile := set.String("key", "", "the operator's private key, from `operator key make`") out := set.String("out", "", "where to write the value (0600); - for standard output. Default ...secret") fromExport := set.String("from-export", "", "read the sealed copy from this `secret export` file instead of the store") + provider := set.String("provider", "", "for a pair credential held from more than one provider: which one") if err := set.Parse(flags); err != nil { return err } @@ -118,7 +119,7 @@ func secretRecover(ctx context.Context, args []string) error { var kept inventory.Kept if *fromExport != "" { - kept, err = keptFromExport(*fromExport, node, module, name) + kept, err = keptFromExport(*fromExport, node, module, name, *provider) if err != nil { return err } @@ -128,7 +129,7 @@ func secretRecover(ctx context.Context, args []string) error { return err } defer open.Close() - kept, err = open.inventory.KeptSecret(ctx, node, module, name) + kept, err = open.inventory.KeptSecret(ctx, node, module, name, *provider) if err != nil { return err } @@ -147,10 +148,7 @@ func secretRecover(ctx context.Context, args []string) error { if path == "" { path = node + "." + module + "." + name + ".secret" } - if _, err := os.Stat(path); err == nil { - return fmt.Errorf("%s already exists; not overwriting it", path) - } - if err := os.WriteFile(path, value, 0o600); err != nil { + if err := writeNew(path, value); err != nil { return err } fmt.Printf("%s on %s: %q recovered to %s (0600) — %d bytes, origin %s\n", @@ -181,14 +179,11 @@ func secretExport(ctx context.Context, args []string) error { if key == "" { return errors.New("the mesh has no operator key, so nothing is sealed to one; `operator key make` and `operator key set` first") } - kept, unrecoverable, err := inv.KeptForOperator(ctx) + doc, err := inv.OperatorExport(ctx) if err != nil { return err } - body, err := json.MarshalIndent(export{ - Export: 1, OperatorKey: key, Fingerprint: secrets.Fingerprint(key), - Kept: kept, Unrecoverable: unrecoverable, - }, "", " ") + body, err := json.MarshalIndent(doc, "", " ") if err != nil { return err } @@ -197,18 +192,58 @@ func secretExport(ctx context.Context, args []string) error { _, err := os.Stdout.Write(body) return err } - if err := os.WriteFile(*out, body, 0o600); err != nil { + // Replaced whole, and made 0600 whether or not it existed: an export is ciphertext and a public + // key, but it is also the list of every secret the mesh has, and a file left at an earlier mode + // while the command says 0600 is a lie in the one place a person checks. + if err := writeReplacing(*out, body); err != nil { return err } fmt.Printf("%d secret(s) exported to %s (0600), sealed to operator key %s — ciphertext, keep it with the key\n", - len(kept), *out, secrets.Fingerprint(key)) - if len(unrecoverable) > 0 { - fmt.Printf(" %d secret(s) are NOT in it: made before the mesh had an operator key\n", len(unrecoverable)) + len(doc.Kept), *out, doc.Fingerprint) + if len(doc.EarlierKey) > 0 { + fmt.Printf(" %d secret(s) are sealed to an EARLIER operator key: recoverable with that key only\n", len(doc.EarlierKey)) + } + if len(doc.Unrecoverable) > 0 { + fmt.Printf(" %d secret(s) are NOT in it: made before the mesh had an operator key\n", len(doc.Unrecoverable)) } return nil } -func keptFromExport(path, node, module, name string) (inventory.Kept, error) { +// writeNew writes a file that must not exist yet, atomically: create-exclusive, 0600. A check +// followed by a write is a window in which a key somebody still needs can be overwritten. +func writeNew(path string, content []byte) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + if os.IsExist(err) { + return fmt.Errorf("%s already exists; not overwriting it", path) + } + return err + } + if _, err := f.Write(content); err != nil { + f.Close() + return err + } + return f.Close() +} + +// writeReplacing writes a file whole, creating or truncating it, and leaves it at 0600 either way. +func writeReplacing(path string, content []byte) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return err + } + if _, err := f.Write(content); err != nil { + f.Close() + return err + } + if err := f.Chmod(0o600); err != nil { + f.Close() + return err + } + return f.Close() +} + +func keptFromExport(path, node, module, name, provider string) (inventory.Kept, error) { raw, err := os.ReadFile(path) if err != nil { return inventory.Kept{}, err @@ -217,12 +252,27 @@ func keptFromExport(path, node, module, name string) (inventory.Kept, error) { if err := json.Unmarshal(raw, &e); err != nil { return inventory.Kept{}, fmt.Errorf("%s is not a secret export: %w", path, err) } - for _, k := range e.Kept { - if k.Node == node && k.Module == module && k.Name == name { - return k, nil + // Sealed to the current key or to an earlier one: both are copies the given key might open, + // and Open says which. Not the unrecoverable list, which holds no copy at all. + var found []inventory.Kept + for _, k := range append(append([]inventory.Kept{}, e.Kept...), e.EarlierKey...) { + if k.Node == node && k.Module == module && k.Name == name && (provider == "" || k.Provider == provider) { + found = append(found, k) } } - return inventory.Kept{}, fmt.Errorf("%s holds no copy of %s's %q on %s", path, module, name, node) + switch len(found) { + case 0: + return inventory.Kept{}, fmt.Errorf("%s holds no copy of %s's %q on %s", path, module, name, node) + case 1: + return found[0], nil + default: + providers := make([]string, 0, len(found)) + for _, f := range found { + providers = append(providers, f.Provider) + } + return inventory.Kept{}, fmt.Errorf("%s holds %s's %q on %s from more than one provider (%s); say which with --provider", + path, module, name, node, strings.Join(providers, ", ")) + } } // split separates what this command is about from how it was asked. diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index ed635ce..c6cf0f3 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -763,10 +763,14 @@ type Kept struct { // KeptExport is what a person keeps beside the operator key, and what a vault keeps on its disk: // every operator-sealed copy, and the honest list of what has none. type KeptExport struct { - Export int `json:"export"` - OperatorKey string `json:"operator-key"` - Fingerprint string `json:"fingerprint"` + Export int `json:"export"` + OperatorKey string `json:"operator-key"` + Fingerprint string `json:"fingerprint"` + // Kept is sealed to OperatorKey. EarlierKey is sealed to a key the mesh has since replaced — + // recoverable with that key, if the person still has it, and with nothing else. Unrecoverable + // has no operator copy at all. Kept []Kept `json:"kept"` + EarlierKey []Kept `json:"sealed-to-earlier-key,omitempty"` Unrecoverable []Kept `json:"unrecoverable,omitempty"` } diff --git a/internal/inventory/operator.go b/internal/inventory/operator.go index 848b77a..7e6b0b5 100644 --- a/internal/inventory/operator.go +++ b/internal/inventory/operator.go @@ -4,10 +4,12 @@ import ( "context" "errors" "fmt" + "strings" "github.com/jackc/pgx/v5" "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/secrets" ) // The operator's sealing key: the one holder of secrets that is not a node. @@ -18,6 +20,29 @@ import ( // What is recorded here is the public half, which is all the mesh needs to seal to it; what it // yields is one more blob per secret that the mesh cannot open. +// operatorColumns is the pair of nullable columns a secret row carries for its operator copy: +// both null when the mesh has no operator key, so a row says plainly that no such copy exists. +func operatorColumns(operator, blob string) (sealed, key *string) { + if operator == "" || blob == "" { + return nil, nil + } + return &blob, &operator +} + +// operatorSeal seals a value somebody supplied to the operator key, when the mesh has one. +func (i *Inventory) operatorSeal(ctx context.Context, value string) (sealed, key *string, err error) { + operator, err := i.OperatorKey(ctx) + if err != nil || operator == "" { + return nil, nil, err + } + blob, err := secrets.Seal(operator, []byte(value)) + if err != nil { + return nil, nil, err + } + sealed, key = operatorColumns(operator, blob) + return sealed, key, nil +} + // OperatorKey is the public key secrets are also sealed to, or empty when the mesh has none. func (i *Inventory) OperatorKey(ctx context.Context) (string, error) { var key string @@ -44,9 +69,12 @@ func (i *Inventory) SetOperatorKey(ctx context.Context, public string) (orphaned return 0, err } defer tx.Rollback(ctx) + // Both tables: a module's own secrets and the pair credentials. A count over one of them said + // "nothing orphaned" about a mesh whose every vault-provided secret had just been. if err := tx.QueryRow(ctx, - `select count(*) from module_secret - where operator_key is not null and operator_key <> $1`, public).Scan(&orphaned); err != nil { + `select (select count(*) from module_secret where operator_key is not null and operator_key <> $1) + + (select count(*) from secret where operator_key is not null and operator_key <> $1)`, + public).Scan(&orphaned); err != nil { return 0, err } if _, err := tx.Exec(ctx, `delete from operator_key where public <> $1`, public); err != nil { @@ -62,12 +90,38 @@ func (i *Inventory) SetOperatorKey(ctx context.Context, public string) (orphaned // Kept is the catalogue's: one secret as the operator can recover it. type Kept = catalogue.Kept -// KeptForOperator is every secret the operator can recover, and which cannot. +// OperatorExport is the export as the operator and the vault both keep it: every secret sealed to +// the mesh's current operator key, every one sealed to an earlier key (recoverable with that key, +// if the person still has it), and every one with no operator copy at all. Nil when the mesh has +// no operator key. One constructor, so the file `secret export` writes and the file the mesh puts +// on the vault's disk cannot drift apart. +func (i *Inventory) OperatorExport(ctx context.Context) (*catalogue.KeptExport, error) { + operator, err := i.OperatorKey(ctx) + if err != nil || operator == "" { + return nil, err + } + kept, earlier, unrecoverable, err := i.KeptForOperator(ctx) + if err != nil { + return nil, err + } + return &catalogue.KeptExport{ + Export: 1, OperatorKey: operator, Fingerprint: secrets.Fingerprint(operator), + Kept: kept, EarlierKey: earlier, Unrecoverable: unrecoverable, + }, nil +} + +// KeptForOperator is every secret by what can open it: the mesh's current operator key, an +// earlier operator key, or nothing. // -// The second list is the honest half: a secret minted before the mesh had an operator key has no -// operator-sealed copy and cannot get one — the plaintext was discarded. Naming those is what lets -// an export say what it does not cover, rather than being taken for complete. -func (i *Inventory) KeptForOperator(ctx context.Context) (kept []Kept, unrecoverable []Kept, err error) { +// The last two are the honest half. A secret minted before the mesh had an operator key has no +// operator-sealed copy and cannot get one — the plaintext was discarded; one sealed to a key the +// mesh has since replaced is not opened by the current key, however the export is labelled. Naming +// both is what lets an export say what it does not cover, rather than being taken for complete. +func (i *Inventory) KeptForOperator(ctx context.Context) (kept, earlier, unrecoverable []Kept, err error) { + current, err := i.OperatorKey(ctx) + if err != nil { + return nil, nil, nil, err + } rows, err := i.store.Pool().Query(ctx, `select 'own', n.name, s.module, s.name, '', s.origin, coalesce(s.operator_sealed, ''), coalesce(s.operator_key, ''), s.made_at @@ -78,27 +132,34 @@ func (i *Inventory) KeptForOperator(ctx context.Context) (kept []Kept, unrecover from secret s join node c on c.id = s.consumer join node p on p.id = s.provider order by 1, 2, 3, 4`) if err != nil { - return nil, nil, err + return nil, nil, nil, err } defer rows.Close() for rows.Next() { var k Kept if err := rows.Scan(&k.Kind, &k.Node, &k.Module, &k.Name, &k.Provider, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt); err != nil { - return nil, nil, err + return nil, nil, nil, err } - if k.Sealed == "" { + switch { + case k.Sealed == "": unrecoverable = append(unrecoverable, k) - continue + case k.Key != current: + earlier = append(earlier, k) + default: + kept = append(kept, k) } - kept = append(kept, k) } - return kept, unrecoverable, rows.Err() + return kept, earlier, unrecoverable, rows.Err() } // KeptSecret is one secret's operator-sealed copy, for recovery. -func (i *Inventory) KeptSecret(ctx context.Context, node, module, name string) (Kept, error) { - // An own secret first, then a pair credential by the provision's name. A module whose own - // secret and requirement share a name is refused at resolution, so the two cannot both answer. +// +// An own secret first, then a pair credential by the provision's name — a module whose own secret +// and requirement share a name is refused at resolution, so the two cannot both answer. A pair +// credential is keyed by provider as well, and a consumer whose provision moved leaves the old +// provider's row behind: two rows is refused with both providers named, never answered with +// whichever came first, unless `provider` says which. +func (i *Inventory) KeptSecret(ctx context.Context, node, module, name, provider string) (Kept, error) { var k Kept err := i.store.Pool().QueryRow(ctx, `select 'own', n.name, s.module, s.name, '', s.origin, coalesce(s.operator_sealed, ''), @@ -107,12 +168,40 @@ func (i *Inventory) KeptSecret(ctx context.Context, node, module, name string) ( where n.name = $1 and s.module = $2 and s.name = $3`, node, module, name). Scan(&k.Kind, &k.Node, &k.Module, &k.Name, &k.Provider, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt) if errors.Is(err, pgx.ErrNoRows) { - err = i.store.Pool().QueryRow(ctx, + rows, qerr := i.store.Pool().Query(ctx, `select 'pair', c.name, s.consumer_module, s.name, p.name, 'made', coalesce(s.operator_sealed, ''), coalesce(s.operator_key, ''), s.created_at from secret s join node c on c.id = s.consumer join node p on p.id = s.provider - where c.name = $1 and s.consumer_module = $2 and s.name = $3`, node, module, name). - Scan(&k.Kind, &k.Node, &k.Module, &k.Name, &k.Provider, &k.Origin, &k.Sealed, &k.Key, &k.MadeAt) + where c.name = $1 and s.consumer_module = $2 and s.name = $3 and ($4 = '' or p.name = $4) + order by p.name`, node, module, name, provider) + if qerr != nil { + return Kept{}, qerr + } + defer rows.Close() + var found []Kept + for rows.Next() { + var row Kept + if err := rows.Scan(&row.Kind, &row.Node, &row.Module, &row.Name, &row.Provider, &row.Origin, &row.Sealed, &row.Key, &row.MadeAt); err != nil { + return Kept{}, err + } + found = append(found, row) + } + if err := rows.Err(); err != nil { + return Kept{}, err + } + switch len(found) { + case 0: + err = pgx.ErrNoRows + case 1: + k, err = found[0], nil + default: + providers := make([]string, 0, len(found)) + for _, f := range found { + providers = append(providers, f.Provider) + } + return Kept{}, fmt.Errorf("%s on %s holds a %q credential from more than one provider (%s); say which with --provider", + module, node, name, strings.Join(providers, ", ")) + } } if errors.Is(err, pgx.ErrNoRows) { return Kept{}, fmt.Errorf("%s on %s holds nothing called %q — neither a secret of its own nor a credential for a provision", module, node, name) diff --git a/internal/inventory/operator_test.go b/internal/inventory/operator_test.go index 6ead7c5..4db167a 100644 --- a/internal/inventory/operator_test.go +++ b/internal/inventory/operator_test.go @@ -2,6 +2,7 @@ package inventory import ( "context" + "strings" "testing" "github.com/novox/mesh-controller/internal/catalogue" @@ -20,10 +21,10 @@ func TestAnOwnSecretIsSealedToTheOperatorToo(t *testing.T) { if _, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser"); err != nil { t.Fatal(err) } - if _, err := inv.KeptSecret(ctx, "consumer", "postgres", "superuser"); err == nil { + if _, err := inv.KeptSecret(ctx, "consumer", "postgres", "superuser", ""); err == nil { t.Fatal("a secret made before the operator key was reported recoverable") } - kept, unrecoverable, err := inv.KeptForOperator(ctx) + kept, _, unrecoverable, err := inv.KeptForOperator(ctx) if err != nil { t.Fatal(err) } @@ -46,14 +47,14 @@ func TestAnOwnSecretIsSealedToTheOperatorToo(t *testing.T) { if err := inv.AcceptSecretForModule(ctx, "provider", "postgres", "replication", "given-by-a-person"); err != nil { t.Fatal(err) } - kept, unrecoverable, err = inv.KeptForOperator(ctx) + kept, _, unrecoverable, err = inv.KeptForOperator(ctx) if err != nil { t.Fatal(err) } if len(kept) != 2 || len(unrecoverable) != 1 { t.Fatalf("after a key: %d kept, %d unrecoverable", len(kept), len(unrecoverable)) } - got, err := inv.KeptSecret(ctx, "provider", "postgres", "replication") + got, err := inv.KeptSecret(ctx, "provider", "postgres", "replication", "") if err != nil { t.Fatal(err) } @@ -67,7 +68,7 @@ func TestAnOwnSecretIsSealedToTheOperatorToo(t *testing.T) { if got.Origin != "accepted" || got.Key != pub { t.Fatalf("kept as %+v", got) } - minted, err := inv.KeptSecret(ctx, "provider", "postgres", "superuser") + minted, err := inv.KeptSecret(ctx, "provider", "postgres", "superuser", "") if err != nil { t.Fatal(err) } @@ -75,26 +76,59 @@ func TestAnOwnSecretIsSealedToTheOperatorToo(t *testing.T) { t.Fatalf("the minted secret did not open to a 40-character value: %v", err) } - // The old secret, remade for a rejoined node, becomes recoverable — it was issued again. + // The old secret is kept, not resealed: asking again is a read, the plaintext is gone, and it + // stays honestly unrecoverable. if _, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser"); err != nil { t.Fatal(err) } - if _, err := inv.KeptSecret(ctx, "consumer", "postgres", "superuser"); err == nil { + if _, err := inv.KeptSecret(ctx, "consumer", "postgres", "superuser", ""); err == nil { t.Fatal("asking again did not remake, yet it became recoverable") } + // Until the node rejoins with a new sealing key: then the secret is remade, and the remake is + // sealed to the operator — the one scenario the vault exists for. + rejoined, err := inv.NodeByName(ctx, "consumer") + if err != nil { + t.Fatal(err) + } + newKey, _ := aSealingKey(t) + if err := inv.RecordSealingKey(ctx, rejoined.ID, newKey); err != nil { + t.Fatal(err) + } + if _, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser"); err != nil { + t.Fatal(err) + } + remade, err := inv.KeptSecret(ctx, "consumer", "postgres", "superuser", "") + if err != nil { + t.Fatalf("the remade secret is not recoverable: %v", err) + } + if _, err := secrets.Open(priv, remade.Sealed); err != nil { + t.Fatal(err) + } - // Replacing the key says how many secrets stay sealed to the old one. + // Replacing the key says how many secrets stay sealed to the old one — and those move out of + // the recoverable list, whatever the export is labelled with. pub2, _, _ := secrets.Keypair() orphaned, err := inv.SetOperatorKey(ctx, pub2) if err != nil { t.Fatal(err) } - if orphaned != 2 { - t.Fatalf("replacing the key orphaned %d, and two were sealed to it", orphaned) + if orphaned != 3 { + t.Fatalf("replacing the key orphaned %d, and three were sealed to it", orphaned) } if now, _ := inv.OperatorKey(ctx); now != pub2 { t.Fatal("the new key is not the mesh's key") } + kept, earlier, _, err := inv.KeptForOperator(ctx) + if err != nil { + t.Fatal(err) + } + if len(kept) != 0 || len(earlier) != 3 { + t.Fatalf("after replacing the key: %d recoverable with it, %d sealed to the earlier key", len(kept), len(earlier)) + } + doc, err := inv.OperatorExport(ctx) + if err != nil || doc == nil || len(doc.EarlierKey) != 3 || len(doc.Kept) != 0 { + t.Fatalf("the export does not say what the current key cannot open: %+v %v", doc, err) + } } // A pair credential — what the vault provides a module — is sealed to the operator too, and the @@ -134,13 +168,42 @@ func TestAPairCredentialIsSealedToTheOperatorToo(t *testing.T) { if err != nil { t.Fatal(err) } - kept, err := inv.KeptSecret(ctx, "consumer", "gitea", "secret") + kept, err := inv.KeptSecret(ctx, "consumer", "gitea", "secret", "") if err != nil { t.Fatal(err) } if kept.Kind != "pair" || kept.Provider != "provider" { t.Fatalf("kept as %+v", kept) } + all, _, _, err := inv.KeptForOperator(ctx) + if err != nil { + t.Fatal(err) + } + var pairs int + for _, k := range all { + if k.Kind == "pair" { + pairs++ + } + } + if pairs != 1 { + t.Fatalf("%d pair credential(s) recoverable, expected 1", pairs) + } + + // A second provider of the same provision: two rows, refused rather than the first one taken, + // unless the provider is named. And replacing the key counts pair credentials as orphaned. + if _, err := inv.SecretFor(ctx, "secret", "consumer", "gitea", "consumer"); err != nil { + t.Fatal(err) + } + if _, err := inv.KeptSecret(ctx, "consumer", "gitea", "secret", ""); err == nil || !strings.Contains(err.Error(), "--provider") { + t.Fatalf("two providers were not refused: %v", err) + } + if byName, err := inv.KeptSecret(ctx, "consumer", "gitea", "secret", "provider"); err != nil || byName.Provider != "provider" { + t.Fatalf("naming the provider did not select it: %+v %v", byName, err) + } + pub2, _, _ := secrets.Keypair() + if orphaned, err := inv.SetOperatorKey(ctx, pub2); err != nil || orphaned != 2 { + t.Fatalf("replacing the key orphaned %d pair credential(s), and two were sealed to it (%v)", orphaned, err) + } fromOperator, err := secrets.Open(priv, kept.Sealed) if err != nil { t.Fatal(err) @@ -153,17 +216,4 @@ func TestAPairCredentialIsSealedToTheOperatorToo(t *testing.T) { if string(fromOperator) != string(fromNode) { t.Fatal("the operator's copy of the pair credential differs from the consumer's") } - all, _, err := inv.KeptForOperator(ctx) - if err != nil { - t.Fatal(err) - } - var pairs int - for _, k := range all { - if k.Kind == "pair" { - pairs++ - } - } - if pairs != 1 { - t.Fatalf("%d pair credential(s) in the export, expected 1", pairs) - } } diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go index 182f0c7..9f7caf7 100644 --- a/internal/inventory/secrets.go +++ b/internal/inventory/secrets.go @@ -80,18 +80,11 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModul if err != nil { return Secret{}, err } - var also []string - if operator != "" { - also = append(also, operator) - } - made, more, err := secrets.MakeAlso(consumerKey, providerKey, also...) + made, blob, err := secrets.MakeWithOperator(consumerKey, providerKey, operator) if err != nil { return Secret{}, err } - var forOperator, operatorKey *string - if operator != "" { - forOperator, operatorKey = &more[0], &operator - } + forOperator, operatorKey := operatorColumns(operator, blob) _, err = i.store.Pool().Exec(ctx, `insert into secret (name, consumer, consumer_module, provider, for_consumer, for_provider, consumer_key, provider_key, operator_sealed, operator_key) @@ -246,21 +239,14 @@ func (i *Inventory) SecretForModule(ctx context.Context, node, module, name stri if err != nil { return "", err } - var also []string - if operator != "" { - also = append(also, operator) - } - made, more, err := secrets.MakeAlso(key, key, also...) - if err != nil { - return "", err - } // Sealed once to the machine — Make seals to two ends because a provision has two; here both // are the same machine, and only one copy is kept — and once more to the operator when the // mesh has one (novox/hq ADR 0085, amended), which is the copy a person can recover from. - var forOperator, operatorKey *string - if operator != "" { - forOperator, operatorKey = &more[0], &operator + made, blob, err := secrets.MakeWithOperator(key, key, operator) + if err != nil { + return "", err } + forOperator, operatorKey := operatorColumns(operator, blob) if _, err := i.store.Pool().Exec(ctx, `insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key) values ($1, $2, $3, $4, $5, 'made', $6, $7) @@ -304,18 +290,10 @@ func (i *Inventory) AcceptSecretForModule(ctx context.Context, node, module, nam } // And to the operator, when the mesh has one: a value a person supplied is the one a person // most needs to get back, since the mesh cannot make another (novox/hq ADR 0085, amended). - operator, err := i.OperatorKey(ctx) + forOperator, operatorKey, err := i.operatorSeal(ctx, value) if err != nil { return err } - var forOperator, operatorKey *string - if operator != "" { - blob, err := secrets.Seal(operator, []byte(value)) - if err != nil { - return err - } - forOperator, operatorKey = &blob, &operator - } _, err = i.store.Pool().Exec(ctx, `insert into module_secret (node, module, name, sealed, node_key, origin, operator_sealed, operator_key) values ($1, $2, $3, $4, $5, 'accepted', $6, $7) diff --git a/internal/secrets/operator_test.go b/internal/secrets/operator_test.go index 41336e3..e6b1996 100644 --- a/internal/secrets/operator_test.go +++ b/internal/secrets/operator_test.go @@ -12,18 +12,21 @@ func TestAThirdRecipientOpensWithItsOwnKeyOnly(t *testing.T) { if err != nil { t.Fatal(err) } - sealed, more, err := MakeAlso(nodePub, nodePub, opPub) + sealed, forOperator, err := MakeWithOperator(nodePub, nodePub, opPub) if err != nil { t.Fatal(err) } - if len(more) != 1 { - t.Fatalf("%d extra blobs for one extra key", len(more)) + if forOperator == "" { + t.Fatal("no blob for the operator") + } + if _, none, err := MakeWithOperator(nodePub, nodePub, ""); err != nil || none != "" { + t.Fatalf("no operator key, yet a blob %q (%v)", none, err) } fromNode, err := Open(nodePriv, sealed.ForConsumer) if err != nil { t.Fatal(err) } - fromOperator, err := Open(opPriv, more[0]) + fromOperator, err := Open(opPriv, forOperator) if err != nil { t.Fatal(err) } @@ -33,7 +36,7 @@ func TestAThirdRecipientOpensWithItsOwnKeyOnly(t *testing.T) { if len(fromNode) != 40 { t.Fatalf("a minted value is %d characters, not 40", len(fromNode)) } - if _, err := Open(nodePriv, more[0]); err == nil { + if _, err := Open(nodePriv, forOperator); err == nil { t.Fatal("the node's key opened the operator's blob") } if _, err := Open(opPriv, sealed.ForConsumer); err == nil { diff --git a/internal/secrets/seal.go b/internal/secrets/seal.go index 3dff1de..a26f3f5 100644 --- a/internal/secrets/seal.go +++ b/internal/secrets/seal.go @@ -51,22 +51,22 @@ type Sealed struct { // rather than reading the old one back — the only version of rotation that is honest about what // the mesh knows. func Make(consumerKey, providerKey string) (Sealed, error) { - sealed, _, err := MakeAlso(consumerKey, providerKey) + sealed, _, err := MakeWithOperator(consumerKey, providerKey, "") return sealed, err } -// MakeAlso is Make with further recipients: the same fresh value, sealed once more to each key in -// `also`, returned in that order. +// MakeWithOperator is Make with a third recipient: the same fresh value, sealed once more to the +// operator's key, returned beside the two node blobs — or "" when the mesh has no operator key. // -// **For the operator key, and nothing else so far** (novox/hq ADR 0085, amended). A secret a module -// holds for itself is sealed to its node and, when the mesh has an operator key, to that as well — -// so a person holding the key can recover it when the node cannot. The plaintext still exists only -// inside this call; a third blob is one more thing the mesh cannot open, not one more copy it can. -func MakeAlso(consumerKey, providerKey string, also ...string) (Sealed, []string, error) { +// The operator is the one holder that is not a node (novox/hq ADR 0085, amended): a person with a +// key that never entered the mesh, who can recover a secret when the node cannot. The plaintext +// still exists only inside this call; a third blob is one more thing the mesh cannot open, not one +// more copy it can. +func MakeWithOperator(consumerKey, providerKey, operatorKey string) (Sealed, string, error) { if consumerKey == "" || providerKey == "" { // Sealing to an empty key would produce a blob nobody can open, stored as though it were // a working credential. The caller knows which node is which and says so. - return Sealed{}, nil, fmt.Errorf("both ends need a sealing key before a secret can be made") + return Sealed{}, "", fmt.Errorf("both ends need a sealing key before a secret can be made") } // 30 bytes, not 32: base64url of 30 is exactly 40 characters, and 40 is the longest secret an @@ -74,7 +74,7 @@ func MakeAlso(consumerKey, providerKey string, also ...string) (Sealed, []string // "fit the tightest backend" rule ADR 0049 sets for the login, on the secret. 240 bits is ample. value := make([]byte, 30) if _, err := rand.Read(value); err != nil { - return Sealed{}, nil, err + return Sealed{}, "", err } // Base64 without padding, because it lands in a configuration file something else parses and // a password containing a newline or a quote is a support call. @@ -82,24 +82,22 @@ func MakeAlso(consumerKey, providerKey string, also ...string) (Sealed, []string forConsumer, err := Seal(consumerKey, []byte(password)) if err != nil { - return Sealed{}, nil, err + return Sealed{}, "", err } forProvider, err := Seal(providerKey, []byte(password)) if err != nil { - return Sealed{}, nil, err + return Sealed{}, "", err } - more := make([]string, 0, len(also)) - for _, key := range also { - blob, err := Seal(key, []byte(password)) - if err != nil { - return Sealed{}, nil, err + var forOperator string + if operatorKey != "" { + if forOperator, err = Seal(operatorKey, []byte(password)); err != nil { + return Sealed{}, "", err } - more = append(more, blob) } return Sealed{ ForConsumer: forConsumer, ForProvider: forProvider, ConsumerKey: consumerKey, ProviderKey: providerKey, - }, more, nil + }, forOperator, nil } // Accept seals a value somebody supplied, rather than one the mesh made.