diff --git a/modules/mesh-cli/cmd/mesh-cli/enrol.go b/modules/mesh-cli/cmd/mesh-cli/enrol.go new file mode 100644 index 00000000..b18a81c8 --- /dev/null +++ b/modules/mesh-cli/cmd/mesh-cli/enrol.go @@ -0,0 +1,97 @@ +package main + +// `nox enrol open`: the operator opens an enrolment token at their own terminal (novox/hq ADR 0587 §6, §8). +// +// The seat mesh-enrolment's `tokens` answers the token sealed to the operator key, which only the operator's private key +// opens; the mesh never holds it. This subcommand is local: it asks nothing of the node-engine's socket and nothing of +// the controller or the bus. It reads the sealed token (the base64 of the sealed box) on standard input and the private +// key from a file (`--key`, the raw X25519 key in base64 that `operator key make` wrote), opens the box, and prints the +// token text a joining machine's `enrol` takes. The key is read from the file named and nowhere else, and never printed. + +import ( + "crypto/ecdh" + "encoding/base64" + "errors" + "fmt" + "io" + "os" + "strings" + + "golang.org/x/crypto/nacl/box" +) + +const enrolUsage = `nox enrol open --key + +Opens an enrolment token sealed to the operator key (the answer of the seat mesh-enrolment's tokens) and prints the token +to type on the new machine. The sealed token is read on standard input; this runs here and asks nothing of the mesh. +` + +// maxSealedToken bounds what is read of standard input: a token is a few hundred bytes. +const maxSealedToken = 16 << 10 + +// enrolOpen runs `enrol open`: args are what follows `enrol open`. +func enrolOpen(args []string, stdin io.Reader, stdinIsTerminal bool, stdout, stderr io.Writer) int { + keyFile := "" + for i := 0; i < len(args); i++ { + switch { + case args[i] == "--key" && i+1 < len(args): + keyFile = args[i+1] + i++ + case strings.HasPrefix(args[i], "--key="): + keyFile = strings.TrimPrefix(args[i], "--key=") + default: + fmt.Fprint(stderr, enrolUsage) + return 2 + } + } + if keyFile == "" { + fmt.Fprint(stderr, enrolUsage) + return 2 + } + if stdinIsTerminal { + fmt.Fprintln(stderr, "mesh-cli: give the sealed token on standard input (a pipe or a file); nothing was opened") + return 2 + } + raw, err := os.ReadFile(keyFile) + if err != nil { + fmt.Fprintf(stderr, "mesh-cli: the operator key could not be read: %v\n", err) + return 1 + } + sealed, err := io.ReadAll(io.LimitReader(stdin, maxSealedToken+1)) + if err != nil || len(sealed) > maxSealedToken { + fmt.Fprintln(stderr, "mesh-cli: the sealed token could not be read from standard input, or is longer than a token is") + return 1 + } + text, err := openSealed(strings.TrimSpace(string(raw)), strings.TrimSpace(string(sealed))) + if err != nil { + fmt.Fprintf(stderr, "mesh-cli: %v\n", err) + return 1 + } + fmt.Fprintln(stdout, text) + return 0 +} + +// openSealed opens a value sealed to the operator key (the sealed box of mesh-vault's operator copies, box.SealAnonymous +// in base64) with the private key, the raw X25519 key in base64. +func openSealed(privateKey, sealed string) (string, error) { + private, err := base64.StdEncoding.DecodeString(privateKey) + if err != nil || len(private) != 32 { + return "", errors.New("that is not an operator private key (the raw X25519 key in base64)") + } + key, err := ecdh.X25519().NewPrivateKey(private) + if err != nil { + return "", fmt.Errorf("that is not a usable operator private key: %w", err) + } + blob, err := base64.StdEncoding.DecodeString(sealed) + if err != nil { + return "", fmt.Errorf("this is not a sealed token: %w", err) + } + var pub, priv [32]byte + copy(pub[:], key.PublicKey().Bytes()) + copy(priv[:], private) + out, ok := box.OpenAnonymous(nil, blob, &pub, &priv) + if !ok { + return "", errors.New("this token is not sealed to that key; nothing is printed") + } + return strings.TrimSpace(string(out)), nil +} diff --git a/modules/mesh-cli/cmd/mesh-cli/enrol_test.go b/modules/mesh-cli/cmd/mesh-cli/enrol_test.go new file mode 100644 index 00000000..61cd7ea2 --- /dev/null +++ b/modules/mesh-cli/cmd/mesh-cli/enrol_test.go @@ -0,0 +1,80 @@ +package main + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "golang.org/x/crypto/nacl/box" +) + +// 774/S51/AC1, AC2 (novox/hq ADR 0587 §6, §8): a token sealed to a test operator key (as mesh-vault seals it: the text +// of the token, anonymous box, base64) is opened with the private half and printed as a token the engine's parser reads +// (base64url of version 1 JSON with node, broker, fingerprint, signer and secret), and nothing is asked of the engine's +// socket, which here is a path nobody listens on; the wrong key prints nothing and says the token is not sealed to it. +func TestEnrolOpenOpensASealedTokenLocallyAndOnlyWithItsKey(t *testing.T) { + t.Setenv("MESH_VERB", "") + token := base64.RawURLEncoding.EncodeToString([]byte(`{"v":1,"node":"new1","broker":"tls://10.77.0.1:4222",` + + `"fingerprint":"sha256:bus","signer":"` + base64.StdEncoding.EncodeToString(make([]byte, 32)) + `","secret":"one-time"}`)) + pub, priv, _ := box.GenerateKey(rand.Reader) + _, otherPriv, _ := box.GenerateKey(rand.Reader) + sealed, err := box.SealAnonymous(nil, []byte(token), pub, rand.Reader) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + keyFile := func(name string, k *[32]byte) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(base64.StdEncoding.EncodeToString(k[:])+"\n"), 0o600); err != nil { + t.Fatal(err) + } + return p + } + before, beforeIn := socket, standardInput + socket = filepath.Join(dir, "nobody-listens.sock") + t.Cleanup(func() { socket, standardInput = before, beforeIn }) + for _, c := range []struct { + name string + args []string + stdin string + code int + out string + err string + }{ + {"the right key", []string{"enrol", "open", "--key", keyFile("op", priv)}, base64.StdEncoding.EncodeToString(sealed) + "\n", 0, token + "\n", ""}, + {"the key as --key=", []string{"enrol", "open", "--key=" + keyFile("op2", priv)}, base64.StdEncoding.EncodeToString(sealed), 0, token + "\n", ""}, + {"the wrong key", []string{"enrol", "open", "--key", keyFile("other", otherPriv)}, base64.StdEncoding.EncodeToString(sealed), 1, "", "not sealed to that key"}, + {"no key", []string{"enrol", "open"}, base64.StdEncoding.EncodeToString(sealed), 2, "", "nox enrol open --key"}, + {"not a sealed token", []string{"enrol", "open", "--key", keyFile("op3", priv)}, "@@@", 1, "", "not a sealed token"}, + } { + standardInput = func() (io.Reader, bool) { return strings.NewReader(c.stdin), false } + var out, errOut bytes.Buffer + code := run(context.Background(), c.args, &out, &errOut) + if code != c.code || out.String() != c.out || !strings.Contains(errOut.String(), c.err) { + t.Errorf("%s: exit %d, stdout %q, stderr %q", c.name, code, out.String(), errOut.String()) + } + } + // What it prints is a token the engine's parser reads. + var parsed struct { + V int `json:"v"` + Node string `json:"node"` + Secret string `json:"secret"` + } + raw, _ := base64.RawURLEncoding.DecodeString(token) + if json.Unmarshal(raw, &parsed) != nil || parsed.V != 1 || parsed.Node != "new1" || parsed.Secret != "one-time" { + t.Errorf("the token is %+v", parsed) + } + // Standard input a terminal is never read. + standardInput = func() (io.Reader, bool) { return strings.NewReader("x"), true } + var out, errOut bytes.Buffer + if code := run(context.Background(), []string{"enrol", "open", "--key", keyFile("op4", priv)}, &out, &errOut); code != 2 || out.Len() != 0 { + t.Errorf("a terminal on standard input: exit %d, stdout %q", code, out.String()) + } +} diff --git a/modules/mesh-cli/cmd/mesh-cli/main.go b/modules/mesh-cli/cmd/mesh-cli/main.go index f77d1933..d4b805e1 100644 --- a/modules/mesh-cli/cmd/mesh-cli/main.go +++ b/modules/mesh-cli/cmd/mesh-cli/main.go @@ -56,6 +56,8 @@ const usage = `mesh-cli — the controller's command lines, from this node (novo mesh-cli e.g. mesh-cli status, mesh-cli settings show zsh --node nox the same, in interactive zsh + nox enrol open --key opens a sealed enrolment token here, asking nothing of the mesh (ADR 0587) + Run it as yourself, never under sudo: the node-engine tells the controller which account asks. What a pipe or a file gives its standard input (at most 64 KiB) goes with the line to the controller's terminal alone, e.g. a secret: mesh-cli secret accept --from - < file @@ -82,6 +84,12 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int { "Nothing was asked\n", verb) return 1 } + // `enrol open` is local: it opens a sealed enrolment token with the operator's private key and asks nothing of the + // node-engine or the mesh (novox/hq ADR 0587 §8). + if len(args) >= 2 && args[0] == "enrol" && args[1] == "open" { + in, terminal := standardInput() + return enrolOpen(args[2:], in, terminal, stdout, stderr) + } in, err := carried() if err != nil { fmt.Fprintf(stderr, "mesh-cli: %v\n", err) diff --git a/modules/mesh-cli/go.mod b/modules/mesh-cli/go.mod index c056d521..d1aea226 100644 --- a/modules/mesh-cli/go.mod +++ b/modules/mesh-cli/go.mod @@ -2,4 +2,14 @@ module mesh-cli go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 + +require golang.org/x/crypto v0.57.0 + +require ( + github.com/klauspost/compress v1.20.0 // indirect + github.com/nats-io/nats.go v1.54.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/sys v0.48.0 // indirect +) diff --git a/modules/mesh-cli/go.sum b/modules/mesh-cli/go.sum index 2e60a9c1..a06c658d 100644 --- a/modules/mesh-cli/go.sum +++ b/modules/mesh-cli/go.sum @@ -1,2 +1,16 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/modules/mesh-vault/cmd/mesh-vault/accounts.go b/modules/mesh-vault/cmd/mesh-vault/accounts.go index 2a024ccb..2e564d6d 100644 --- a/modules/mesh-vault/cmd/mesh-vault/accounts.go +++ b/modules/mesh-vault/cmd/mesh-vault/accounts.go @@ -55,8 +55,20 @@ type AccountHash struct { Node string `json:"node,omitempty"` // Rotating is a rotation under way over a second login (rotation.go, ADR 0561 §4): absent when none is. Rotating *Rotating `json:"rotating,omitempty"` + // Kept marks an account kept for an enrolment or a node's login (enrolment.go, noderotation.go): `enrolment` for a + // token's user `enrol.`, `node` for a node's login. Such a key is dropped by the verb that ends it, never + // retired as an account no longer wanted (retire.go). + Kept string `json:"kept,omitempty"` + // KeysDigest is, for a joining node's login, the digest of the keys it was minted for (ADR 0587 §3). + KeysDigest string `json:"keys-digest,omitempty"` } +// The kinds of account the vault keeps beside the wanted ones (AccountHash.Kept). +const ( + KeptEnrolment = "enrolment" + KeptNode = "node" +) + // AccountStore is the vault's state of bus-account hashes: read and written by the vault's instances alone. type AccountStore interface { Get(user string) (*AccountHash, error) diff --git a/modules/mesh-vault/cmd/mesh-vault/adopt.go b/modules/mesh-vault/cmd/mesh-vault/adopt.go index 5d005ca3..c274783d 100644 --- a/modules/mesh-vault/cmd/mesh-vault/adopt.go +++ b/modules/mesh-vault/cmd/mesh-vault/adopt.go @@ -137,6 +137,14 @@ func (v *Vault) adopt(node string, raw []byte, by string, force bool) (Adoption, return Adoption{}, err } + // The users the loaded list holds, for a node's rotation to see its new login in it and the old one gone (ADR 0583 §3). + loaded := make(map[string]string, len(accounts)) + for _, acc := range accounts { + loaded[acc.User] = acc.Hash + } + v.loadedListHeld(h.From, loaded) + defer v.advanceRotations() + v.hashMu.Lock() defer v.hashMu.Unlock() held, err := v.heldUsers() diff --git a/modules/mesh-vault/cmd/mesh-vault/enrolment.go b/modules/mesh-vault/cmd/mesh-vault/enrolment.go new file mode 100644 index 00000000..17f47978 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/enrolment.go @@ -0,0 +1,577 @@ +package main + +// The vault's part of enrolment (novox/hq ADR 0587 §3, §5, §8; ADR 0561 §4, §5; issue 774). +// +// A machine joins with a token a person types. The seat mesh-enrolment decides who may join; the vault holds the secrets. +// Its verb `enrolment-token` makes the token's one-time secret, keeps it with its operator copy as `enrol//token`, +// writes its bcrypt hash under `bus-accounts` as the user `enrol.`, and answers at issue the public fields alone +// (never the token). Asked to show it, it answers the whole token sealed to the operator key, which the operator opens at +// their own terminal (`nox enrol open`); asked to drop it, it takes the token and its hash away. Its verb `node-account` +// makes the account `node.` of a joining machine at the redeem: once for a node and a set of keys, sealed to the key +// the machine presented, with the node's key in `node-logins` naming that login in use and nothing rotating. Only the +// seat mesh-enrolment may call either: the runtime proves who asks. + +import ( + "crypto/ecdh" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "golang.org/x/crypto/bcrypt" +) + +// The names enrolment gives: the module that holds the seat mesh-enrolment, the prefix of a token's bus user, and the +// vault's state of each node's logins. +const ( + EnrolmentModule = delivery.EnrolmentModule + EnrolUserPrefix = delivery.EnrolmentUserPrefix + NodeLoginsState = delivery.NodeLoginsState + // MeshHostModule is the module a node's own login secret is kept under (`own//mesh-host/`, ADR 0583 §3). + MeshHostModule = "mesh-host" +) + +// Events the vault says for enrolment's accounts are account-hashed (accounts.go), never carrying a hash or a value. + +// enrolNode says node is a name a joining machine can have: one the mesh writes, with no dot, and not shaped like a +// second login. +func enrolNode(node string) error { + if !namePart.MatchString(node) || strings.Contains(node, ".") || strings.Contains(node, SecondLoginSuffix) { + return fmt.Errorf("%q is not a name a node can have: lower-case words, no dot, and not ending in %s", node, SecondLoginSuffix) + } + return nil +} + +// enrolSecret is the vault's name of a node's enrolment token. +func enrolSecret(node string) string { return "enrol/" + node + "/token" } + +// enrolSecretName says name is an enrolment token, which is handed to nobody: only shown, sealed to the operator key. +func enrolSecretName(name string) bool { return strings.HasPrefix(name, "enrol/") } + +// loginSecret is the vault's name of a node's login, the value its node-engine takes (ADR 0583 §3 insight). +func loginSecret(node, login string) string { + return "own/" + node + "/" + MeshHostModule + "/" + login +} + +// nodeLoginSecret says name is a node's login secret, and for which node and login. A node's login is handed to that +// node's node-engine alone. +func nodeLoginSecret(name string) (node, login string, ok bool) { + parts := strings.Split(name, "/") + if len(parts) != 4 || parts[0] != "own" || parts[2] != MeshHostModule { + return "", "", false + } + base := strings.TrimSuffix(parts[3], SecondLoginSuffix) + if base != NodeEnginePrefix+parts[1] { + return "", "", false + } + return parts[1], parts[3], true +} + +// loginRefusal says why caller may not have the secret name when it is a node's login, or "": a node's login is the +// node-engine of that node's alone (`node.` or its second login), sealed to the key that machine stated. +func (v *Vault) loginRefusal(caller, name string) (isLogin bool, why string) { + node, _, ok := nodeLoginSecret(name) + if !ok { + return false, "" + } + engine, _, _ := strings.Cut(strings.TrimPrefix(caller, NodeEnginePrefix), SecondLoginSuffix) + if !strings.HasPrefix(caller, NodeEnginePrefix) || engine != node { + return true, fmt.Sprintf("%s is a login of %s's node-engine and is handed to that machine's engine alone (%s or its second login)", + name, node, NodeEnginePrefix+node) + } + return true, "" +} + +// loginKeyRefusal says why a node's login is not sealed to the key asked: it is sealed to the key the node's node-engine +// stated (node-key) and to no other, or "". +func (v *Vault) loginKeyRefusal(caller, publicKey string) string { + node, _ := loginNode(caller) + stated, had, err := v.Store.NodeKey(node) + if err != nil { + return "the key " + node + "'s node-engine stated could not be read: " + err.Error() + } + if !had { + return "the vault holds no sealing key that " + node + "'s node-engine stated (node-key)" + } + want, err := publicPEM(stated) + if err != nil { + return "the key " + node + "'s node-engine stated is not usable: " + err.Error() + } + given, err := askerKey(publicKey) + if err != nil || strings.TrimSpace(given) != strings.TrimSpace(want) { + return "a node's login is sealed to the key its node-engine stated, and the key sent is another" + } + return "" +} + +// fromEnrolment refuses every caller but the seat mesh-enrolment, as the runtime proved it. +func (v *Vault) fromEnrolment(caller, verb string) error { + callers := v.EnrolCallers + if callers == nil { + callers = stdio.Callers{EnrolmentModule} + } + if !callers.Allows(caller) { + if caller == "" { + caller = stdio.Unproven + } + return fmt.Errorf("%s is asked by the seat %s alone, which decides who may join; %s is not it (novox/hq ADR 0587 §3)", + verb, EnrolmentModule, caller) + } + return nil +} + +// TokenBase is what a token states of the bus beside its secret: where the bus is and the certificate to expect there. +// The machine's first tunnel comes with the issue call (ADR 0587 §3, insight 2026-11-02). +type TokenBase struct { + Broker string `json:"broker"` + Fingerprint string `json:"fingerprint"` +} + +// EnrolmentTokenArgs are `enrolment-token`'s arguments (the seat's contract, mesh-sdk delivery): the node, when it expires +// (the seat's word), the machine's first tunnel at issue, and which of the three acts: issue (neither switch), sealed or +// drop. +type EnrolmentTokenArgs struct { + Node string + Expires string + Tunnel *delivery.TokenTunnel + Sealed bool + Drop bool +} + +// EnrolmentTokenAnswer is mesh-sdk's delivery.EnrolmentTokenAnswer, the wire shape mesh-enrolment decodes: at issue the +// bus's address, its certificate's fingerprint and the vault's public signing key; when sealed, the whole token sealed to +// the operator key; at drop, whether it was dropped. Never a token in the clear. +type EnrolmentTokenAnswer = delivery.EnrolmentTokenAnswer + +// enrolTunnel is the vault's name of the first tunnel a node's token carries, kept beside the token. +func enrolTunnel(node string) string { return "enrol/" + node + "/tunnel" } + +// EnrolmentToken answers caller's `enrolment-token`. +func (v *Vault) EnrolmentToken(caller string, a EnrolmentTokenArgs) (EnrolmentTokenAnswer, error) { + if err := v.fromEnrolment(caller, "enrolment-token"); err != nil { + return EnrolmentTokenAnswer{}, err + } + if err := enrolNode(a.Node); err != nil { + return EnrolmentTokenAnswer{}, err + } + if v.Accounts == nil { + return EnrolmentTokenAnswer{}, errors.New("the vault has no state of bus accounts to write to") + } + switch { + case a.Drop && a.Sealed: + return EnrolmentTokenAnswer{}, errors.New("a token is answered sealed or dropped, not both") + case a.Drop: + return v.dropEnrolmentToken(caller, a.Node) + case a.Sealed: + return v.showEnrolmentToken(caller, a.Node) + } + return v.issueEnrolmentToken(caller, a) +} + +// issueEnrolmentToken makes the secret afresh (an earlier token of the node is replaced), keeps it with its operator copy, +// and writes its hash as the user `enrol.`. Its answer carries the public fields alone. +func (v *Vault) issueEnrolmentToken(caller string, a EnrolmentTokenArgs) (EnrolmentTokenAnswer, error) { + // What the machine must trust is known before anything is kept: the bus, its certificate, the vault's signing key, + // and the first tunnel whole (ADR 0587 §3), so the token shown later is one a machine can use. + base, err := v.tokenBase() + if err != nil { + return EnrolmentTokenAnswer{}, err + } + keys, err := v.keys() + if err != nil { + return EnrolmentTokenAnswer{}, fmt.Errorf("the token carries the vault's signing key, which is not held yet: %w", err) + } + signer := keys.key.Public().(ed25519.PublicKey) + if a.Tunnel == nil { + return EnrolmentTokenAnswer{}, errors.New("enrolment-token at issue carries the machine's first tunnel (`tunnel`: key, address, range, " + + "hub_key, hub_endpoint), so the machine reaches the bus over the private network (novox/hq ADR 0587 §3)") + } + if missing := (delivery.Token{Version: 1, Broker: base.Broker, Fingerprint: base.Fingerprint, Signer: signer, + Secret: "-", Tunnel: a.Tunnel}).Missing(); len(missing) > 0 { + return EnrolmentTokenAnswer{}, fmt.Errorf("the token of %s would lack %s: nothing was issued", a.Node, strings.Join(missing, ", ")) + } + tunnel, err := json.Marshal(a.Tunnel) + if err != nil { + return EnrolmentTokenAnswer{}, err + } + raw := make([]byte, 30) + if _, err := rand.Read(raw); err != nil { + return EnrolmentTokenAnswer{}, err + } + secret := base64.RawURLEncoding.EncodeToString(raw) + name, user := enrolSecret(a.Node), EnrolUserPrefix+a.Node + c, err := v.keepWith(name, secret, "the vault", caller, OriginMade, keepOptions{quiet: true}) + if err != nil { + return EnrolmentTokenAnswer{}, fmt.Errorf("the token of %s is not kept: %w", a.Node, err) + } + hash, err := bcrypt.GenerateFromPassword([]byte(secret), v.hashCost()) + if err != nil { + return EnrolmentTokenAnswer{}, err + } + if _, err := v.keepWith(enrolTunnel(a.Node), string(tunnel), "the vault", caller, OriginMade, keepOptions{quiet: true}); err != nil { + return EnrolmentTokenAnswer{}, fmt.Errorf("the first tunnel of %s is not kept: %w", a.Node, err) + } + at := v.Now().UTC() + fp := Fingerprint(secret) + if err := v.Accounts.Put(AccountHash{User: user, Hash: string(hash), Of: name, Generation: c.Generation, Fingerprint: fp, + At: at, Kept: KeptEnrolment}); err != nil { + return EnrolmentTokenAnswer{}, fmt.Errorf("the hash of %s could not be written to %s: %w", user, BusAccountsState, err) + } + v.saidHashed(user, name, c.Generation, fp, at) + _ = v.recordHanding(Handing{To: caller, Name: name, Generation: c.Generation, Outcome: "issued", Why: "enrolment token of " + a.Node}) + v.log("made the enrolment token of %s for %s (generation %d); its hash is the user %s", a.Node, caller, c.Generation, user) + return EnrolmentTokenAnswer{Node: a.Node, Broker: base.Broker, Fingerprint: base.Fingerprint, + Signer: base64.StdEncoding.EncodeToString(signer)}, nil +} + +// showEnrolmentToken answers the whole token sealed to the operator key: the secret held, the bus's address and +// certificate fingerprint, and the vault's public signing key. +func (v *Vault) showEnrolmentToken(caller, node string) (EnrolmentTokenAnswer, error) { + rec, held, err := v.Store.Read(enrolSecret(node)) + if err != nil { + return EnrolmentTokenAnswer{}, err + } + if !held { + return EnrolmentTokenAnswer{}, fmt.Errorf("the vault holds no enrolment token of %s", node) + } + secret, err := openAtRest(rec, v) + if err != nil { + return EnrolmentTokenAnswer{}, fmt.Errorf("the token of %s does not open with the vault's key: %w", node, err) + } + operator, err := v.Store.OperatorKey() + if err != nil { + return EnrolmentTokenAnswer{}, err + } + if operator == "" { + return EnrolmentTokenAnswer{}, errors.New("the vault has no operator key to seal the token to (ADR 0113 §22)") + } + base, err := v.tokenBase() + if err != nil { + return EnrolmentTokenAnswer{}, err + } + keys, err := v.keys() + if err != nil { + return EnrolmentTokenAnswer{}, fmt.Errorf("the token carries the vault's signing key, which is not held yet: %w", err) + } + var tunnel *delivery.TokenTunnel + if trec, theld, err := v.Store.Read(enrolTunnel(node)); err != nil { + return EnrolmentTokenAnswer{}, err + } else if theld { + raw, err := openAtRest(trec, v) + if err != nil { + return EnrolmentTokenAnswer{}, fmt.Errorf("the first tunnel of %s does not open with the vault's key: %w", node, err) + } + tunnel = &delivery.TokenTunnel{} + if err := json.Unmarshal([]byte(raw), tunnel); err != nil { + return EnrolmentTokenAnswer{}, fmt.Errorf("the first tunnel of %s is not readable: %w", node, err) + } + } + signer := keys.key.Public().(ed25519.PublicKey) + tok := delivery.Token{Version: 1, Node: node, Broker: base.Broker, Fingerprint: base.Fingerprint, + Signer: []byte(signer), Secret: secret, Tunnel: tunnel} + if tunnel == nil { + return EnrolmentTokenAnswer{}, fmt.Errorf("the vault keeps no first tunnel for %s's token: issue it again with its tunnel", node) + } + text, err := tok.Encode() + if err != nil { + return EnrolmentTokenAnswer{}, err + } + sealed, err := operatorSeal(operator, []byte(text)) + if err != nil { + return EnrolmentTokenAnswer{}, err + } + _ = v.recordHanding(Handing{To: caller, Name: enrolSecret(node), Generation: rec.Generation, Outcome: "shown", + Why: "sealed to the operator key"}) + return EnrolmentTokenAnswer{Node: node, Broker: base.Broker, Fingerprint: base.Fingerprint, + Signer: base64.StdEncoding.EncodeToString(signer), Sealed: sealed}, nil +} + +// tokenBase is what a token states of the bus, or why the vault knows none. +func (v *Vault) tokenBase() (TokenBase, error) { + if v.TokenBase == nil { + return TokenBase{}, errors.New("the vault does not know the bus's address and certificate fingerprint, so it makes no token to show") + } + return v.TokenBase() +} + +// dropEnrolmentToken takes the token and its hash away, at every end of an enrolment (ADR 0587 §5). Nothing held is +// answered as dropped nothing. +func (v *Vault) dropEnrolmentToken(caller, node string) (EnrolmentTokenAnswer, error) { + name, user := enrolSecret(node), EnrolUserPrefix+node + out := EnrolmentTokenAnswer{Node: node} + a, err := v.Accounts.Get(user) + if err != nil { + return out, err + } + if a != nil { + if a.Kept != KeptEnrolment { + return out, fmt.Errorf("%s is no enrolment token's user: nothing was dropped", user) + } + if err := v.Accounts.Delete(user); err != nil { + return out, fmt.Errorf("the hash of %s could not be taken out of %s: %w", user, BusAccountsState, err) + } + out.Dropped = true + } + if _, held, err := v.Store.Read(name); err != nil { + return out, err + } else if held { + if err := v.Store.Delete(name); err != nil { + return out, fmt.Errorf("the token of %s could not be deleted: %w", node, err) + } + out.Dropped = true + } + if _, held, err := v.Store.Read(enrolTunnel(node)); err != nil { + return out, err + } else if held { + if err := v.Store.Delete(enrolTunnel(node)); err != nil { + return out, fmt.Errorf("the first tunnel of %s could not be deleted: %w", node, err) + } + } + if out.Dropped { + _ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "dropped", Why: "enrolment token of " + node}) + v.log("dropped the enrolment token of %s and its hash, asked by %s", node, caller) + } + return out, nil +} + +// saidHashed says account-hashed for a user whose key the vault wrote: never the hash or the value. +func (v *Vault) saidHashed(user, of string, generation int, fingerprint string, at time.Time) { + if v.Emit == nil { + return + } + if err := v.Emit(AccountHashedEvent, map[string]any{"user": user, "of": of, "generation": generation, + "fingerprint": fingerprint, "at": at.Format(time.RFC3339Nano)}); err != nil { + v.log("the hash of %s is written, but %s could not be said: %v", user, AccountHashedEvent, err) + } +} + +// NodeAccountArgs are `node-account`'s arguments: the joining node, the sealing key its machine presented (raw X25519 in +// base64, or SPKI PEM), the digest of the keys it presented, and whether to retire the account instead. +type NodeAccountArgs struct { + Node string + SealingKey string + KeysDigest string + Retire bool +} + +// NodeAccountAnswer is mesh-sdk's delivery.NodeAccountAnswer, the wire shape mesh-enrolment decodes and relays into the +// redeem answer: the login, and the account sealed to the presented key, as the node-engine opens it (the password, +// sealed anonymously to its raw X25519 key, in base64); or that it was retired. +type NodeAccountAnswer = delivery.NodeAccountAnswer + +// NodeAccount answers caller's `node-account`: the joining node's account, minted once for a node and a set of keys and +// answered sealed to the key presented; or, with retire, taken away again at a revoke. +func (v *Vault) NodeAccount(caller string, a NodeAccountArgs) (NodeAccountAnswer, error) { + if err := v.fromEnrolment(caller, "node-account"); err != nil { + return NodeAccountAnswer{}, err + } + if err := enrolNode(a.Node); err != nil { + return NodeAccountAnswer{}, err + } + if v.Accounts == nil { + return NodeAccountAnswer{}, errors.New("the vault has no state of bus accounts to write to") + } + user := NodeEnginePrefix + a.Node + secret := loginSecret(a.Node, user) + v.hashMu.Lock() + defer v.hashMu.Unlock() + was, err := v.Accounts.Get(user) + if err != nil { + return NodeAccountAnswer{}, fmt.Errorf("%s %s could not be read: %w", BusAccountsState, user, err) + } + if a.Retire { + return v.retireNodeAccount(caller, a.Node, user, secret, was) + } + if was != nil && (was.Kept != KeptNode || was.Adopted) { + return NodeAccountAnswer{}, fmt.Errorf("%s already has an account the vault did not make for an enrolment: a node's account is never "+ + "minted anew (novox/hq ADR 0561 §4)", user) + } + if _, held, err := v.Store.Read(enrolSecret(a.Node)); err != nil { + return NodeAccountAnswer{}, err + } else if !held { + return NodeAccountAnswer{}, fmt.Errorf("%s holds no enrolment token, so no enrolment is under way: no account is made", a.Node) + } + to, err := sealingRaw(a.SealingKey) + if err != nil { + return NodeAccountAnswer{}, err + } + if strings.TrimSpace(a.KeysDigest) == "" { + return NodeAccountAnswer{}, errors.New("node-account names the digest of the keys the machine presented (`keys_digest`)") + } + if was == nil { + if err := v.mintNodeAccount(caller, a.Node, user, secret, a.KeysDigest); err != nil { + return NodeAccountAnswer{}, err + } + } else if was.KeysDigest != a.KeysDigest { + return NodeAccountAnswer{}, fmt.Errorf("%s was made for another set of keys than the ones presented now: revoke the enrolment and issue it again", user) + } + rec, held, err := v.Store.Read(secret) + if err != nil || !held { + return NodeAccountAnswer{}, fmt.Errorf("the vault holds no value of %s to answer: %v", user, err) + } + value, err := openAtRest(rec, v) + if err != nil { + return NodeAccountAnswer{}, fmt.Errorf("%s does not open with the vault's key: %w", secret, err) + } + var acct struct { + Password string `json:"password"` + } + if err := json.Unmarshal([]byte(value), &acct); err != nil || acct.Password == "" { + return NodeAccountAnswer{}, fmt.Errorf("%s holds no password to answer", secret) + } + sealed, err := operatorSeal(to, []byte(acct.Password)) + if err != nil { + return NodeAccountAnswer{}, err + } + _ = v.recordHanding(Handing{To: caller, Name: secret, Generation: rec.Generation, Outcome: "handed", Why: "node account of a joining node, sealed to its presented key"}) + return NodeAccountAnswer{Node: a.Node, Login: user, Account: sealed}, nil +} + +// mintNodeAccount makes a joining node's account: its value kept as the login's own secret, its hash in bus-accounts, and +// the node's key in node-logins naming it in use with nothing rotating. +func (v *Vault) mintNodeAccount(caller, node, user, secret, digest string) error { + value, password, err := v.accountValue(node, user) + if err != nil { + return err + } + c, err := v.keepWith(secret, value, "the vault", caller, OriginMade, keepOptions{quiet: true}) + if err != nil { + return fmt.Errorf("the account of %s is not kept: %w", node, err) + } + hash, err := bcrypt.GenerateFromPassword([]byte(password), v.hashCost()) + if err != nil { + return err + } + at := v.Now().UTC() + fp := Fingerprint(value) + if err := v.Accounts.Put(AccountHash{User: user, Hash: string(hash), Of: secret, Generation: c.Generation, Fingerprint: fp, + At: at, Kept: KeptNode, KeysDigest: digest}); err != nil { + return fmt.Errorf("the hash of %s could not be written to %s: %w", user, BusAccountsState, err) + } + v.saidHashed(user, secret, c.Generation, fp, at) + if err := v.putLogin(NodeLogin{Node: node, Login: user, Since: at}); err != nil { + return fmt.Errorf("the account of %s is made, and %s %s could not be written: %w", node, NodeLoginsState, node, err) + } + v.log("made the account %s of a joining node for %s", user, caller) + return nil +} + +// accountValue is a node's login whole, in the shape the controller's accounts have (url, fingerprint, node, user, +// password), and its password. +func (v *Vault) accountValue(node, user string) (value, password string, err error) { + raw := make([]byte, 30) + if _, err := rand.Read(raw); err != nil { + return "", "", err + } + password = base64.RawURLEncoding.EncodeToString(raw) + acct := map[string]string{"node": node, "user": user, "password": password} + if v.TokenBase != nil { + if base, err := v.TokenBase(); err == nil { + acct["url"], acct["fingerprint"] = base.Broker, base.Fingerprint + } + } + out, err := json.Marshal(acct) + return string(out), password, err +} + +// retireNodeAccount takes a joining node's account away: its hash, its value and its key in node-logins. Only one the +// vault made for an enrolment, and only while nothing rotates. +func (v *Vault) retireNodeAccount(caller, node, user, secret string, was *AccountHash) (NodeAccountAnswer, error) { + out := NodeAccountAnswer{Node: node, Login: user} + if was != nil && was.Kept != KeptNode { + return out, fmt.Errorf("%s is not an account the vault made for an enrolment: nothing was retired", user) + } + if nl, err := v.getLogin(node); err != nil { + return out, err + } else if nl != nil && nl.Rotating != "" { + return out, fmt.Errorf("%s is rotating its login: nothing was retired", node) + } + if was != nil { + if err := v.Accounts.Delete(user); err != nil { + return out, fmt.Errorf("the hash of %s could not be taken out of %s: %w", user, BusAccountsState, err) + } + out.Retired = true + } + if _, held, err := v.Store.Read(secret); err != nil { + return out, err + } else if held && was != nil { + if err := v.Store.Delete(secret); err != nil { + return out, fmt.Errorf("the value of %s could not be deleted: %w", user, err) + } + } + if out.Retired { + if err := v.deleteLogin(node); err != nil { + return out, err + } + _ = v.recordHanding(Handing{To: caller, Name: secret, Outcome: "retired", Why: "node account of an enrolment revoked"}) + v.log("retired the account %s of a revoked enrolment, asked by %s", user, caller) + } + return out, nil +} + +// sealingRaw takes the key a machine presents to be sealed to, as raw X25519 in base64 or SPKI PEM, and gives it raw in +// base64: the form the node-engine's sealing key opens an anonymous box with. +func sealingRaw(key string) (string, error) { + key = strings.TrimSpace(key) + if key == "" { + return "", errors.New("node-account names the sealing key the machine presented (`sealing_key`)") + } + if raw, err := base64.StdEncoding.DecodeString(key); err == nil && len(raw) == 32 { + return key, nil + } + block, _ := pem.Decode([]byte(key)) + if block == nil { + return "", errors.New("the sealing key presented is neither a raw X25519 key in base64 nor SPKI PEM") + } + parsed, err := x509.ParsePKIXPublicKey(block.Bytes) + pub, ok := parsed.(*ecdh.PublicKey) + if err != nil || !ok || pub.Curve() != ecdh.X25519() { + return "", errors.New("the sealing key presented is not an X25519 key") + } + return base64.StdEncoding.EncodeToString(pub.Bytes()), nil +} + +// sealingPEM takes the key a machine presents to be sealed to, as raw X25519 in base64 or SPKI PEM, and gives PEM. +func sealingPEM(key string) (string, error) { + key = strings.TrimSpace(key) + if key == "" { + return "", errors.New("node-account names the sealing key the machine presented (`sealing_key`)") + } + if strings.Contains(key, "BEGIN") { + return askerKey(key) + } + pem, err := publicPEM(key) + if err != nil { + return "", fmt.Errorf("the sealing key presented is not a usable key: %w", err) + } + return pem, nil +} + +// TokenBaseFromFile reads what a token states of the bus from the vault's own account file (`{url, fingerprint}` as the +// controller's accounts have), in the directory of the vault's state. It answers an error when there is none. +func TokenBaseFromFile(stateDir string) func() (TokenBase, error) { + return func() (TokenBase, error) { + raw, err := os.ReadFile(filepath.Join(stateDir, "broker")) + if err != nil { + return TokenBase{}, fmt.Errorf("the vault's own account is not readable, so the bus's address is not known: %w", err) + } + var acct struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint"` + } + if json.Unmarshal(raw, &acct) != nil || acct.URL == "" || acct.Fingerprint == "" { + return TokenBase{}, errors.New("the vault's own account names no bus address and certificate fingerprint") + } + return TokenBase{Broker: acct.URL, Fingerprint: acct.Fingerprint}, nil + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/enrolment_test.go b/modules/mesh-vault/cmd/mesh-vault/enrolment_test.go new file mode 100644 index 00000000..d8e04307 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/enrolment_test.go @@ -0,0 +1,503 @@ +package main + +import ( + "crypto/ecdh" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "fmt" + "slices" + "sort" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "golang.org/x/crypto/bcrypt" + "golang.org/x/crypto/nacl/box" +) + +// loginsState is the vault's state node-logins in memory; like the runtime it refuses a value naming a credential. +type loginsState struct { + mu sync.Mutex + keys map[string]NodeLogin +} + +func (l *loginsState) Get(node string) (*NodeLogin, error) { + l.mu.Lock() + defer l.mu.Unlock() + n, ok := l.keys[node] + if !ok { + return nil, nil + } + return &n, nil +} + +func (l *loginsState) Put(n NodeLogin) error { + l.mu.Lock() + defer l.mu.Unlock() + raw, err := delivery.EncodeNodeLogin(n) + if err != nil { + return err + } + if field := credentialField(raw); field != "" { + return fmt.Errorf("%s carries a field %q, which names a credential", n.Node, field) + } + if l.keys == nil { + l.keys = map[string]NodeLogin{} + } + l.keys[n.Node] = n + return nil +} + +func (l *loginsState) Delete(node string) error { + l.mu.Lock() + defer l.mu.Unlock() + delete(l.keys, node) + return nil +} + +func (l *loginsState) Keys() ([]string, error) { + l.mu.Lock() + defer l.mu.Unlock() + var out []string + for k := range l.keys { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +// enrolling is a vault that mints, with the lease handed away, a loaded user list at revision 570, the bus on novox, the +// nodes g14, ace and novox placed, their existing logins adopted, and hand acts of person.jochen's. +type enrolling struct { + testVault + accounts *fakeAccounts + logins *loginsState + acts map[string]HandActSeen +} + +func newEnrolling(t *testing.T) *enrolling { + t.Helper() + v := newVault(t, nil) + e := &enrolling{testVault: v, accounts: &fakeAccounts{keys: map[string]AccountHash{}}, logins: &loginsState{keys: map[string]NodeLogin{}}, acts: map[string]HandActSeen{}} + v.Accounts, v.Logins = e.accounts, e.logins + v.Placement = placementsOf(t, map[string][]string{"novox": {"nats"}, "g14": {"web"}, "ace": {"shop"}}) + v.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil) + v.LoadedListHeard(570) + v.TokenBase = func() (TokenBase, error) { + return TokenBase{Broker: "tls://10.77.0.1:4222", Fingerprint: "sha256:bus"}, nil + } + v.ReadAct = func(id string) (*HandActSeen, error) { + if a, ok := e.acts[id]; ok { + return &a, nil + } + return nil, nil + } + for _, n := range []string{"g14", "ace", "novox"} { + e.accounts.keys["node."+n] = AccountHash{User: "node." + n, Hash: "$2a$04$adopted", Adopted: true, At: time.Now()} + pub, _ := aSealingKey(t) + if err := v.recordNodeKey(n, pub, "node."+n); err != nil { + t.Fatal(err) + } + } + return e +} + +func (e *enrolling) act(id, by, node string) { + e.acts[id] = HandActSeen{ID: id, By: by, At: time.Now().Add(time.Hour), Verb: "rotate-node", Args: []string{node}} +} + +// 774/S43/AC1-AC3, S44/AC1-AC3, S45/AC1 (novox/hq ADR 0587 §3, §5, §6, §8): the vault's enrolment verbs +// are called through their real tool handlers with the arguments mesh-enrolment sends, and each wire answer is decoded +// into mesh-sdk's delivery types the holder decodes (EnrolmentTokenAnswer, NodeAccountAnswer, SignAnswer and the signed +// parts of RedeemAnswer), with every field the holder reads filled. Only mesh-enrolment is answered; a token is issued +// only with its first tunnel whole, its hash is bcrypt of the kept secret, `sealed` answers it sealed to the operator key +// (a token whose Missing() is empty) and issues nothing; the account is the same for the same keys and opens with the +// sealing key presented, as the node-engine opens it; a joining node's certificate is signed for the serving key presented +// only when the keys have the digest node-account recorded; a retire and a drop take it all away. +func TestTheVaultsEnrolmentAnswersAreMeshSDKsWireShapes(t *testing.T) { + e := newEnrolling(t) + if _, err := e.EnsureSigning(); err != nil { + t.Fatal(err) + } + // new1 is placed with its private-network address, as mesh-enrolment writes it before it asks for the certificate. + values := map[string][]byte{} + for node, p := range map[string]delivery.Placement{"novox": {Node: "novox", Modules: []delivery.PlacedModule{{Module: "nats"}}}, + "g14": {Node: "g14"}, "new1": {Node: "new1", Decided: delivery.NodeDecided{Address: "10.77.0.9"}}} { + p.V = delivery.Version + kv, err := delivery.PlacementValues(p) + if err != nil { + t.Fatal(err) + } + for k, raw := range kv { + values[k] = raw + } + _ = node + } + e.Placement = PlacementReader{Read: func(key string) ([]byte, error) { return values[key], nil }, + Keys: func() ([]string, error) { + var out []string + for k := range values { + out = append(out, k) + } + return out, nil + }} + defs := definitions(&Env{Vault: e.Vault}) + call := func(verb, caller string, args map[string]any) (json.RawMessage, error) { + out, err := toolNamed(t, defs, Seat+"."+verb).run(caller, args) + if err != nil { + return nil, err + } + return json.Marshal(out) + } + const holder = "novox.mesh-enrolment" + signer, _ := e.keys() + signerB64 := base64.StdEncoding.EncodeToString(signer.key.Public().(ed25519.PublicKey)) + tunnel := map[string]any{"key": "machine-tunnel-key", "address": "10.77.0.9", "range": "10.77.0.0/24", + "hub_key": "hub-tunnel-key", "hub_endpoint": "203.0.113.1:51820"} + without := func(field string) map[string]any { + out := map[string]any{} + for k, x := range tunnel { + if k != field { + out[k] = x + } + } + return out + } + sealingKey, _ := ecdh.X25519().GenerateKey(rand.Reader) + sealingPub := base64.StdEncoding.EncodeToString(sealingKey.PublicKey().Bytes()) + identity, _, _ := ed25519.GenerateKey(rand.Reader) + serving, _, _ := ed25519.GenerateKey(rand.Reader) + keys := map[string]any{"public_key": base64.StdEncoding.EncodeToString(identity), "overlay_key": "overlay-pub", + "sealing_key": sealingPub, "serving_key": base64.StdEncoding.EncodeToString(serving)} + digest := delivery.KeysDigest(keys["public_key"].(string), "overlay-pub", sealingPub, keys["serving_key"].(string)) + with := func(base map[string]any, more map[string]any) map[string]any { + out := map[string]any{} + for k, x := range base { + out[k] = x + } + for k, x := range more { + out[k] = x + } + return out + } + reply := delivery.RedeemAnswer{Node: "new1", Login: "node.new1", Account: "sealed", Certificate: "pem", Signer: signerB64} + + var issued, sealed, dropped delivery.EnrolmentTokenAnswer + var account, accountAgain, retired delivery.NodeAccountAnswer + var cert, replySig delivery.SignAnswer + var secret, accountHash string + for _, c := range []struct { + name, verb, caller string + args map[string]any + into any + filled []string // the json fields the holder reads, which must not be empty + refused string // a part of the refusal, when refused + }{ + // Only the seat mesh-enrolment is answered: not the controller, a node, a person or a caller not proven. + {"a node asks a token", "enrolment-token", "node.g14", map[string]any{"node": "new1", "tunnel": tunnel}, nil, nil, "mesh-enrolment"}, + {"the controller asks a token", "enrolment-token", "controller", map[string]any{"node": "new1", "tunnel": tunnel}, nil, nil, "mesh-enrolment"}, + {"a person asks an account", "node-account", "person.jochen", map[string]any{"node": "new1"}, nil, nil, "mesh-enrolment"}, + {"unproven asks an enrolment signed", "sign", "", map[string]any{"kind": delivery.KindEnrolment, "node": "new1", "reply": "AAEC"}, nil, nil, "proved"}, + {"a grants part asks an enrolment signed", "sign", "novox.mesh-grants", map[string]any{"kind": delivery.KindEnrolment, "node": "new1", "reply": "AAEC"}, nil, nil, "mesh-enrolment"}, + // Insight 2026-11-02: the first tunnel is whole at issue, or nothing is issued. + {"no tunnel", "enrolment-token", holder, map[string]any{"node": "new1"}, nil, nil, "first tunnel"}, + {"a tunnel without the hub's key", "enrolment-token", holder, map[string]any{"node": "new1", "tunnel": without("hub_key")}, nil, nil, "hub's tunnel key"}, + {"a tunnel without the hub's endpoint", "enrolment-token", holder, map[string]any{"node": "new1", "tunnel": without("hub_endpoint")}, nil, nil, "where the hub's tunnel is dialled"}, + {"a tunnel without the address", "enrolment-token", holder, map[string]any{"node": "new1", "tunnel": without("address")}, nil, nil, "this machine's address"}, + {"a tunnel without the machine's tunnel key", "enrolment-token", holder, map[string]any{"node": "new1", "tunnel": without("key")}, nil, nil, "the tunnel key it was issued for"}, + {"a tunnel without the range", "enrolment-token", holder, map[string]any{"node": "new1", "tunnel": without("range")}, nil, nil, "the private network's range"}, + {"a tunnel with a field it has not", "enrolment-token", holder, map[string]any{"node": "new1", "tunnel": with(tunnel, map[string]any{"port": 1})}, nil, nil, "not a first tunnel"}, + // Issue answers what the machine must trust; sealed and drop as the holder sends them, the switch as a word. + {"issue", "enrolment-token", holder, map[string]any{"node": "new1", "expires": "2026-11-02T12:00:00Z", "tunnel": tunnel}, &issued, []string{"node", "broker", "fingerprint", "signer"}, ""}, + {"sealed", "enrolment-token", holder, map[string]any{"node": "new1", "sealed": "true"}, &sealed, []string{"node", "sealed"}, ""}, + // The account at the redeem, twice with the same keys; other keys and an adopted node's are refused. + {"node-account", "node-account", holder, map[string]any{"node": "new1", "sealing_key": sealingPub, "keys_digest": digest}, &account, []string{"node", "login", "account"}, ""}, + {"node-account again", "node-account", holder, map[string]any{"node": "new1", "sealing_key": sealingPub, "keys_digest": digest}, &accountAgain, []string{"node", "login", "account"}, ""}, + {"node-account for other keys", "node-account", holder, map[string]any{"node": "new1", "sealing_key": sealingPub, "keys_digest": "sha256:other"}, nil, nil, "another set of keys"}, + {"node-account of an adopted node", "node-account", holder, map[string]any{"node": "g14", "sealing_key": sealingPub, "keys_digest": digest}, nil, nil, "never minted anew"}, + // The certificate for the serving key presented, checked against node-account's digest; the reply signed. + {"a certificate without the keys", "sign", holder, map[string]any{"kind": delivery.KindNodeCertificate, "node": "new1", "serving_key": keys["serving_key"]}, nil, nil, "keys it presented"}, + {"a certificate for another serving key", "sign", holder, with(keys, map[string]any{"kind": delivery.KindNodeCertificate, "node": "new1", "serving_key": base64.StdEncoding.EncodeToString(identity)}), nil, nil, "not the ones its account was made for"}, + {"a certificate", "sign", holder, with(keys, map[string]any{"kind": delivery.KindNodeCertificate, "node": "new1"}), &cert, []string{"kind", "node", "certificate", "authority", "signer", "digest"}, ""}, + {"the reply signed", "sign", holder, map[string]any{"kind": delivery.KindEnrolment, "node": "new1", "reply": base64.StdEncoding.EncodeToString(reply.SignedBytes())}, &replySig, []string{"kind", "node", "signature", "signer", "digest"}, ""}, + {"a reply of a node with no token", "sign", holder, map[string]any{"kind": delivery.KindEnrolment, "node": "g14", "reply": "AAEC"}, nil, nil, "no enrolment token"}, + // A revoke: the account retired, the token and its hash dropped. + {"retire", "node-account", holder, map[string]any{"node": "new1", "retire": "true"}, &retired, []string{"node", "retired"}, ""}, + {"drop", "enrolment-token", holder, map[string]any{"node": "new1", "drop": "true"}, &dropped, []string{"node", "dropped"}, ""}, + } { + raw, err := call(c.verb, c.caller, c.args) + if c.refused != "" { + if err == nil || !strings.Contains(err.Error(), c.refused) { + t.Errorf("%s: answered %s, %v; want a refusal saying %q", c.name, raw, err, c.refused) + } + continue + } + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + if err := json.Unmarshal(raw, c.into); err != nil { + t.Fatalf("%s: %s does not decode as mesh-sdk's shape: %v", c.name, raw, err) + } + var fields map[string]any + back, _ := json.Marshal(c.into) + _ = json.Unmarshal(back, &fields) + for _, f := range c.filled { + if x, ok := fields[f]; !ok || x == "" || x == false { + t.Errorf("%s: the wire answer %s leaves %q empty", c.name, raw, f) + } + } + if c.name == "issue" { + // What was kept: the secret, whose bcrypt hash is enrol.new1; listed, never handed out, never answered. + rec, held, _ := e.Store.Read("enrol/new1/token") + secret, _ = openAtRest(rec, e.Vault) + key := e.accounts.keys["enrol.new1"] + if !held || secret == "" || strings.Contains(string(raw), secret) || strings.Contains(strings.Join(e.bus.raw, " "), secret) || + key.Kept != KeptEnrolment || bcrypt.CompareHashAndPassword([]byte(key.Hash), []byte(secret)) != nil { + t.Fatalf("the token is held %v, its hash %+v, or its secret is in an answer or an event: %s", held, key, raw) + } + if held, _ := e.EnrolmentsHeld(); len(held) != 1 || held[0].User != "enrol.new1" { + t.Errorf("secret_holders names %+v", held) + } + } + if c.name == "node-account" { + accountHash = e.accounts.keys["node.new1"].Hash + if _, err := e.Get(holder, "enrol/new1/token", pemOf(t, mustPub(t))); err == nil { + t.Error("an enrolment token was handed out by get") + } + } + } + if issued.Broker != "tls://10.77.0.1:4222" || issued.Fingerprint != "sha256:bus" || issued.Signer != signerB64 || issued.Sealed != "" { + t.Errorf("issue answered %+v", issued) + } + // sealed issued nothing new: the token it seals is the one whose hash is enrol.new1, with its tunnel, and usable whole. + tok, err := delivery.ParseToken(openOperator(t, e.operatorPublic, e.operatorPrivate, sealed.Sealed)) + if err != nil || len(tok.Missing()) != 0 || tok.Node != "new1" || tok.Broker != issued.Broker || tok.Tunnel == nil || + tok.Tunnel.HubEndpoint != "203.0.113.1:51820" || base64.StdEncoding.EncodeToString(tok.Signer) != signerB64 || + tok.Secret != secret { + t.Errorf("the sealed token is %+v, %v", tok, err) + } + // The account opens with the sealing key presented, as the node-engine opens it, and is the same both times. + open := func(a delivery.NodeAccountAnswer) string { + blob, _ := base64.StdEncoding.DecodeString(a.Account) + var pub, priv [32]byte + copy(pub[:], sealingKey.PublicKey().Bytes()) + copy(priv[:], sealingKey.Bytes()) + out, _ := box.OpenAnonymous(nil, blob, &pub, &priv) + return string(out) + } + if p := open(account); p == "" || p != open(accountAgain) || account.Login != "node.new1" || + bcrypt.CompareHashAndPassword([]byte(accountHash), []byte(p)) != nil { + t.Errorf("the account is %+v / %+v", account, accountAgain) + } + // The certificate certifies the serving key presented, for new1's internal name. + block, _ := pem.Decode([]byte(cert.Certificate)) + if block == nil { + t.Fatalf("the certificate is %q", cert.Certificate) + } + parsed, err := x509.ParseCertificate(block.Bytes) + if err != nil || !ed25519.PublicKey(serving).Equal(parsed.PublicKey) { + t.Errorf("the certificate is of another key: %v", err) + } + ctx, _ := delivery.SignContext(delivery.KindEnrolment) + sig, _ := base64.StdEncoding.DecodeString(replySig.Signature) + if !ed25519.Verify(signer.key.Public().(ed25519.PublicKey), append([]byte(ctx), reply.SignedBytes()...), sig) { + t.Error("the redeem answer's signature does not verify under the enrolment context") + } + // After the retire and the drop nothing of the enrolment remains. + if _, ok := e.accounts.keys["node.new1"]; ok || e.logins.keys["new1"].Node != "" { + t.Error("the retired account's hash or node-logins key remains") + } + for _, name := range []string{"own/new1/mesh-host/node.new1", "enrol/new1/token", "enrol/new1/tunnel"} { + if _, held, _ := e.Store.Read(name); held { + t.Errorf("%s remains after the retire and the drop", name) + } + } + if _, ok := e.accounts.keys["enrol.new1"]; ok { + t.Error("the token's hash remains after a drop") + } +} + +func mustPub(t *testing.T) string { + t.Helper() + pub, _ := aSealingKey(t) + return pub +} + +// 774/S46/AC1-AC3, S47/AC1-AC2, S48/AC1, S49/AC1 (novox/hq ADR 0583 §1 to §7): a rotation is answered only for a hand act +// read back by its id; one runs at a time and the bus node waits for the rest; the new login is ready only when the loaded +// list holds it; a confirm from any caller but the new login retires nothing; a proven confirm removes the old hash and the +// rotation is done once the loaded list lacks it; an abandon keeps the old hash; and a node's login is its engine's alone. +func TestANodesLoginRotatesOneAtATimeAndEndsWithTheListWithoutTheOldOne(t *testing.T) { + e := newEnrolling(t) + ask := func(node, act string) (RotateNodeAnswer, error) { + return e.RotateNode("person.jochen", RotateNodeArgs{Node: node, Why: "a rotation under test", Act: act}) + } + // AC1: no recorded act, another caller's act, and an old act are all refused. + e.acts["theirs"] = HandActSeen{ID: "theirs", By: "person.other", At: time.Now().Add(time.Hour), Verb: "rotate-node", Args: []string{"g14"}} + e.acts["old"] = HandActSeen{ID: "old", By: "person.jochen", At: time.Unix(0, 0), Verb: "rotate-node", Args: []string{"g14"}} + e.acts["elsewhere"] = HandActSeen{ID: "elsewhere", By: "person.jochen", At: time.Now().Add(time.Hour), Verb: "rotate-node", Args: []string{"ace"}} + e.acts["restart"] = HandActSeen{ID: "restart", By: "person.jochen", At: time.Now().Add(time.Hour), Verb: "restart", Args: []string{"g14"}} + for _, act := range []string{"", "missing", "theirs", "old", "elsewhere", "restart"} { + if _, err := ask("g14", act); err == nil { + t.Errorf("act %q warranted a rotation", act) + } + } + if len(e.logins.keys) != 0 { + t.Fatalf("a refused rotation wrote %+v", e.logins.keys) + } + e.act("a1", "person.jochen", "g14") + e.act("a2", "person.jochen", "ace") + e.act("a3", "person.jochen", "novox") + first, err := ask("g14", "a1") + if err != nil || first.Queued || first.Rotating != "node.g14--b" || first.Stage != StageBegun { + t.Fatalf("g14 began as %+v, %v", first, err) + } + // The second login has a hash of its own value, the first is marked rotating, and the value waits for the engine. + second := e.accounts.keys["node.g14--b"] + rec, held, _ := e.Store.Read("own/g14/mesh-host/node.g14--b") + value, _ := openAtRest(rec, e.Vault) + var acct struct{ User, Password string } + if !held || json.Unmarshal([]byte(value), &acct) != nil || acct.User != "node.g14--b" || + bcrypt.CompareHashAndPassword([]byte(second.Hash), []byte(acct.Password)) != nil || e.accounts.keys["node.g14"].Rotating == nil { + t.Fatalf("g14's second login is held %v as %s, keyed %+v", held, value, second) + } + // AC2: one at a time, the bus node last. + ace, err := ask("ace", "a2") + if err != nil || !ace.Queued || strings.Join(ace.Behind, ",") != "g14" { + t.Fatalf("ace asked while g14 runs: %+v, %v", ace, err) + } + bus, err := ask("novox", "a3") + if err != nil || !bus.Queued || strings.Join(bus.Behind, ",") != "g14,ace" { + t.Fatalf("novox, the bus node, asked: %+v, %v", bus, err) + } + if _, err := ask("g14", "a1"); err == nil { + t.Error("g14 began a rotation twice") + } + // AC3: the loaded list lacks the second login, so nothing is ready; once it holds it, node-account-ready is said. + e.loadedListHeld("sha256:l1", map[string]string{"node.g14": "$2a$04$adopted"}) + e.advanceRotations() + if said(e.bus, NodeAccountReadyEvent) != 0 || e.logins.keys["g14"].Stage != StageBegun { + t.Fatalf("ready without the login in the loaded list: %+v", e.logins.keys["g14"]) + } + e.loadedListHeld("sha256:l2", map[string]string{"node.g14": "$2a$04$adopted", "node.g14--b": second.Hash}) + e.advanceRotations() + if said(e.bus, NodeAccountReadyEvent) != 1 || e.logins.keys["g14"].Stage != StageReady { + t.Fatalf("not ready with the login in the loaded list: %+v", e.logins.keys["g14"]) + } + // S49/AC1: the new login's value is g14's engine's alone, sealed to the key it stated. + stated, _, _ := e.Store.NodeKey("g14") + statedPEM, _ := publicPEM(stated) + if a, err := e.Get("node.g14", "own/g14/mesh-host/node.g14--b", statedPEM); err != nil || a.Sealed.Ct == "" { + t.Errorf("g14's engine was answered %+v, %v", a, err) + } + if _, err := e.Get("node.ace", "own/g14/mesh-host/node.g14--b", statedPEM); err == nil || !strings.Contains(err.Error(), "engine") { + t.Errorf("ace's engine asked for g14's login: %v", err) + } + if _, err := e.Get("node.g14", "own/g14/mesh-host/node.g14--b", pemOf(t, mustPub(t))); err == nil { + t.Error("g14's login was sealed to a key its engine did not state") + } + if own, err := e.OwnOfNode("node.ace"); err != nil || len(own.Own) != 0 { + t.Errorf("ace's engine was handed %+v, %v", own, err) + } + // S47/AC1: a confirm from the old login, from another node's login or from a person retires nothing. + for _, caller := range []string{"node.g14", "node.ace--b", "person.jochen", "controller"} { + if _, err := e.NodeAccountConfirm(caller); err == nil { + t.Errorf("%q confirmed g14's rotation", caller) + } + } + if _, ok := e.accounts.keys["node.g14"]; !ok || e.logins.keys["g14"].Stage != StageReady { + t.Fatalf("a refused confirm retired something: %+v", e.logins.keys["g14"]) + } + // S47/AC2: the proven confirm removes the old hash; done only when the loaded list lacks it. + if c, err := e.NodeAccountConfirm("node.g14--b"); err != nil || c.Stage != StageConfirmed { + t.Fatalf("the confirm answered %+v, %v", c, err) + } + if _, ok := e.accounts.keys["node.g14"]; ok || said(e.bus, RotationDoneEvent) != 0 { + t.Fatal("the old hash remains, or the rotation was done while the loaded list still held it") + } + e.loadedListHeld("sha256:l3", map[string]string{"node.g14--b": second.Hash}) + e.advanceRotations() + if nl := e.logins.keys["g14"]; said(e.bus, RotationDoneEvent) != 1 || nl.Login != "node.g14--b" || nl.Rotating != "" || nl.Stage != "" { + t.Fatalf("after the loaded list lacks the old login g14 is %+v", nl) + } + if _, held, _ := e.Store.Read("own/g14/mesh-host/node.g14"); held { + t.Error("the old login's value remains after the rotation is done") + } + if _, err := e.AbandonRotation("person.jochen", "g14"); err == nil { + t.Error("a finished rotation was abandoned") + } + // Blocker 5 (ADR 0583 §2): the act is recorded in g14's key and warrants no second rotation. + if acts := e.logins.keys["g14"].Acts; !slices.Equal(acts, []string{"a1"}) { + t.Errorf("g14's key records the acts %v", acts) + } + if _, err := ask("g14", "a1"); err == nil || !strings.Contains(err.Error(), "already") { + t.Errorf("a used hand act warranted another rotation: %v", err) + } + // The queue moves on, the bus node after every other: ace first; novox only once ace is done. + e.RotationPass() + if e.logins.keys["ace"].Rotating == "" || e.logins.keys["novox"].Rotating != "" { + t.Fatalf("after g14 the queue began ace=%+v novox=%+v", e.logins.keys["ace"], e.logins.keys["novox"]) + } + e.RotationPass() + if e.logins.keys["novox"].Rotating != "" { + t.Fatal("the bus node began while ace still rotated") + } + // S48/AC1: an abandon drops the new login and its value, keeps the old hash and clears the rotating part. + if _, err := e.AbandonRotation("person.jochen", "ace"); err != nil { + t.Fatal(err) + } + if _, ok := e.accounts.keys["node.ace--b"]; ok || e.accounts.keys["node.ace"].Rotating != nil || e.accounts.keys["node.ace"].Hash == "" || + e.logins.keys["ace"].Rotating != "" || e.logins.keys["ace"].Login != "node.ace" { + t.Errorf("an abandon left ace as %+v / %+v", e.accounts.keys["node.ace"], e.logins.keys["ace"]) + } + if _, held, _ := e.Store.Read("own/ace/mesh-host/node.ace--b"); held { + t.Error("an abandoned login's value remains") + } + e.RotationPass() + if e.logins.keys["novox"].Rotating == "" { + t.Fatal("the bus node did not begin once every other was done") + } +} + +// said counts the events of a kind the vault said. +func said(p *published, event string) int { + p.mu.Lock() + defer p.mu.Unlock() + n := 0 + for _, e := range p.events { + if e["event"] == event { + n++ + } + } + return n +} + +// 774/S48/AC2 (novox/hq ADR 0583 §1 insight): the key of a node that leaves the placement is deleted, heard from the +// placement's watch or found by the minute pass; one that cannot be read deletes nothing. +func TestANodesLoginsKeyGoesWhenItLeavesThePlacement(t *testing.T) { + e := newEnrolling(t) + for _, n := range []string{"g14", "gone", "gone2"} { + e.logins.keys[n] = NodeLogin{Node: n, Login: "node." + n} + } + e.NodeLeft("g14") + if _, ok := e.logins.keys["g14"]; !ok { + t.Error("g14 is placed, and its key was deleted") + } + e.NodeLeft("gone") + e.LoginsPass() + if len(e.logins.keys) != 1 || e.logins.keys["g14"].Node == "" { + t.Errorf("the keys after a node left and a pass: %+v", e.logins.keys) + } + e.logins.keys["gone"] = NodeLogin{Node: "gone"} + e.Placement = PlacementReader{} + e.LoginsPass() + if _, ok := e.logins.keys["gone"]; !ok { + t.Error("a placement that is not read deleted a key") + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/enrolment_tools.go b/modules/mesh-vault/cmd/mesh-vault/enrolment_tools.go new file mode 100644 index 00000000..7b5c244d --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/enrolment_tools.go @@ -0,0 +1,185 @@ +package main + +// The tools of enrolment and node rotation (novox/hq ADR 0587, ADR 0583): the seat's verbs the vault serves, and the +// tools that read what it holds of them. Arguments are strings and booleans the runtime hands; the caller is the one the +// runtime proved. + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// EnrolmentHeld is one enrolment token the vault holds, as secret_holders lists it: never the token or its hash. +type EnrolmentHeld struct { + Node string `json:"node"` + User string `json:"user"` + Generation int `json:"generation"` + Fingerprint string `json:"fingerprint"` + At time.Time `json:"at"` + // Account says the node's own account is made too (node-account answered). + Account bool `json:"account_made"` +} + +// EnrolmentsHeld lists the enrolment tokens held, by node. +func (v *Vault) EnrolmentsHeld() ([]EnrolmentHeld, error) { + out := []EnrolmentHeld{} + if v.Accounts == nil { + return out, nil + } + users, err := v.Accounts.Users() + if err != nil { + return out, err + } + for _, u := range users { + a, err := v.Accounts.Get(u) + if err != nil || a == nil || a.Kept != KeptEnrolment { + continue + } + node := u[len(EnrolUserPrefix):] + n, _ := v.Accounts.Get(NodeEnginePrefix + node) + out = append(out, EnrolmentHeld{Node: node, User: u, Generation: a.Generation, Fingerprint: a.Fingerprint, At: a.At, + Account: n != nil && n.Kept == KeptNode}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node }) + return out, nil +} + +func enrolmentTools(env *Env) []toolDef { + node := str("the node's name") + return []toolDef{ + {Name: Seat + ".enrolment-token", + Description: "The enrolment token of a joining machine, for the seat mesh-enrolment alone (novox/hq ADR 0587 §3, §5, " + + "§8). Issue (the default): takes the machine's first tunnel, makes the one-time secret, keeps both with the operator " + + "copy, writes the secret's hash as the bus user enrol., and answers the bus's address, its certificate's " + + "fingerprint and the vault's signing key, never the token. sealed: the whole token sealed to the operator key, " + + "for the operator to open at their terminal. drop: the token and its hash gone, at every end of an enrolment.", + Input: map[string]any{"node": node, "expires": str("when the token ends, as the seat states it"), + "tunnel": map[string]any{"type": "object", "description": "the machine's first tunnel at issue: hub_key, " + + "hub_endpoint, address (and key, range)"}, + "sealed": map[string]any{"type": "boolean", "description": "answer the token sealed to the operator key"}, + "drop": map[string]any{"type": "boolean", "description": "drop the token and its hash"}}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + tunnel, err := tunnelArg(a["tunnel"]) + if err != nil { + return nil, err + } + return v.EnrolmentToken(caller, EnrolmentTokenArgs{Node: arg(a, "node"), Expires: arg(a, "expires"), + Tunnel: tunnel, Sealed: isTrue(a["sealed"]), Drop: isTrue(a["drop"])}) + }}, + {Name: Seat + ".node-account", + Description: "A joining node's account node., for the seat mesh-enrolment alone at the redeem (novox/hq ADR 0587 " + + "§3): made once for a node and the keys it presented, answered sealed to the sealing key presented, with the " + + "node's key in node-logins naming the login in use. Asked again for the same keys it answers the same account. " + + "retire: the account, its value and its key taken away at a revoke.", + Input: map[string]any{"node": node, "sealing_key": str("the machine's X25519 sealing key, raw in base64 or SPKI PEM"), + "keys_digest": str("the digest of the keys the machine presented"), + "retire": map[string]any{"type": "boolean", "description": "take the account away"}}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.NodeAccount(caller, NodeAccountArgs{Node: arg(a, "node"), SealingKey: arg(a, "sealing_key"), + KeysDigest: arg(a, "keys_digest"), Retire: isTrue(a["retire"])}) + }}, + {Name: Seat + ".rotate-node", + Description: "Begin the rotation of a node's login (novox/hq ADR 0583 §2, §7): answered only for a call that carries a hand " + + "act recorded through mesh-hand-acts, read back by its id. One rotation at a time mesh-wide: a call asked while " + + "another runs is queued and begun when it is done, the bus node after every other. Makes the other login " + + "(node.--b or node.), keeps its value for the node's engine, and says rotation-begun.", + Input: map[string]any{"node": node, "why": str("why the login rotates"), + "act": str("the id of the hand act recorded through mesh-hand-acts that warrants it")}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.RotateNode(caller, RotateNodeArgs{Node: arg(a, "node"), Why: arg(a, "why"), Act: arg(a, "act")}) + }}, + {Name: Seat + ".node-account-confirm", + Description: "A node-engine, dialled as its new login, says it connects with it (novox/hq ADR 0583 §3, §4): the runtime " + + "proves the login, the old login's hash is removed, and rotation-done follows when the loaded list lacks it.", + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.NodeAccountConfirm(caller) + }}, + {Name: Seat + ".abandon", + Description: "Abandon a node's rotation before its confirmation (novox/hq ADR 0583 §5): the new login and its value are " + + "dropped, the old hash kept, the rotating part of the node's key cleared; a rotation only asked leaves the queue.", + Input: map[string]any{"node": node}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.AbandonRotation(caller, arg(a, "node")) + }}, + {Name: "node_logins", + Description: "Every node's key of the vault's state node-logins: the login its node-engine dials as and, while a rotation " + + "runs, the other login and its stage; with the rotations queued and the enrolment tokens held. Never a value or a hash.", + run: func(string, map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + if v.Logins == nil { + return nil, errors.New("the vault has no state node-logins") + } + keys, err := v.Logins.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + rows := []NodeLogin{} + for _, k := range keys { + if nl, err := v.Logins.Get(k); err == nil && nl != nil { + rows = append(rows, *nl) + } + } + held, _ := v.EnrolmentsHeld() + return map[string]any{"logins": rows, "rotation": v.RotationState(), "enrolments": held}, nil + }}, + } +} + +// tunnelArg reads enrolment-token's `tunnel`, an object or its JSON as a string, as mesh-sdk's TokenTunnel; nil when +// absent. +func tunnelArg(v any) (*delivery.TokenTunnel, error) { + var raw []byte + switch t := v.(type) { + case nil: + return nil, nil + case string: + if strings.TrimSpace(t) == "" { + return nil, nil + } + raw = []byte(t) + default: + b, err := json.Marshal(t) + if err != nil { + return nil, err + } + raw = b + } + var tt delivery.TokenTunnel + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&tt); err != nil { + return nil, fmt.Errorf("`tunnel` is not a first tunnel (hub_key, hub_endpoint, address, key, range): %w", err) + } + return &tt, nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/main.go b/modules/mesh-vault/cmd/mesh-vault/main.go index 089c2abc..a76ac978 100644 --- a/modules/mesh-vault/cmd/mesh-vault/main.go +++ b/modules/mesh-vault/cmd/mesh-vault/main.go @@ -60,7 +60,8 @@ func build() *Env { Assignments: Mirror{Keys: assignmentKeys, Read: readAssignment}, Emit: emit, Ask: stdio.Ask, Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, LeaseRead: readLease, WantRead: readWant, HaveKeys: func() ([]string, error) { return stdio.State(HaveRead).Keys() }, Placement: placement, Signers: busSigners{}, - AskCallers: askCallers(os.Getenv("MESH_VAULT_SECRET_ASK_CALLERS")), + AskCallers: askCallers(os.Getenv("MESH_VAULT_SECRET_ASK_CALLERS")), EnrolCallers: askCallers(os.Getenv("MESH_VAULT_ENROLMENT_CALLERS")), + Logins: busNodeLogins{}, ReadAct: readHandAct, TokenBase: TokenBaseFromFile(os.Getenv("MESH_VAULT_STATE")), Bound: Bound{ Get: func(key string) ([]byte, error) { e, err := stdio.State(BoundState).Get(key) @@ -86,6 +87,46 @@ func build() *Env { return env } +// readHandAct reads a hand act back by its id through the seat mesh-hand-acts (the vault's manifest invokes it): nil when +// the log holds none. +func readHandAct(id string) (*HandActSeen, error) { + raw, err := stdio.Ask("mesh-hand-acts.hand-acts", map[string]any{"days": 2}) + if err != nil { + return nil, err + } + var listing struct { + Acts []HandActSeen `json:"acts"` + } + if err := json.Unmarshal(raw, &listing); err != nil { + return nil, fmt.Errorf("mesh-hand-acts answered what is not its listing: %w", err) + } + for _, a := range listing.Acts { + if a.ID == id { + return &a, nil + } + } + return nil, nil +} + +// watchLeavers deletes a node's key of node-logins when its node key of the placement is deleted (noderotation.go): a +// watch, never a poll. It answers whether the watch began; the minute pass (LoginsPass) catches what it missed. +func watchLeavers(env *Env) bool { + if env.Vault == nil { + return true + } + err := stdio.State(PlacementRead).Watch("*.node", func(c stdio.StateChange) error { + if c.Op == "delete" || c.Op == "purge" || len(c.Value) == 0 { + env.Vault.NodeLeft(strings.TrimSuffix(c.Key, ".node")) + } + return nil + }) + if err != nil { + say("the placement's nodes are not watched for one that leaves; the minute pass finds it: %v", err) + return false + } + return true +} + // assigned is the assignments mirror as the vault's watch keeps it. var assigned *WatchedAssignments @@ -280,7 +321,11 @@ func main() { watched, watching := watchHandings(env), watchAssignments() lease := watchLease(env) wants := watchWants(env) + leavers := watchLeavers(env) for { + if !leavers { + leavers = watchLeavers(env) + } if !watched { watched = watchHandings(env) } @@ -304,6 +349,8 @@ func main() { env.Vault.HashOwed() env.Vault.MintOwed() env.Vault.RetirePass() + env.Vault.RotationPass() + env.Vault.LoginsPass() } env.pairPass() if env.Custody != nil { diff --git a/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go b/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go index 821a03c7..62b39a82 100644 --- a/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go @@ -60,6 +60,8 @@ type player struct { next int pending map[string]chan wire said []string + // ask answers a mesh/ask (a call of another module's tool); nil answers none, as a call with no responder. + ask func(key string, body json.RawMessage) (any, error) } // bucketOf is the bucket a state names: the vault's own by its local name, another's as `.`. @@ -215,8 +217,20 @@ func (p *player) handle(m wire) { p.said = append(p.said, e.Key+" "+string(e.Body)) p.mu.Unlock() p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod.mesh-vault.event."+e.Key, e.Body)) + case "mesh/ask": + var ask struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &ask) + if p.ask != nil { + result, err := p.ask(ask.Key, ask.Body) + p.answer(m.ID, result, err) + return + } + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) default: - // mesh/ask and the rest: nothing answers on this bus, as a call with no responder. + // the rest: nothing answers on this bus, as a call with no responder. p.answer(m.ID, nil, errors.New("nats: no responders available for request")) } } diff --git a/modules/mesh-vault/cmd/mesh-vault/noderotation.go b/modules/mesh-vault/cmd/mesh-vault/noderotation.go new file mode 100644 index 00000000..9c074f3f --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/noderotation.go @@ -0,0 +1,681 @@ +package main + +// A node's two logins and their rotation (novox/hq ADR 0583 §1 to §7 and its insights; issue 774). +// +// A node-engine dials the bus as `node.` or as `node.--b`, the two logins alternating. The vault's state +// `node-logins` holds one key per node, ``: the login in use, and while a rotation runs the other login with its stage +// (begun, ready, confirmed). Each node-engine reads its own key alone (the bus grants it that read); the vault writes it +// alone. +// +// `rotate-node` begins a rotation: answered only for a call that carries a hand act recorded through mesh-hand-acts, read +// back by its id; one rotation at a time mesh-wide, a call asked meanwhile is queued and begun at the pass that finds the +// first done; the bus node last, after every other node asked. It makes the other login's value, keeps it as +// `own//mesh-host/` (sealed, with the operator copy; the node-engine takes it through `own`, sealed to its own +// key), writes its hash as the key `` of bus-accounts and marks the login in use as rotating, so the grants part +// composes both. `node-account-ready` is said only when the user list the bus node loaded holds the new login's hash. +// `node-account-confirm` is the node-engine's, dialled as the new login, which the runtime proves: the old login's hash +// is then removed, and `rotation-done` is said once the loaded list lacks it, when the old value is dropped and the +// rotating part of the key cleared. `abandon` drops the new login and its value before the confirmation and keeps the +// old hash. A key whose node leaves the placement is deleted. + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "golang.org/x/crypto/bcrypt" +) + +// The events a rotation says and the stages of a node's key are mesh-sdk's (delivery/node_logins.go); a rotation warranted +// by a hand act is warranted for a day. +const ( + RotationBegunEvent = delivery.RotationBegunEvent + NodeAccountReadyEvent = delivery.NodeAccountReadyEvent + RotationDoneEvent = delivery.RotationDoneEvent + + StageBegun = delivery.RotationBegun + StageReady = delivery.RotationReady + StageConfirmed = delivery.RotationConfirmed + + // HandActWithin is how recent a hand act is that warrants a rotation. + HandActWithin = 24 * time.Hour +) + +// NodeLogin is one key of the vault's state node-logins (mesh-sdk's contract): the login the node's node-engine dials as, +// and while a rotation runs the other login with its stage. Never a value or a hash. +type NodeLogin = delivery.NodeLogin + +// NodeLoginStore is the vault's state `node-logins`, by node. +type NodeLoginStore interface { + Get(node string) (*NodeLogin, error) + Put(n NodeLogin) error + Delete(node string) error + Keys() ([]string, error) +} + +// busNodeLogins is NodeLoginStore over the vault's own state through its runtime. +type busNodeLogins struct{} + +func (busNodeLogins) Get(node string) (*NodeLogin, error) { + e, err := stdio.State(NodeLoginsState).Get(node) + if err != nil || e == nil { + return nil, err + } + n, err := delivery.DecodeNodeLogin(e.Value) + if err != nil { + return nil, fmt.Errorf("%s %s is not a node's login: %w", NodeLoginsState, node, err) + } + return &n, nil +} + +func (busNodeLogins) Put(n NodeLogin) error { + raw, err := delivery.EncodeNodeLogin(n) + if err != nil { + return err + } + _, err = stdio.State(NodeLoginsState).Put(n.Node, json.RawMessage(raw)) + return err +} + +func (busNodeLogins) Delete(node string) error { return stdio.State(NodeLoginsState).Delete(node) } + +func (busNodeLogins) Keys() ([]string, error) { return stdio.State(NodeLoginsState).Keys() } + +// HandActSeen is a hand act as the vault reads it back from mesh-hand-acts by its id (mesh-sdk's handacts.Act): who +// recorded it, when, and what it was (its verb and arguments). +type HandActSeen struct { + ID string `json:"id"` + By string `json:"by"` + At time.Time `json:"at"` + Verb string `json:"verb"` + Args []string `json:"arguments,omitempty"` +} + +// ActReader reads a hand act back by its id: nil when mesh-hand-acts holds none. +type ActReader func(id string) (*HandActSeen, error) + +// queuedRotation is a `rotate-node` asked while another ran, begun at the pass that finds none running. +type queuedRotation struct { + Node string `json:"node"` + By string `json:"by"` + Why string `json:"why"` + Act string `json:"act"` + At time.Time `json:"at"` +} + +// nodeRotation is what the vault keeps in memory of its rotations: the user list the bus node loaded (user to hash), and +// the rotations queued. The record is node-logins and bus-accounts; a restart forgets the queue, and the one who asked +// asks again. +type nodeRotation struct { + mu sync.Mutex + loaded map[string]string + loadedFrom string + queue []queuedRotation +} + +func (v *Vault) getLogin(node string) (*NodeLogin, error) { + if v.Logins == nil { + return nil, errors.New("the vault has no state " + NodeLoginsState + " to read") + } + return v.Logins.Get(node) +} + +// putLogin writes node's key; a key written without its hand acts keeps the ones recorded (ADR 0583 §2). +func (v *Vault) putLogin(n NodeLogin) error { + if v.Logins == nil { + return errors.New("the vault has no state " + NodeLoginsState + " to write") + } + if n.Acts == nil { + was, err := v.Logins.Get(n.Node) + if err != nil { + return err + } + if was != nil { + n.Acts = was.Acts + } + } + return v.Logins.Put(n) +} + +func (v *Vault) deleteLogin(node string) error { + if v.Logins == nil { + return nil + } + return v.Logins.Delete(node) +} + +// currentLogin is the login node dials as: its key's, or `node.` before any key is written. +func (v *Vault) currentLogin(node string) (string, error) { + nl, err := v.getLogin(node) + if err != nil { + return "", err + } + if nl != nil && nl.Login != "" { + return nl.Login, nil + } + return NodeEnginePrefix + node, nil +} + +// otherLogin is the login a node alternates to from login. +func otherLogin(login string) string { + if base, second := strings.CutSuffix(login, SecondLoginSuffix); second { + return base + } + return login + SecondLoginSuffix +} + +// loginNode is the node whose login a bus user is: `node.` or `node.--b`. +func loginNode(user string) (string, bool) { + rest, ok := strings.CutPrefix(user, NodeEnginePrefix) + if !ok { + return "", false + } + node := strings.TrimSuffix(rest, SecondLoginSuffix) + return node, node != "" && enrolNode(node) == nil +} + +// checkAct reads the hand act back by its id and refuses a rotation it does not warrant: none recorded, recorded by +// another caller, older than a day, or an act that is not rotate-node of this node. +func (v *Vault) checkAct(caller, node, id string) error { + if strings.TrimSpace(id) == "" { + return errors.New("rotate-node carries the hand act that warrants it (`act`): record it through mesh-hand-acts first (novox/hq ADR 0583 §2)") + } + if v.ReadAct == nil { + return errors.New("the vault reads no hand acts, so it answers no rotation (novox/hq ADR 0583 §2)") + } + act, err := v.ReadAct(id) + if err != nil { + return fmt.Errorf("the hand act %s could not be read back from mesh-hand-acts: %w", id, err) + } + switch { + case act == nil: + return fmt.Errorf("mesh-hand-acts holds no hand act %s: record one first (novox/hq ADR 0583 §2)", id) + case act.By != caller: + return fmt.Errorf("the hand act %s was recorded by %s, not by %s who asks: a rotation is warranted by its own caller's act", id, act.By, caller) + case v.Now().Sub(act.At) > HandActWithin: + return fmt.Errorf("the hand act %s is older than %s: record a new one", id, HandActWithin) + case act.Verb != delivery.RotateNodeVerb && !strings.HasSuffix(act.Verb, "."+delivery.RotateNodeVerb): + return fmt.Errorf("the hand act %s is %q, not %s: it warrants no rotation", id, act.Verb, delivery.RotateNodeVerb) + case !slices.Contains(act.Args, node): + return fmt.Errorf("the hand act %s is a rotation of %s, not of %s", id, strings.Join(act.Args, " "), node) + } + return nil +} + +// rotatingNodes are the nodes whose key says a rotation runs, sorted. +func (v *Vault) rotatingNodes() ([]NodeLogin, error) { + if v.Logins == nil { + return nil, nil + } + keys, err := v.Logins.Keys() + if err != nil { + return nil, err + } + var out []NodeLogin + for _, k := range keys { + nl, err := v.Logins.Get(k) + if err != nil { + return nil, err + } + if nl != nil && nl.Rotating != "" { + out = append(out, *nl) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node }) + return out, nil +} + +// RotateNodeArgs are `rotate-node`'s arguments. +type RotateNodeArgs struct { + Node string + Why string + Act string +} + +// RotateNodeAnswer says a rotation began, or that it waits for the ones before it. Never a value. +type RotateNodeAnswer struct { + Node string `json:"node"` + Login string `json:"login,omitempty"` + Rotating string `json:"rotating,omitempty"` + Stage string `json:"stage,omitempty"` + Revision uint64 `json:"list_revision,omitempty"` + Queued bool `json:"queued,omitempty"` + // Behind are the nodes whose rotation it waits for, running or asked before. + Behind []string `json:"behind,omitempty"` +} + +// RotateNode answers caller's `rotate-node`. +func (v *Vault) RotateNode(caller string, a RotateNodeArgs) (RotateNodeAnswer, error) { + if err := fromAGrantedCaller(caller, "rotate-node"); err != nil { + return RotateNodeAnswer{}, err + } + if err := enrolNode(a.Node); err != nil { + return RotateNodeAnswer{}, err + } + if strings.TrimSpace(a.Why) == "" { + return RotateNodeAnswer{}, errors.New("rotate-node says why (`why`)") + } + if err := v.checkAct(caller, a.Node, a.Act); err != nil { + return RotateNodeAnswer{}, err + } + if v.Accounts == nil { + return RotateNodeAnswer{}, errors.New("the vault has no state of bus accounts to write to") + } + v.leaseAfresh() + if on, why := v.leaseHanded(); !on { + return RotateNodeAnswer{}, fmt.Errorf("no rotation begins: %s", why) + } + v.minting.mu.Lock() + revision := v.minting.loadedRevision + v.minting.mu.Unlock() + if revision == 0 { + return RotateNodeAnswer{}, errors.New("no rotation begins: the revision of the user list the bus loaded is not known yet") + } + bus, err := v.busNode() + if err != nil { + return RotateNodeAnswer{}, fmt.Errorf("no rotation is ordered: the node holding the bus is not known (the bus node rotates last): %w", err) + } + v.nr.mu.Lock() + defer v.nr.mu.Unlock() + if nl, err := v.getLogin(a.Node); err != nil { + return RotateNodeAnswer{}, err + } else if nl != nil && nl.Rotating != "" { + return RotateNodeAnswer{}, fmt.Errorf("%s is rotating already, since %s (second login %s)", a.Node, nl.Since.Format(time.RFC3339), nl.Rotating) + } + for _, q := range v.nr.queue { + if q.Node == a.Node { + return RotateNodeAnswer{}, fmt.Errorf("%s's rotation is asked already, by %s, and waits its turn", a.Node, q.By) + } + } + if err := v.recordAct(a.Node, a.Act); err != nil { + return RotateNodeAnswer{}, err + } + running, err := v.rotatingNodes() + if err != nil { + return RotateNodeAnswer{}, err + } + var behind []string + for _, r := range running { + behind = append(behind, r.Node) + } + if a.Node == bus { + for _, q := range v.nr.queue { + behind = append(behind, q.Node) + } + } + if len(behind) > 0 { + v.nr.queue = append(v.nr.queue, queuedRotation{Node: a.Node, By: caller, Why: a.Why, Act: a.Act, At: v.Now().UTC()}) + v.log("the rotation of %s waits for %s (asked by %s): one rotation at a time, the bus node %s last", a.Node, + strings.Join(behind, ", "), caller, bus) + return RotateNodeAnswer{Node: a.Node, Queued: true, Behind: behind}, nil + } + nl, err := v.beginNodeRotation(caller, a.Node, a.Why, a.Act, revision) + if err != nil { + return RotateNodeAnswer{}, err + } + return RotateNodeAnswer{Node: a.Node, Login: nl.Login, Rotating: nl.Rotating, Stage: nl.Stage, Revision: revision}, nil +} + +// recordAct keeps the hand act in node's key of node-logins, refusing one already used: an act warrants one rotation +// (ADR 0583 §2). Under nr.mu. +func (v *Vault) recordAct(node, act string) error { + if v.Logins == nil { + return errors.New("the vault has no state " + NodeLoginsState + " to record the rotation in") + } + nl, err := v.getLogin(node) + if err != nil { + return err + } + if nl != nil && slices.Contains(nl.Acts, act) { + return fmt.Errorf("the hand act %s warranted a rotation of %s already: record a new one", act, node) + } + if nl == nil { + cur, err := v.currentLogin(node) + if err != nil { + return err + } + nl = &NodeLogin{Node: node, Login: cur, Since: v.Now().UTC()} + } + nl.Acts = append(slices.Clone(nl.Acts), act) + if err := v.Logins.Put(*nl); err != nil { + return fmt.Errorf("the hand act %s could not be recorded in %s %s: %w", act, NodeLoginsState, node, err) + } + return nil +} + +// beginNodeRotation makes the other login and records the rotation; under nr.mu. +func (v *Vault) beginNodeRotation(by, node, why, act string, revision uint64) (NodeLogin, error) { + cur, err := v.currentLogin(node) + if err != nil { + return NodeLogin{}, err + } + curKey, err := v.Accounts.Get(cur) + if err != nil { + return NodeLogin{}, fmt.Errorf("%s %s could not be read: %w", BusAccountsState, cur, err) + } + if curKey == nil { + return NodeLogin{}, fmt.Errorf("%s holds no key of %s, the login %s dials as: there is nothing to rotate", BusAccountsState, cur, node) + } + second := otherLogin(cur) + value, password, err := v.accountValue(node, second) + if err != nil { + return NodeLogin{}, err + } + secret := loginSecret(node, second) + c, err := v.keep(secret, value, "the vault", "", OriginMade) + if err != nil { + return NodeLogin{}, fmt.Errorf("the login %s is not kept: %w", second, err) + } + hash, err := bcrypt.GenerateFromPassword([]byte(password), v.hashCost()) + if err != nil { + return NodeLogin{}, err + } + at := v.Now().UTC() + fp := Fingerprint(value) + if err := v.Accounts.Put(AccountHash{User: second, Hash: string(hash), Of: secret, Generation: c.Generation, + Fingerprint: fp, At: at, Kept: KeptNode}); err != nil { + return NodeLogin{}, fmt.Errorf("the hash of %s could not be written to %s: %w", second, BusAccountsState, err) + } + v.saidHashed(second, secret, c.Generation, fp, at) + marked := *curKey + marked.Rotating = &Rotating{Login: second, Generation: c.Generation, Since: at, Revision: revision} + if err := v.Accounts.Put(marked); err != nil { + return NodeLogin{}, fmt.Errorf("%s could not be marked rotating: %w", cur, err) + } + nl := NodeLogin{Node: node, Login: cur, Rotating: second, Stage: StageBegun, Since: at, ListRevision: revision} + if err := v.putLogin(nl); err != nil { + return NodeLogin{}, fmt.Errorf("%s %s could not be written: %w", NodeLoginsState, node, err) + } + v.sayRotation(RotationBegunEvent, nl, map[string]any{"why": why, "act": act, "by": by}) + v.log("the rotation of %s begins at the loaded list's revision %d: %s alternates to %s (asked by %s, hand act %s)", node, + revision, cur, second, by, act) + return nl, nil +} + +// sayRotation says an event of a rotation: the node, its logins, the stage; never a value or a hash. +func (v *Vault) sayRotation(event string, nl NodeLogin, extra map[string]any) { + if v.Emit == nil { + return + } + body := map[string]any{"node": nl.Node, "login": nl.Login, "rotating": nl.Rotating, "stage": nl.Stage, + "at": v.Now().UTC().Format(time.RFC3339Nano)} + for k, x := range extra { + body[k] = x + } + if err := v.Emit(event, body); err != nil { + v.log("the rotation of %s moved, but %s could not be said: %v", nl.Node, event, err) + } +} + +// loadedListHeld keeps the users the bus node's loaded list holds, with their hashes (what its handing opened to). +func (v *Vault) loadedListHeld(from string, users map[string]string) { + v.nr.mu.Lock() + v.nr.loaded, v.nr.loadedFrom = users, from + v.nr.mu.Unlock() +} + +// loadedHolds says the loaded list holds user with exactly hash; loadedLacks that it is known and holds no user. +func (v *Vault) loadedHolds(user, hash string) bool { + v.nr.mu.Lock() + defer v.nr.mu.Unlock() + return v.nr.loaded != nil && hash != "" && v.nr.loaded[user] == hash +} + +func (v *Vault) loadedLacks(user string) bool { + v.nr.mu.Lock() + defer v.nr.mu.Unlock() + if len(v.nr.loaded) == 0 { + return false + } + _, held := v.nr.loaded[user] + return !held +} + +// advanceRotations moves every rotation as far as the loaded list lets it: begun to ready when the list holds the new +// login, confirmed to done when it lacks the old one. At each adoption of the loaded list and at the minute pass. +func (v *Vault) advanceRotations() { + running, err := v.rotatingNodes() + if err != nil { + v.log("the rotations could not be read: %v", err) + return + } + for _, nl := range running { + switch nl.Stage { + case StageBegun: + k, err := v.Accounts.Get(nl.Rotating) + if err != nil || k == nil || !v.loadedHolds(nl.Rotating, k.Hash) { + continue + } + nl.Stage = StageReady + if err := v.putLogin(nl); err != nil { + v.log("%s is ready, and %s could not be written: %v", nl.Rotating, NodeLoginsState, err) + continue + } + v.sayRotation(NodeAccountReadyEvent, nl, nil) + v.log("the loaded user list holds %s: %s may dial as it", nl.Rotating, nl.Node) + case StageConfirmed: + if !v.loadedLacks(nl.Rotating) { + continue + } + v.finishRotation(nl) + } + } +} + +// finishRotation ends a confirmed rotation whose old hash the loaded list lacks: the old value dropped, the rotating part +// of the key cleared, rotation-done said. +func (v *Vault) finishRotation(nl NodeLogin) { + old := nl.Rotating + if err := v.Store.Delete(loginSecret(nl.Node, old)); err != nil { + v.log("the old login %s of %s is out of the list, and its value could not be dropped: %v", old, nl.Node, err) + return + } + done := NodeLogin{Node: nl.Node, Login: nl.Login, Since: v.Now().UTC()} + if err := v.putLogin(done); err != nil { + v.log("the rotation of %s is done, and %s could not be written: %v", nl.Node, NodeLoginsState, err) + return + } + v.sayRotation(RotationDoneEvent, done, map[string]any{"dropped": old}) + v.log("the rotation of %s is done: it dials as %s, and %s is gone from the list and the vault", nl.Node, nl.Login, old) +} + +// ConfirmAnswer is what `node-account-confirm` answers. +type ConfirmAnswer struct { + Node string `json:"node"` + Login string `json:"login"` + Stage string `json:"stage"` +} + +// NodeAccountConfirm answers `node-account-confirm`: the node-engine, dialled as the new login (which the runtime +// proves), says it connects with it. The old login's hash is then removed from bus-accounts; rotation-done follows when +// the loaded list lacks it. +func (v *Vault) NodeAccountConfirm(caller string) (ConfirmAnswer, error) { + if err := fromAGrantedCaller(caller, "node-account-confirm"); err != nil { + return ConfirmAnswer{}, err + } + node, ok := loginNode(caller) + if !ok { + return ConfirmAnswer{}, fmt.Errorf("%s is no node-engine's login: nothing is retired", caller) + } + nl, err := v.getLogin(node) + if err != nil { + return ConfirmAnswer{}, err + } + if nl == nil || nl.Rotating != caller || (nl.Stage != StageBegun && nl.Stage != StageReady) { + return ConfirmAnswer{}, fmt.Errorf("%s is not the new login of a rotation of %s under way: nothing is retired", caller, node) + } + old := nl.Login + if err := v.Accounts.Delete(old); err != nil { + return ConfirmAnswer{}, fmt.Errorf("the hash of %s could not be taken out of %s: %w", old, BusAccountsState, err) + } + confirmed := NodeLogin{Node: node, Login: caller, Rotating: old, Stage: StageConfirmed, Since: v.Now().UTC(), ListRevision: nl.ListRevision} + if err := v.putLogin(confirmed); err != nil { + return ConfirmAnswer{}, fmt.Errorf("the old hash of %s is out, and %s %s could not be written: %w", node, NodeLoginsState, node, err) + } + v.log("%s confirmed %s: the hash of %s is removed from %s", node, caller, old, BusAccountsState) + v.advanceRotations() + return ConfirmAnswer{Node: node, Login: caller, Stage: StageConfirmed}, nil +} + +// AbandonAnswer says what `abandon` took away. +type AbandonAnswer struct { + Node string `json:"node"` + Login string `json:"login,omitempty"` + Dropped string `json:"dropped,omitempty"` + Queued bool `json:"was_queued,omitempty"` +} + +// AbandonRotation answers `abandon`: the new login and its value are dropped, the old hash kept and the rotating part +// cleared; a rotation only asked is taken off the queue. Not after the confirmation, when the old hash is already gone. +func (v *Vault) AbandonRotation(caller, node string) (AbandonAnswer, error) { + if err := fromAGrantedCaller(caller, "abandon"); err != nil { + return AbandonAnswer{}, err + } + if err := enrolNode(node); err != nil { + return AbandonAnswer{}, err + } + v.nr.mu.Lock() + defer v.nr.mu.Unlock() + out := AbandonAnswer{Node: node} + if i := slices.IndexFunc(v.nr.queue, func(q queuedRotation) bool { return q.Node == node }); i >= 0 { + v.nr.queue = slices.Delete(v.nr.queue, i, i+1) + out.Queued = true + } + nl, err := v.getLogin(node) + if err != nil { + return out, err + } + if nl == nil || nl.Rotating == "" { + if out.Queued { + return out, nil + } + return out, fmt.Errorf("%s rotates nothing: nothing was abandoned", node) + } + if nl.Stage == StageConfirmed { + return out, fmt.Errorf("the rotation of %s is confirmed: the old login %s is out of the list, and %s is the login in use; rotate again to change it", + node, nl.Rotating, nl.Login) + } + second := nl.Rotating + if err := v.Accounts.Delete(second); err != nil { + return out, fmt.Errorf("the hash of %s could not be taken out of %s: %w", second, BusAccountsState, err) + } + if err := v.Store.Delete(loginSecret(node, second)); err != nil { + return out, fmt.Errorf("the value of %s could not be dropped: %w", second, err) + } + if k, err := v.Accounts.Get(nl.Login); err == nil && k != nil && k.Rotating != nil { + cleared := *k + cleared.Rotating = nil + if err := v.Accounts.Put(cleared); err != nil { + return out, fmt.Errorf("%s could not be marked as not rotating: %w", nl.Login, err) + } + } + if err := v.putLogin(NodeLogin{Node: node, Login: nl.Login, Since: v.Now().UTC()}); err != nil { + return out, fmt.Errorf("%s %s could not be written: %w", NodeLoginsState, node, err) + } + _ = v.recordHanding(Handing{To: caller, Name: loginSecret(node, second), Outcome: "dropped", Why: "rotation of " + node + " abandoned"}) + v.log("the rotation of %s is abandoned by %s: %s and its value are dropped, %s keeps its hash", node, caller, second, nl.Login) + out.Login, out.Dropped = nl.Login, second + return out, nil +} + +// RotationPass is the minute pass of rotations: each moves as far as the loaded list lets it, and the next queued one +// begins when none runs (the bus node after every other). +func (v *Vault) RotationPass() { + if v.Logins == nil || v.Accounts == nil { + return + } + v.advanceRotations() + running, err := v.rotatingNodes() + if err != nil || len(running) > 0 { + return + } + v.leaseAfresh() + if on, _ := v.leaseHanded(); !on { + return + } + bus, err := v.busNode() + if err != nil { + return + } + v.minting.mu.Lock() + revision := v.minting.loadedRevision + v.minting.mu.Unlock() + if revision == 0 { + return + } + v.nr.mu.Lock() + defer v.nr.mu.Unlock() + next := slices.IndexFunc(v.nr.queue, func(q queuedRotation) bool { return q.Node != bus }) + if next < 0 { + next = slices.IndexFunc(v.nr.queue, func(q queuedRotation) bool { return q.Node == bus }) + } + if next < 0 { + return + } + q := v.nr.queue[next] + v.nr.queue = slices.Delete(v.nr.queue, next, next+1) + if _, err := v.beginNodeRotation(q.By, q.Node, q.Why, q.Act, revision); err != nil { + v.log("the queued rotation of %s did not begin: %v", q.Node, err) + } +} + +// RotationState is what vault_status says of rotations: the keys of node-logins that rotate, and the queue. +type RotationState struct { + Rotating []NodeLogin `json:"rotating"` + Queued []queuedRotation `json:"queued"` +} + +// RotationState answers how the rotations stand. +func (v *Vault) RotationState() RotationState { + running, _ := v.rotatingNodes() + v.nr.mu.Lock() + defer v.nr.mu.Unlock() + return RotationState{Rotating: append([]NodeLogin{}, running...), Queued: append([]queuedRotation{}, v.nr.queue...)} +} + +// LoginsPass deletes the key of every node that left the placement (ADR 0583 §1 insight). A placement that cannot be +// read, or holds no node, deletes nothing. +func (v *Vault) LoginsPass() { + if v.Logins == nil || v.Placement.Keys == nil { + return + } + placed, err := v.placements() + if err != nil || len(placed) == 0 { + return + } + keys, err := v.Logins.Keys() + if err != nil { + return + } + for _, k := range keys { + if _, in := placed[k]; !in { + v.nodeLeft(k) + } + } +} + +// NodeLeft is heard when a node's key of the placement is deleted: its key of node-logins goes with it. +func (v *Vault) NodeLeft(node string) { + if v.Logins == nil || v.Placement.Read == nil { + return + } + if _, _, found, err := v.nodePlaced(node); err != nil || found { + return + } + v.nodeLeft(node) +} + +func (v *Vault) nodeLeft(node string) { + if err := v.Logins.Delete(node); err != nil { + v.log("%s left the placement, and its key of %s could not be deleted: %v", node, NodeLoginsState, err) + return + } + v.log("%s left the placement: its key of %s is deleted", node, NodeLoginsState) +} diff --git a/modules/mesh-vault/cmd/mesh-vault/retire.go b/modules/mesh-vault/cmd/mesh-vault/retire.go index e0ea2960..5862adf7 100644 --- a/modules/mesh-vault/cmd/mesh-vault/retire.go +++ b/modules/mesh-vault/cmd/mesh-vault/retire.go @@ -118,7 +118,7 @@ func (v *Vault) goneAccounts() ([]string, int, error) { if err != nil { return nil, 0, err } - if a == nil || a.Adopted || a.Of == "" { + if a == nil || a.Adopted || a.Of == "" || a.Kept != "" { continue } held++ @@ -207,6 +207,9 @@ func (v *Vault) retireOne(user, why, by string) error { if a == nil || a.Adopted || a.Of == "" { return fmt.Errorf("the vault holds no account of %s it made or was handed: an adopted hash is never retired here", user) } + if a.Kept != "" { + return fmt.Errorf("%s is kept for an enrolment or a node's login, which ends with its own verb (enrolment-token, node-account, node-account-confirm, abandon): it is never retired as unwanted", user) + } rec, held, err := v.Store.Read(a.Of) if err != nil { return err diff --git a/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go b/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go new file mode 100644 index 00000000..3a80e078 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/rotation_integration_test.go @@ -0,0 +1,242 @@ +//go:build integration + +package main + +// The vault whole against a real bus, rotating a node's login with a stand-in engine (novox/hq ADR 0583 §1 to §7, issue +// 774): the built binary played over its stdin and stdout as the node's tool runtime plays it, as in the mint test. The +// test is the engine and the bus node: it states its sealing key, takes the new login through `own`, and "reloads the +// list" by handing the bus-accounts hashes in the bus module's have, as the bus node's node-engine does. At every step at +// least one of the engine's logins authenticates against the list loaded last, and with the list never reloaded the vault +// never says node-account-ready. + +import ( + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "golang.org/x/crypto/bcrypt" + "golang.org/x/crypto/nacl/box" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/seal" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +// 774/S50/AC1. +func TestTheVaultWholeRotatesANodesLoginWithAStandInEngine(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, "mesh-vault") + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("the vault does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, b := range []string{"mesh-vault_bus-accounts", "mesh-vault_node-logins", "mesh-grants_accounts", "mesh-controller_lease", + "mesh-provisioner_plans", "node-engine_have", "mesh-delivery-assigner_placement"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 2}); err != nil { + t.Fatal(err) + } + } + opPub, _, err := box.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + root := filepath.Join(dir, "root") + _ = os.MkdirAll(root, 0o700) + export, _ := json.Marshal(map[string]string{"operator-key": base64.StdEncoding.EncodeToString(opPub[:])}) + if err := os.WriteFile(filepath.Join(root, "export.json"), export, 0o600); err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_VAULT_STORE="+filepath.Join(dir, "store"), "MESH_VAULT_STATE="+filepath.Join(dir, "state"), + "MESH_VAULT_ROOT="+root, "MESH_VAULT_LEDGER="+filepath.Join(dir, "ledger"), "MESH_RECEIVES="+filepath.Join(dir, "mesh.json"), + "MESH_NODE=novox", "MESH_MODULE=mesh-vault") + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{t: t, ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + // mesh-hand-acts answers the one hand act the test recorded. + p.ask = func(key string, _ json.RawMessage) (any, error) { + if key != "mesh-hand-acts.hand-acts" { + return nil, errors.New("nats: no responders available for request") + } + return map[string]any{"acts": []HandActSeen{{ID: "act-1", By: "person.jochen", At: time.Now(), Verb: "rotate-node", Args: []string{"g14"}}}}, nil + } + go p.serve(stdout) + + // The bus node (novox, running nats) and the node that rotates (g14); the lease handed away; g14's login adopted from + // the loaded list, its password known to the engine. + for node, modules := range map[string][]string{"novox": {"nats"}, "g14": {"web"}} { + pl := delivery.Placement{V: delivery.Version, Node: node} + for _, m := range modules { + pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m}) + } + values, err := delivery.PlacementValues(pl) + if err != nil { + t.Fatal(err) + } + for k, raw := range values { + kv, _ := p.kv("mesh-delivery-assigner.placement") + if _, err := kv.Put(ctx, k, raw); err != nil { + t.Fatal(err) + } + } + } + p.put(t, "mesh-controller.lease", LeaseKey, map[string]any{"handed-to": "the operator's hand-over"}) + oldPassword := "the-old-password-of-g14" + oldHash, _ := bcrypt.GenerateFromPassword([]byte(oldPassword), bcrypt.MinCost) + p.put(t, "mesh-vault.bus-accounts", "node.g14", AccountHash{User: "node.g14", Hash: string(oldHash), Adopted: true}) + // The vault's public key, which the bus node's engine seals its handing to. + var vaultKey seal.KeyPair + eventually(t, "the vault's key", func() bool { + raw, err := os.ReadFile(filepath.Join(dir, "state", OwnKeyFile)) + return err == nil && json.Unmarshal(raw, &vaultKey) == nil && vaultKey.PublicKey != "" + }) + + // The bus node reloads the list from bus-accounts and hands the hashes of what it loaded in its have; the engine's logins + // are its passwords. An authentication is a bcrypt compare against the list loaded last. + passwords := map[string]string{"node.g14": oldPassword} + var loaded map[string]string + revision := uint64(0) + reload := func() { + kv, _ := p.kv("mesh-vault.bus-accounts") + keys, err := kv.Keys(ctx) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { + t.Fatal(err) + } + sort.Strings(keys) + var accounts []delivery.HandedAccount + loaded = map[string]string{} + for _, k := range keys { + a := p.hashOf(t, k) + if a == nil || !delivery.IsBcrypt(a.Hash) { + continue + } + accounts = append(accounts, delivery.HandedAccount{User: k, Hash: a.Hash}) + loaded[k] = a.Hash + } + revision++ + list, _ := json.Marshal(accounts) + handed, err := delivery.HandAccounts(accounts, vaultKey.PublicKey, Fingerprint(string(list)), time.Now()) + if err != nil { + t.Fatal(err) + } + raw, err := delivery.EncodeHave(delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK, + UserList: &delivery.UserListWritten{Source: "k", Revision: revision, Users: len(accounts), Handed: &handed}}) + if err != nil { + t.Fatal(err) + } + hk, _ := p.kv("node-engine.have") + if _, err := hk.Put(ctx, "novox.nats", raw); err != nil { + t.Fatal(err) + } + } + authenticates := func(step string) { + t.Helper() + for user, password := range passwords { + if h, ok := loaded[user]; ok && bcrypt.CompareHashAndPassword([]byte(h), []byte(password)) == nil { + return + } + } + t.Fatalf("%s: none of the engine's logins %v authenticates against the loaded list %v", step, passwords, loaded) + } + said := func(event string) bool { + p.mu.Lock() + defer p.mu.Unlock() + return strings.Contains(strings.Join(p.said, "\n"), event+" ") + } + reload() + authenticates("before the rotation") + + enginePub, enginePriv := aSealingKey(t) + if _, err := p.call(Seat+".node-key", "node.g14", map[string]any{"key": enginePub}); err != nil { + t.Fatal(err) + } + // The rotation begins once the vault has heard the loaded list's revision. + var begun RotateNodeAnswer + eventually(t, "the rotation begins", func() bool { + raw, err := p.call(Seat+".rotate-node", "person.jochen", map[string]any{"node": "g14", "why": "the weekly rotation", "act": "act-1"}) + return err == nil && json.Unmarshal(raw, &begun) == nil && begun.Rotating == "node.g14--b" + }) + authenticates("rotation begun") + // The list is never reloaded: node-account-ready is never said. + time.Sleep(time.Second) + if said(NodeAccountReadyEvent) { + t.Fatal("node-account-ready was said while the loaded list lacks the new login") + } + // The engine takes the new login through own, sealed to its key. + raw, err := p.call(Seat+".own", "node.g14", map[string]any{}) + if err != nil { + t.Fatal(err) + } + var own OwnAnswer + if err := json.Unmarshal(raw, &own); err != nil || len(own.Own) != 1 || own.Own[0].Module != MeshHostModule || own.Own[0].Name != "node.g14--b" { + t.Fatalf("own answers %+v, %v", own.Own, err) + } + value, err := seal.Open(own.Own[0].Sealed, enginePriv) + if err != nil { + t.Fatal(err) + } + var account struct{ User, Password string } + if err := json.Unmarshal([]byte(value), &account); err != nil || account.User != "node.g14--b" { + t.Fatalf("the new login is %s, %v", value, err) + } + passwords["node.g14--b"] = account.Password + // The bus node reloads the list: the new login is in it, both authenticate, node-account-ready is said. + reload() + eventually(t, "node-account-ready", func() bool { return said(NodeAccountReadyEvent) }) + authenticates("the list holds both logins") + if bcrypt.CompareHashAndPassword([]byte(loaded["node.g14--b"]), []byte(account.Password)) != nil { + t.Fatal("the loaded hash of the new login is not of the value the engine took") + } + // The engine, dialled as the new login, confirms; the vault removes the old hash, and the next reload drops it. + if _, err := p.call(Seat+".node-account-confirm", "node.g14--b", map[string]any{}); err != nil { + t.Fatal(err) + } + if h := p.hashOf(t, "node.g14"); h != nil { + t.Fatalf("the old login's hash remains after the confirm: %+v", h) + } + authenticates("the confirm, before the list reloads") + reload() + authenticates("the list without the old login") + eventually(t, "rotation-done", func() bool { return said(RotationDoneEvent) }) + kv, _ := p.kv("mesh-vault.node-logins") + e, err := kv.Get(ctx, "g14") + if err != nil { + t.Fatal(err) + } + var nl NodeLogin + _ = json.Unmarshal(e.Value(), &nl) + if nl.Login != "node.g14--b" || nl.Rotating != "" || nl.Stage != "" { + t.Errorf("node-logins g14 is %+v", nl) + } + if h := p.hashOf(t, "node.g14--b"); h == nil || h.Rotating != nil { + t.Errorf("the new login's key is %+v", h) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/signing.go b/modules/mesh-vault/cmd/mesh-vault/signing.go index a4c66d9a..705e91b5 100644 --- a/modules/mesh-vault/cmd/mesh-vault/signing.go +++ b/modules/mesh-vault/cmd/mesh-vault/signing.go @@ -240,6 +240,15 @@ type SignArgs struct { Kind string Node string Reply string + // Keys are, for a joining node's certificate asked by mesh-enrolment, the public keys the machine presented at its + // redeem (delivery.Redeem's): the serving key is certified, the four are checked against the digest node-account + // recorded. + Keys PresentedKeys +} + +// PresentedKeys are the public keys a joining machine presented at its redeem, as delivery.Redeem names them. +type PresentedKeys struct { + Public, Overlay, Sealing, Serving string } // Sign answers caller's `sign`. The caller is the one the runtime proved; who may ask which kind is the vault's rule, @@ -251,16 +260,98 @@ func (v *Vault) Sign(caller string, a SignArgs) (delivery.SignAnswer, error) { } switch a.Kind { case delivery.KindNodeCertificate: + if v.fromEnrolment(caller, "sign") == nil { + return v.signRedeemingCertificate(caller, a.Node, a.Keys) + } return v.signNodeCertificate(caller, a.Node) case delivery.KindEnrolment: - return delivery.SignAnswer{}, fmt.Errorf("%s may not ask for an enrolment's signature: no caller is named for it until "+ - "adoption's record names the seat that adopts (novox/hq ADR 0561 §5)", caller) + if err := v.fromEnrolment(caller, "sign an enrolment"); err != nil { + return delivery.SignAnswer{}, fmt.Errorf("%s may not ask for an enrolment's signature: only the seat %s, which "+ + "decides who joins, is named for it (novox/hq ADR 0561 §5, ADR 0587 §6)", caller, EnrolmentModule) + } + return v.signEnrolment(caller, a.Node, a.Reply) case delivery.KindAuthority: return delivery.SignAnswer{}, fmt.Errorf("the authority is the vault's own, signed for itself; %s may not ask it", caller) } return delivery.SignAnswer{}, fmt.Errorf("%q is not a kind the vault signs (%s)", a.Kind, strings.Join(delivery.SignKinds, ", ")) } +// redeeming says node has an enrolment token held: a machine joining, which the seat mesh-enrolment redeems. +func (v *Vault) redeeming(node string) error { + if err := enrolNode(node); err != nil { + return err + } + if _, held, err := v.Store.Read(enrolSecret(node)); err != nil { + return err + } else if !held { + return fmt.Errorf("%s holds no enrolment token: it is not a machine joining, so nothing is signed for it", node) + } + return nil +} + +// signRedeemingCertificate signs the certificate of a machine that is joining, for the seat mesh-enrolment at the redeem +// (ADR 0587 §6). The machine has written no record yet, so the serving key is the one it presented at the redeem, which +// counts only when the four keys presented have the digest node-account recorded for its account; the name is its +// placement's, which mesh-enrolment writes before it asks. +func (v *Vault) signRedeemingCertificate(caller, node string, keys PresentedKeys) (delivery.SignAnswer, error) { + if err := v.redeeming(node); err != nil { + return delivery.SignAnswer{}, err + } + if v.Accounts == nil { + return delivery.SignAnswer{}, errors.New("the vault has no state of bus accounts, so it cannot read the keys a joining node presented") + } + acct, err := v.Accounts.Get(NodeEnginePrefix + node) + if err != nil { + return delivery.SignAnswer{}, fmt.Errorf("%s %s could not be read: %w", BusAccountsState, NodeEnginePrefix+node, err) + } + if acct == nil || acct.Kept != KeptNode || acct.KeysDigest == "" { + return delivery.SignAnswer{}, fmt.Errorf("%s has no account made at its redeem (node-account), so no keys are recorded to certify", node) + } + if keys.Serving == "" || keys.Public == "" || keys.Sealing == "" { + return delivery.SignAnswer{}, errors.New("a joining node's certificate is asked with the keys it presented at the redeem " + + "(`public_key`, `overlay_key`, `sealing_key`, `serving_key`), which are checked against node-account's digest") + } + if got := delivery.KeysDigest(keys.Public, keys.Overlay, keys.Sealing, keys.Serving); got != acct.KeysDigest { + return delivery.SignAnswer{}, fmt.Errorf("the keys presented for %s are not the ones its account was made for: nothing is signed", node) + } + pub, err := base64.StdEncoding.DecodeString(keys.Serving) + if err != nil || len(pub) != ed25519.PublicKeySize { + return delivery.SignAnswer{}, fmt.Errorf("%s's serving key is not an ed25519 public key in base64", node) + } + name, err := v.internalName(node) + if err != nil { + return delivery.SignAnswer{}, err + } + return v.signCertificateWith(caller, node, name, ed25519.PublicKey(pub)) +} + +// signEnrolment signs the bytes of an enrolment reply under the enrolment context, for a node that is joining; the +// signature is the vault's key's, and the answer carries it with the signer's fingerprint. +func (v *Vault) signEnrolment(caller, node, reply string) (delivery.SignAnswer, error) { + if err := v.redeeming(node); err != nil { + return delivery.SignAnswer{}, err + } + msg, err := base64.StdEncoding.DecodeString(reply) + if err != nil || len(msg) == 0 { + return delivery.SignAnswer{}, errors.New("an enrolment's signature is asked for the reply's bytes in base64 (`reply`)") + } + context, err := delivery.SignContext(delivery.KindEnrolment) + if err != nil { + return delivery.SignAnswer{}, err + } + k, err := v.keys() + if err != nil { + return delivery.SignAnswer{}, err + } + answer := delivery.SignAnswer{Kind: delivery.KindEnrolment, Node: node, + Signature: base64.StdEncoding.EncodeToString(delivery.SignMessage(k.key, context, msg)), + Signer: delivery.SignerFingerprint(k.key.Public().(ed25519.PublicKey)), Digest: delivery.DigestOf(msg)} + if err := v.saySigned(caller, SigningKeySecret, answer); err != nil { + return delivery.SignAnswer{}, err + } + return answer, nil +} + // signNodeCertificate signs the certificate of the calling machine's internal name for the serving key its record states. func (v *Vault) signNodeCertificate(caller, asked string) (delivery.SignAnswer, error) { node, ok := nodeEngineOf(caller) @@ -271,6 +362,11 @@ func (v *Vault) signNodeCertificate(caller, asked string) (delivery.SignAnswer, if asked != "" && asked != node { return delivery.SignAnswer{}, fmt.Errorf("%s asks for the certificate of %s, and is %s's node-engine: a machine asks for its own alone", caller, asked, node) } + return v.signCertificateFor(caller, node) +} + +// signCertificateFor signs node's certificate from its placement's name and its record's serving key. +func (v *Vault) signCertificateFor(caller, node string) (delivery.SignAnswer, error) { name, err := v.internalName(node) if err != nil { return delivery.SignAnswer{}, err @@ -279,6 +375,11 @@ func (v *Vault) signNodeCertificate(caller, asked string) (delivery.SignAnswer, if err != nil { return delivery.SignAnswer{}, err } + return v.signCertificateWith(caller, node, name, serving) +} + +// signCertificateWith signs the certificate of name for serving, for node. +func (v *Vault) signCertificateWith(caller, node, name string, serving ed25519.PublicKey) (delivery.SignAnswer, error) { k, err := v.keys() if err != nil { return delivery.SignAnswer{}, err diff --git a/modules/mesh-vault/cmd/mesh-vault/store.go b/modules/mesh-vault/cmd/mesh-vault/store.go index 8ede49d2..b7ec8b28 100644 --- a/modules/mesh-vault/cmd/mesh-vault/store.go +++ b/modules/mesh-vault/cmd/mesh-vault/store.go @@ -157,6 +157,19 @@ func (s *Store) Write(r Record) error { return writeWhole(s.pathOf(r.Name), append(raw, '\n')) } +// Delete takes one secret out of the store: for the enrolment token and a joining node's account, which end with +// their enrolment (novox/hq ADR 0587 §5), never for a value the mesh keeps. Nothing held is no error. +func (s *Store) Delete(name string) error { + if err := validName(name); err != nil { + return err + } + path := s.pathOf(name) + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return syncDir(filepath.Dir(path)) +} + // Empty says whether the store holds no secret at all. func (s *Store) Empty() (bool, error) { all, err := s.List() diff --git a/modules/mesh-vault/cmd/mesh-vault/tools.go b/modules/mesh-vault/cmd/mesh-vault/tools.go index ee282f7d..ce181856 100644 --- a/modules/mesh-vault/cmd/mesh-vault/tools.go +++ b/modules/mesh-vault/cmd/mesh-vault/tools.go @@ -75,7 +75,7 @@ func str(description string) map[string]any { } func definitions(env *Env) []toolDef { - return []toolDef{ + return append(enrolmentTools(env), []toolDef{ {Name: Seat + ".get", Description: "One secret, sealed to the public key the asker sends (the sealed box of ADR 0206), with its " + "generation; answered only to an asker the vault allows, as the runtime proved it (ADR 0349 §11). " + @@ -395,8 +395,13 @@ func definitions(env *Env) []toolDef { "granted, how many times it has been rotated and when, and the fingerprint of the current value. " + "Fingerprints only — the value is never returned.", run: func(string, map[string]any) (any, error) { + enrolments := []EnrolmentHeld{} + if env.Vault != nil { + enrolments, _ = env.Vault.EnrolmentsHeld() + } if env.Ledger == nil { - return map[string]any{"holders": []any{}, "count": 0, "error": "the vault has no ledger (MESH_VAULT_LEDGER is not set)"}, nil + return map[string]any{"holders": []any{}, "count": 0, "enrolments": enrolments, + "error": "the vault has no ledger (MESH_VAULT_LEDGER is not set)"}, nil } var asked []Contribution if env.Receives != "" { @@ -415,7 +420,7 @@ func definitions(env *Env) []toolDef { } holders = append(holders, row) } - return map[string]any{"holders": holders, "count": len(holders)}, nil + return map[string]any{"holders": holders, "count": len(holders), "enrolments": enrolments}, nil }}, {Name: "secret_verify", Description: "Check one holder's secret without seeing it: the fingerprint the vault recorded against the " + @@ -581,14 +586,20 @@ func definitions(env *Env) []toolDef { "adoption's record (ADR 0561 §5). Answers the certificate, the authority, the signer's fingerprint and the " + "digest, and says signed. Never a key.", Input: map[string]any{"kind": str("node-certificate or enrolment"), - "node": str("the machine, for a node certificate: the caller's own (left out, the caller's)"), - "reply": str("the enrolment reply's bytes in base64, for an enrolment")}, + "node": str("the machine, for a node certificate: the caller's own (left out, the caller's)"), + "reply": str("the enrolment reply's bytes in base64, for an enrolment"), + "public_key": str("a joining node's identity key, as presented at its redeem (mesh-enrolment alone)"), + "overlay_key": str("a joining node's overlay key, as presented at its redeem"), + "sealing_key": str("a joining node's sealing key, as presented at its redeem"), + "serving_key": str("a joining node's serving key to certify, as presented at its redeem")}, run: func(caller string, a map[string]any) (any, error) { v, err := env.ready() if err != nil { return nil, err } - return v.Sign(caller, SignArgs{Kind: arg(a, "kind"), Node: arg(a, "node"), Reply: arg(a, "reply")}) + return v.Sign(caller, SignArgs{Kind: arg(a, "kind"), Node: arg(a, "node"), Reply: arg(a, "reply"), + Keys: PresentedKeys{Public: arg(a, "public_key"), Overlay: arg(a, "overlay_key"), Sealing: arg(a, "sealing_key"), + Serving: arg(a, "serving_key")}}) }}, {Name: "vault_signers", Description: "The vault's signing key and authority (novox/hq ADR 0550 §2): whether each is held with its operator " + @@ -647,9 +658,9 @@ func definitions(env *Env) []toolDef { key, _ := env.Vault.Store.OperatorKey() return map[string]any{"ready": true, "operator_key": key, "copies_on_an_earlier_key": left, "operator_key_refused": env.keyRefused(), "handed_rolled": env.Vault.Store.Rolled(), - "minting": env.Vault.MintState(), "retiring": env.Vault.RetireState(), "signing": env.Vault.SigningState(), "ok": left == 0 && env.keyRefused() == "" && key != ""}, nil + "minting": env.Vault.MintState(), "retiring": env.Vault.RetireState(), "signing": env.Vault.SigningState(), "ok": left == 0 && env.keyRefused() == "" && key != "", "rotation": env.Vault.RotationState()}, nil }}, - } + }...) } // actInput is the arguments of an act on retirement, as the SDK's harness takes them. diff --git a/modules/mesh-vault/cmd/mesh-vault/vault.go b/modules/mesh-vault/cmd/mesh-vault/vault.go index b9f7585e..010e2808 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault.go @@ -76,6 +76,17 @@ type Vault struct { // Signers is the vault's state `signers`, where it publishes its signing halves (signing.go, ADR 0550 §2): nil writes // none. Signers SignersStore + // EnrolCallers are the callers of the enrolment verbs `enrolment-token` and `node-account` and of `sign` for an + // enrolment: the seat mesh-enrolment (enrolment.go). Nil: that module alone. + EnrolCallers stdio.Callers + // TokenBase says where the bus is and its certificate's fingerprint, for the token shown (enrolment.go): nil shows none. + TokenBase func() (TokenBase, error) + // Logins is the vault's state node-logins (noderotation.go): nil writes none. + Logins NodeLoginStore + // ReadAct reads a hand act back from mesh-hand-acts by its id, which a rotation carries (noderotation.go): nil answers none. + ReadAct ActReader + // nr is what the vault keeps in memory of its node rotations (noderotation.go). + nr nodeRotation // signingWhy is why the last signing pass did not finish, under mu (signing.go). signingWhy string // callsMu guards pending: the keys of the calls open now (intake.go), never written anywhere. @@ -341,14 +352,30 @@ func (v *Vault) Get(caller, name, publicKey string) (Answer, error) { if signingSecret(name) { return Answer{}, fmt.Errorf("%s is the vault's own signing secret and is handed to nobody: ask the verb sign", name) } - // A value issued for one holder is that holder's alone, whatever else would allow it (issue 749). - if why := notTheHolder(caller, name); why != "" { + // An enrolment token is shown to the operator sealed to the operator key, and handed to nobody (ADR 0587 §8). + if enrolSecretName(name) { + return Answer{}, fmt.Errorf("%s is an enrolment token and is handed to nobody: the operator opens it sealed to the operator key", name) + } + // A node's login is its node-engine's alone, sealed to the key that machine stated (ADR 0583 §3 insight). + isLogin, why := v.loginRefusal(caller, name) + if isLogin { + if why == "" { + why = v.loginKeyRefusal(caller, publicKey) + } + if why != "" { + v.log("refused %s to %s: %s", name, caller, why) + _ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "refused", Why: why}) + return Answer{}, fmt.Errorf("%s may not ask for %s: %s", caller, name, why) + } + } else if why := notTheHolder(caller, name); why != "" { v.log("refused %s to %s: %s", name, caller, why) _ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "refused", Why: why}) return Answer{}, fmt.Errorf("%s may not ask for %s: %s", caller, name, why) } // The core's own credentials, to the controller alone, sealed only to their machine's key (ADR 0349 §16). - if caller == ControllerCaller { + if isLogin { + // Judged above: the node's own engine, sealed to its own key. + } else if caller == ControllerCaller { if why := v.coreRefusal(name, publicKey); why != "" { v.log("refused %s to %s: %s", name, caller, why) _ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "refused", Why: why}) diff --git a/modules/mesh-vault/cmd/mesh-vault/vault_test.go b/modules/mesh-vault/cmd/mesh-vault/vault_test.go index d1ab87c9..5bda6d41 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault_test.go @@ -146,10 +146,10 @@ func TestNoBucketHoldsASecret(t *testing.T) { if err := json.Unmarshal(raw, &manifest); err != nil { t.Fatal(err) } - if s, _ := json.Marshal(manifest["state"]); string(s) != `[{"history":2,"name":"bus-accounts"},"signers","bound"]` { + if s, _ := json.Marshal(manifest["state"]); string(s) != `[{"history":2,"name":"bus-accounts"},"signers","bound","node-logins"]` { t.Errorf("the manifest declares buckets %s: a secret is in no bucket, sealed or not (ADR 0201 §13), and the "+ "buckets are the accounts' hashes (ADR 0547 clause 9), the signing keys' public halves (ADR 0550 §2) and the "+ - "credentials' outcomes, names and generations alone (ADR 0570 clauses 3, 6)", s) + "credentials' outcomes, names and generations alone (ADR 0570 clauses 3, 6) and each node's login in use (ADR 0583)", s) } if r, _ := json.Marshal(manifest["reads"]); string(r) != `["node-engine.have","mesh-grants.accounts","mesh-controller.lease","mesh-delivery-assigner.placement","mesh-provisioner.plans","mesh-provisioner.provisions"]` { t.Errorf("the manifest reads %s: the vault reads the seat mesh-provisioner's plans and provisions, the haves, the "+ @@ -173,6 +173,13 @@ func TestNoBucketHoldsASecret(t *testing.T) { code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Keys(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Delete(", "", 1)) } + // Each node's login in use and its rotation's stage, in noderotation.go alone: one read, one write, its keys and a + // delete of its own bucket (ADR 0583 §1 insight); never a secret. + if f == "noderotation.go" { + for _, call := range []string{"Get(", "Put(", "Delete(", "Keys("} { + code = []byte(strings.Replace(string(code), "stdio.State(NodeLoginsState)."+call, "", -1)) + } + } // The signing keys' public halves, in signing.go alone: one read and one write of its own bucket (ADR 0550 §2). if f == "signing.go" { code = []byte(strings.Replace(string(code), "stdio.State(delivery.SignersState).Get(", "", 1)) @@ -194,6 +201,8 @@ func TestNoBucketHoldsASecret(t *testing.T) { // And the seat's placement, where modules stand (placed.go): its keys and one key, read. code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Keys(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Get(", "", 1)) + // And one watch of its nodes' keys, for a node that leaves: its node-logins key goes with it (noderotation.go). + code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Watch(", "", 1)) // And its outcome entries (novox/hq ADR 0570 clauses 3, 6): a value's name and generation, never a value; and // one read of the seat's provisions, which hold none. for _, call := range []string{"Get(", "Put(", "Delete(", "Keys("} { diff --git a/modules/mesh-vault/go.mod b/modules/mesh-vault/go.mod index b97a5aff..c3a48a37 100644 --- a/modules/mesh-vault/go.mod +++ b/modules/mesh-vault/go.mod @@ -3,7 +3,7 @@ module mesh-vault go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/modules/mesh-vault/go.sum b/modules/mesh-vault/go.sum index 5fc11b2f..4ab391eb 100644 --- a/modules/mesh-vault/go.sum +++ b/modules/mesh-vault/go.sum @@ -1,5 +1,7 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index efea0cb1..7e3d2cc3 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -21,7 +21,13 @@ "account-hashed", "own-replaced", "account-issued", - "signed" + "signed", + "rotation-begun", + "node-account-ready", + "rotation-done" + ], + "invokes": [ + "seat:mesh-hand-acts.hand-acts" ], "reads": [ "node-engine.have", @@ -37,7 +43,8 @@ "history": 2 }, "signers", - "bound" + "bound", + "node-logins" ], "receives": { "secret": "${dir:grants}/mesh.json" @@ -156,7 +163,8 @@ "MESH_VAULT_ROOT": "${dir:root}", "MESH_VAULT_STORE": "${dir:store}", "MESH_VAULT_STATE": "${dir:state}", - "MESH_VAULT_SECRET_ASK_CALLERS": "node-tools" + "MESH_VAULT_SECRET_ASK_CALLERS": "node-tools", + "MESH_VAULT_ENROLMENT_CALLERS": "mesh-enrolment" } } ] @@ -182,7 +190,8 @@ "own_take_switch", "vault_signers", "vault_signed", - "signers_reassert" + "signers_reassert", + "node_logins" ], "claims": [ { @@ -199,7 +208,12 @@ "move-asks", "own", "issue-account", - "sign" + "sign", + "enrolment-token", + "node-account", + "rotate-node", + "node-account-confirm", + "abandon" ] } ]