Merge pull request 'The vault mints a joining node's token, account and certificate, and rotates a node's login over two; nox enrol open (issue 774)' (#591) from feat/l3-enrol into main

This commit was merged in pull request #591.
This commit is contained in:
2026-11-02 07:56:27 +00:00
23 changed files with 2683 additions and 25 deletions
+97
View File
@@ -0,0 +1,97 @@
package main
// `nox enrol open`: the operator opens an enrolment token at their own terminal (novox/hq ADR 0587 §6, §8).
//
// The seat mesh-enrolment's `tokens` answers the token sealed to the operator key, which only the operator's private key
// opens; the mesh never holds it. This subcommand is local: it asks nothing of the node-engine's socket and nothing of
// the controller or the bus. It reads the sealed token (the base64 of the sealed box) on standard input and the private
// key from a file (`--key`, the raw X25519 key in base64 that `operator key make` wrote), opens the box, and prints the
// token text a joining machine's `enrol` takes. The key is read from the file named and nowhere else, and never printed.
import (
"crypto/ecdh"
"encoding/base64"
"errors"
"fmt"
"io"
"os"
"strings"
"golang.org/x/crypto/nacl/box"
)
const enrolUsage = `nox enrol open --key <operator private key file>
Opens an enrolment token sealed to the operator key (the answer of the seat mesh-enrolment's tokens) and prints the token
to type on the new machine. The sealed token is read on standard input; this runs here and asks nothing of the mesh.
`
// maxSealedToken bounds what is read of standard input: a token is a few hundred bytes.
const maxSealedToken = 16 << 10
// enrolOpen runs `enrol open`: args are what follows `enrol open`.
func enrolOpen(args []string, stdin io.Reader, stdinIsTerminal bool, stdout, stderr io.Writer) int {
keyFile := ""
for i := 0; i < len(args); i++ {
switch {
case args[i] == "--key" && i+1 < len(args):
keyFile = args[i+1]
i++
case strings.HasPrefix(args[i], "--key="):
keyFile = strings.TrimPrefix(args[i], "--key=")
default:
fmt.Fprint(stderr, enrolUsage)
return 2
}
}
if keyFile == "" {
fmt.Fprint(stderr, enrolUsage)
return 2
}
if stdinIsTerminal {
fmt.Fprintln(stderr, "mesh-cli: give the sealed token on standard input (a pipe or a file); nothing was opened")
return 2
}
raw, err := os.ReadFile(keyFile)
if err != nil {
fmt.Fprintf(stderr, "mesh-cli: the operator key could not be read: %v\n", err)
return 1
}
sealed, err := io.ReadAll(io.LimitReader(stdin, maxSealedToken+1))
if err != nil || len(sealed) > maxSealedToken {
fmt.Fprintln(stderr, "mesh-cli: the sealed token could not be read from standard input, or is longer than a token is")
return 1
}
text, err := openSealed(strings.TrimSpace(string(raw)), strings.TrimSpace(string(sealed)))
if err != nil {
fmt.Fprintf(stderr, "mesh-cli: %v\n", err)
return 1
}
fmt.Fprintln(stdout, text)
return 0
}
// openSealed opens a value sealed to the operator key (the sealed box of mesh-vault's operator copies, box.SealAnonymous
// in base64) with the private key, the raw X25519 key in base64.
func openSealed(privateKey, sealed string) (string, error) {
private, err := base64.StdEncoding.DecodeString(privateKey)
if err != nil || len(private) != 32 {
return "", errors.New("that is not an operator private key (the raw X25519 key in base64)")
}
key, err := ecdh.X25519().NewPrivateKey(private)
if err != nil {
return "", fmt.Errorf("that is not a usable operator private key: %w", err)
}
blob, err := base64.StdEncoding.DecodeString(sealed)
if err != nil {
return "", fmt.Errorf("this is not a sealed token: %w", err)
}
var pub, priv [32]byte
copy(pub[:], key.PublicKey().Bytes())
copy(priv[:], private)
out, ok := box.OpenAnonymous(nil, blob, &pub, &priv)
if !ok {
return "", errors.New("this token is not sealed to that key; nothing is printed")
}
return strings.TrimSpace(string(out)), nil
}
@@ -0,0 +1,80 @@
package main
import (
"bytes"
"context"
"crypto/rand"
"encoding/base64"
"encoding/json"
"io"
"os"
"path/filepath"
"strings"
"testing"
"golang.org/x/crypto/nacl/box"
)
// 774/S51/AC1, AC2 (novox/hq ADR 0587 §6, §8): a token sealed to a test operator key (as mesh-vault seals it: the text
// of the token, anonymous box, base64) is opened with the private half and printed as a token the engine's parser reads
// (base64url of version 1 JSON with node, broker, fingerprint, signer and secret), and nothing is asked of the engine's
// socket, which here is a path nobody listens on; the wrong key prints nothing and says the token is not sealed to it.
func TestEnrolOpenOpensASealedTokenLocallyAndOnlyWithItsKey(t *testing.T) {
t.Setenv("MESH_VERB", "")
token := base64.RawURLEncoding.EncodeToString([]byte(`{"v":1,"node":"new1","broker":"tls://10.77.0.1:4222",` +
`"fingerprint":"sha256:bus","signer":"` + base64.StdEncoding.EncodeToString(make([]byte, 32)) + `","secret":"one-time"}`))
pub, priv, _ := box.GenerateKey(rand.Reader)
_, otherPriv, _ := box.GenerateKey(rand.Reader)
sealed, err := box.SealAnonymous(nil, []byte(token), pub, rand.Reader)
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
keyFile := func(name string, k *[32]byte) string {
p := filepath.Join(dir, name)
if err := os.WriteFile(p, []byte(base64.StdEncoding.EncodeToString(k[:])+"\n"), 0o600); err != nil {
t.Fatal(err)
}
return p
}
before, beforeIn := socket, standardInput
socket = filepath.Join(dir, "nobody-listens.sock")
t.Cleanup(func() { socket, standardInput = before, beforeIn })
for _, c := range []struct {
name string
args []string
stdin string
code int
out string
err string
}{
{"the right key", []string{"enrol", "open", "--key", keyFile("op", priv)}, base64.StdEncoding.EncodeToString(sealed) + "\n", 0, token + "\n", ""},
{"the key as --key=", []string{"enrol", "open", "--key=" + keyFile("op2", priv)}, base64.StdEncoding.EncodeToString(sealed), 0, token + "\n", ""},
{"the wrong key", []string{"enrol", "open", "--key", keyFile("other", otherPriv)}, base64.StdEncoding.EncodeToString(sealed), 1, "", "not sealed to that key"},
{"no key", []string{"enrol", "open"}, base64.StdEncoding.EncodeToString(sealed), 2, "", "nox enrol open --key"},
{"not a sealed token", []string{"enrol", "open", "--key", keyFile("op3", priv)}, "@@@", 1, "", "not a sealed token"},
} {
standardInput = func() (io.Reader, bool) { return strings.NewReader(c.stdin), false }
var out, errOut bytes.Buffer
code := run(context.Background(), c.args, &out, &errOut)
if code != c.code || out.String() != c.out || !strings.Contains(errOut.String(), c.err) {
t.Errorf("%s: exit %d, stdout %q, stderr %q", c.name, code, out.String(), errOut.String())
}
}
// What it prints is a token the engine's parser reads.
var parsed struct {
V int `json:"v"`
Node string `json:"node"`
Secret string `json:"secret"`
}
raw, _ := base64.RawURLEncoding.DecodeString(token)
if json.Unmarshal(raw, &parsed) != nil || parsed.V != 1 || parsed.Node != "new1" || parsed.Secret != "one-time" {
t.Errorf("the token is %+v", parsed)
}
// Standard input a terminal is never read.
standardInput = func() (io.Reader, bool) { return strings.NewReader("x"), true }
var out, errOut bytes.Buffer
if code := run(context.Background(), []string{"enrol", "open", "--key", keyFile("op4", priv)}, &out, &errOut); code != 2 || out.Len() != 0 {
t.Errorf("a terminal on standard input: exit %d, stdout %q", code, out.String())
}
}
+8
View File
@@ -56,6 +56,8 @@ const usage = `mesh-cli — the controller's command lines, from this node (novo
mesh-cli <command line> e.g. mesh-cli status, mesh-cli settings show zsh --node <node> mesh-cli <command line> e.g. mesh-cli status, mesh-cli settings show zsh --node <node>
nox <command line> the same, in interactive zsh nox <command line> the same, in interactive zsh
nox enrol open --key <file> opens a sealed enrolment token here, asking nothing of the mesh (ADR 0587)
Run it as yourself, never under sudo: the node-engine tells the controller which account asks. Run it as yourself, never under sudo: the node-engine tells the controller which account asks.
What a pipe or a file gives its standard input (at most 64 KiB) goes with the line to the controller's What a pipe or a file gives its standard input (at most 64 KiB) goes with the line to the controller's
terminal alone, e.g. a secret: mesh-cli secret accept <node> <module> <name> --from - < file terminal alone, e.g. a secret: mesh-cli secret accept <node> <module> <name> --from - < file
@@ -82,6 +84,12 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
"Nothing was asked\n", verb) "Nothing was asked\n", verb)
return 1 return 1
} }
// `enrol open` is local: it opens a sealed enrolment token with the operator's private key and asks nothing of the
// node-engine or the mesh (novox/hq ADR 0587 §8).
if len(args) >= 2 && args[0] == "enrol" && args[1] == "open" {
in, terminal := standardInput()
return enrolOpen(args[2:], in, terminal, stdout, stderr)
}
in, err := carried() in, err := carried()
if err != nil { if err != nil {
fmt.Fprintf(stderr, "mesh-cli: %v\n", err) fmt.Fprintf(stderr, "mesh-cli: %v\n", err)
+11 -1
View File
@@ -2,4 +2,14 @@ module mesh-cli
go 1.26.0 go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21
require golang.org/x/crypto v0.57.0
require (
github.com/klauspost/compress v1.20.0 // indirect
github.com/nats-io/nats.go v1.54.0 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/sys v0.48.0 // indirect
)
+14
View File
@@ -1,2 +1,16 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
@@ -55,8 +55,20 @@ type AccountHash struct {
Node string `json:"node,omitempty"` Node string `json:"node,omitempty"`
// Rotating is a rotation under way over a second login (rotation.go, ADR 0561 §4): absent when none is. // Rotating is a rotation under way over a second login (rotation.go, ADR 0561 §4): absent when none is.
Rotating *Rotating `json:"rotating,omitempty"` Rotating *Rotating `json:"rotating,omitempty"`
// Kept marks an account kept for an enrolment or a node's login (enrolment.go, noderotation.go): `enrolment` for a
// token's user `enrol.<node>`, `node` for a node's login. Such a key is dropped by the verb that ends it, never
// retired as an account no longer wanted (retire.go).
Kept string `json:"kept,omitempty"`
// KeysDigest is, for a joining node's login, the digest of the keys it was minted for (ADR 0587 §3).
KeysDigest string `json:"keys-digest,omitempty"`
} }
// The kinds of account the vault keeps beside the wanted ones (AccountHash.Kept).
const (
KeptEnrolment = "enrolment"
KeptNode = "node"
)
// AccountStore is the vault's state of bus-account hashes: read and written by the vault's instances alone. // AccountStore is the vault's state of bus-account hashes: read and written by the vault's instances alone.
type AccountStore interface { type AccountStore interface {
Get(user string) (*AccountHash, error) Get(user string) (*AccountHash, error)
@@ -137,6 +137,14 @@ func (v *Vault) adopt(node string, raw []byte, by string, force bool) (Adoption,
return Adoption{}, err return Adoption{}, err
} }
// The users the loaded list holds, for a node's rotation to see its new login in it and the old one gone (ADR 0583 §3).
loaded := make(map[string]string, len(accounts))
for _, acc := range accounts {
loaded[acc.User] = acc.Hash
}
v.loadedListHeld(h.From, loaded)
defer v.advanceRotations()
v.hashMu.Lock() v.hashMu.Lock()
defer v.hashMu.Unlock() defer v.hashMu.Unlock()
held, err := v.heldUsers() held, err := v.heldUsers()
@@ -0,0 +1,577 @@
package main
// The vault's part of enrolment (novox/hq ADR 0587 §3, §5, §8; ADR 0561 §4, §5; issue 774).
//
// A machine joins with a token a person types. The seat mesh-enrolment decides who may join; the vault holds the secrets.
// Its verb `enrolment-token` makes the token's one-time secret, keeps it with its operator copy as `enrol/<node>/token`,
// writes its bcrypt hash under `bus-accounts` as the user `enrol.<node>`, and answers at issue the public fields alone
// (never the token). Asked to show it, it answers the whole token sealed to the operator key, which the operator opens at
// their own terminal (`nox enrol open`); asked to drop it, it takes the token and its hash away. Its verb `node-account`
// makes the account `node.<node>` of a joining machine at the redeem: once for a node and a set of keys, sealed to the key
// the machine presented, with the node's key in `node-logins` naming that login in use and nothing rotating. Only the
// seat mesh-enrolment may call either: the runtime proves who asks.
import (
"crypto/ecdh"
"crypto/ed25519"
"crypto/rand"
"crypto/x509"
"encoding/base64"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"golang.org/x/crypto/bcrypt"
)
// The names enrolment gives: the module that holds the seat mesh-enrolment, the prefix of a token's bus user, and the
// vault's state of each node's logins.
const (
EnrolmentModule = delivery.EnrolmentModule
EnrolUserPrefix = delivery.EnrolmentUserPrefix
NodeLoginsState = delivery.NodeLoginsState
// MeshHostModule is the module a node's own login secret is kept under (`own/<node>/mesh-host/<login>`, ADR 0583 §3).
MeshHostModule = "mesh-host"
)
// Events the vault says for enrolment's accounts are account-hashed (accounts.go), never carrying a hash or a value.
// enrolNode says node is a name a joining machine can have: one the mesh writes, with no dot, and not shaped like a
// second login.
func enrolNode(node string) error {
if !namePart.MatchString(node) || strings.Contains(node, ".") || strings.Contains(node, SecondLoginSuffix) {
return fmt.Errorf("%q is not a name a node can have: lower-case words, no dot, and not ending in %s", node, SecondLoginSuffix)
}
return nil
}
// enrolSecret is the vault's name of a node's enrolment token.
func enrolSecret(node string) string { return "enrol/" + node + "/token" }
// enrolSecretName says name is an enrolment token, which is handed to nobody: only shown, sealed to the operator key.
func enrolSecretName(name string) bool { return strings.HasPrefix(name, "enrol/") }
// loginSecret is the vault's name of a node's login, the value its node-engine takes (ADR 0583 §3 insight).
func loginSecret(node, login string) string {
return "own/" + node + "/" + MeshHostModule + "/" + login
}
// nodeLoginSecret says name is a node's login secret, and for which node and login. A node's login is handed to that
// node's node-engine alone.
func nodeLoginSecret(name string) (node, login string, ok bool) {
parts := strings.Split(name, "/")
if len(parts) != 4 || parts[0] != "own" || parts[2] != MeshHostModule {
return "", "", false
}
base := strings.TrimSuffix(parts[3], SecondLoginSuffix)
if base != NodeEnginePrefix+parts[1] {
return "", "", false
}
return parts[1], parts[3], true
}
// loginRefusal says why caller may not have the secret name when it is a node's login, or "": a node's login is the
// node-engine of that node's alone (`node.<node>` or its second login), sealed to the key that machine stated.
func (v *Vault) loginRefusal(caller, name string) (isLogin bool, why string) {
node, _, ok := nodeLoginSecret(name)
if !ok {
return false, ""
}
engine, _, _ := strings.Cut(strings.TrimPrefix(caller, NodeEnginePrefix), SecondLoginSuffix)
if !strings.HasPrefix(caller, NodeEnginePrefix) || engine != node {
return true, fmt.Sprintf("%s is a login of %s's node-engine and is handed to that machine's engine alone (%s or its second login)",
name, node, NodeEnginePrefix+node)
}
return true, ""
}
// loginKeyRefusal says why a node's login is not sealed to the key asked: it is sealed to the key the node's node-engine
// stated (node-key) and to no other, or "".
func (v *Vault) loginKeyRefusal(caller, publicKey string) string {
node, _ := loginNode(caller)
stated, had, err := v.Store.NodeKey(node)
if err != nil {
return "the key " + node + "'s node-engine stated could not be read: " + err.Error()
}
if !had {
return "the vault holds no sealing key that " + node + "'s node-engine stated (node-key)"
}
want, err := publicPEM(stated)
if err != nil {
return "the key " + node + "'s node-engine stated is not usable: " + err.Error()
}
given, err := askerKey(publicKey)
if err != nil || strings.TrimSpace(given) != strings.TrimSpace(want) {
return "a node's login is sealed to the key its node-engine stated, and the key sent is another"
}
return ""
}
// fromEnrolment refuses every caller but the seat mesh-enrolment, as the runtime proved it.
func (v *Vault) fromEnrolment(caller, verb string) error {
callers := v.EnrolCallers
if callers == nil {
callers = stdio.Callers{EnrolmentModule}
}
if !callers.Allows(caller) {
if caller == "" {
caller = stdio.Unproven
}
return fmt.Errorf("%s is asked by the seat %s alone, which decides who may join; %s is not it (novox/hq ADR 0587 §3)",
verb, EnrolmentModule, caller)
}
return nil
}
// TokenBase is what a token states of the bus beside its secret: where the bus is and the certificate to expect there.
// The machine's first tunnel comes with the issue call (ADR 0587 §3, insight 2026-11-02).
type TokenBase struct {
Broker string `json:"broker"`
Fingerprint string `json:"fingerprint"`
}
// EnrolmentTokenArgs are `enrolment-token`'s arguments (the seat's contract, mesh-sdk delivery): the node, when it expires
// (the seat's word), the machine's first tunnel at issue, and which of the three acts: issue (neither switch), sealed or
// drop.
type EnrolmentTokenArgs struct {
Node string
Expires string
Tunnel *delivery.TokenTunnel
Sealed bool
Drop bool
}
// EnrolmentTokenAnswer is mesh-sdk's delivery.EnrolmentTokenAnswer, the wire shape mesh-enrolment decodes: at issue the
// bus's address, its certificate's fingerprint and the vault's public signing key; when sealed, the whole token sealed to
// the operator key; at drop, whether it was dropped. Never a token in the clear.
type EnrolmentTokenAnswer = delivery.EnrolmentTokenAnswer
// enrolTunnel is the vault's name of the first tunnel a node's token carries, kept beside the token.
func enrolTunnel(node string) string { return "enrol/" + node + "/tunnel" }
// EnrolmentToken answers caller's `enrolment-token`.
func (v *Vault) EnrolmentToken(caller string, a EnrolmentTokenArgs) (EnrolmentTokenAnswer, error) {
if err := v.fromEnrolment(caller, "enrolment-token"); err != nil {
return EnrolmentTokenAnswer{}, err
}
if err := enrolNode(a.Node); err != nil {
return EnrolmentTokenAnswer{}, err
}
if v.Accounts == nil {
return EnrolmentTokenAnswer{}, errors.New("the vault has no state of bus accounts to write to")
}
switch {
case a.Drop && a.Sealed:
return EnrolmentTokenAnswer{}, errors.New("a token is answered sealed or dropped, not both")
case a.Drop:
return v.dropEnrolmentToken(caller, a.Node)
case a.Sealed:
return v.showEnrolmentToken(caller, a.Node)
}
return v.issueEnrolmentToken(caller, a)
}
// issueEnrolmentToken makes the secret afresh (an earlier token of the node is replaced), keeps it with its operator copy,
// and writes its hash as the user `enrol.<node>`. Its answer carries the public fields alone.
func (v *Vault) issueEnrolmentToken(caller string, a EnrolmentTokenArgs) (EnrolmentTokenAnswer, error) {
// What the machine must trust is known before anything is kept: the bus, its certificate, the vault's signing key,
// and the first tunnel whole (ADR 0587 §3), so the token shown later is one a machine can use.
base, err := v.tokenBase()
if err != nil {
return EnrolmentTokenAnswer{}, err
}
keys, err := v.keys()
if err != nil {
return EnrolmentTokenAnswer{}, fmt.Errorf("the token carries the vault's signing key, which is not held yet: %w", err)
}
signer := keys.key.Public().(ed25519.PublicKey)
if a.Tunnel == nil {
return EnrolmentTokenAnswer{}, errors.New("enrolment-token at issue carries the machine's first tunnel (`tunnel`: key, address, range, " +
"hub_key, hub_endpoint), so the machine reaches the bus over the private network (novox/hq ADR 0587 §3)")
}
if missing := (delivery.Token{Version: 1, Broker: base.Broker, Fingerprint: base.Fingerprint, Signer: signer,
Secret: "-", Tunnel: a.Tunnel}).Missing(); len(missing) > 0 {
return EnrolmentTokenAnswer{}, fmt.Errorf("the token of %s would lack %s: nothing was issued", a.Node, strings.Join(missing, ", "))
}
tunnel, err := json.Marshal(a.Tunnel)
if err != nil {
return EnrolmentTokenAnswer{}, err
}
raw := make([]byte, 30)
if _, err := rand.Read(raw); err != nil {
return EnrolmentTokenAnswer{}, err
}
secret := base64.RawURLEncoding.EncodeToString(raw)
name, user := enrolSecret(a.Node), EnrolUserPrefix+a.Node
c, err := v.keepWith(name, secret, "the vault", caller, OriginMade, keepOptions{quiet: true})
if err != nil {
return EnrolmentTokenAnswer{}, fmt.Errorf("the token of %s is not kept: %w", a.Node, err)
}
hash, err := bcrypt.GenerateFromPassword([]byte(secret), v.hashCost())
if err != nil {
return EnrolmentTokenAnswer{}, err
}
if _, err := v.keepWith(enrolTunnel(a.Node), string(tunnel), "the vault", caller, OriginMade, keepOptions{quiet: true}); err != nil {
return EnrolmentTokenAnswer{}, fmt.Errorf("the first tunnel of %s is not kept: %w", a.Node, err)
}
at := v.Now().UTC()
fp := Fingerprint(secret)
if err := v.Accounts.Put(AccountHash{User: user, Hash: string(hash), Of: name, Generation: c.Generation, Fingerprint: fp,
At: at, Kept: KeptEnrolment}); err != nil {
return EnrolmentTokenAnswer{}, fmt.Errorf("the hash of %s could not be written to %s: %w", user, BusAccountsState, err)
}
v.saidHashed(user, name, c.Generation, fp, at)
_ = v.recordHanding(Handing{To: caller, Name: name, Generation: c.Generation, Outcome: "issued", Why: "enrolment token of " + a.Node})
v.log("made the enrolment token of %s for %s (generation %d); its hash is the user %s", a.Node, caller, c.Generation, user)
return EnrolmentTokenAnswer{Node: a.Node, Broker: base.Broker, Fingerprint: base.Fingerprint,
Signer: base64.StdEncoding.EncodeToString(signer)}, nil
}
// showEnrolmentToken answers the whole token sealed to the operator key: the secret held, the bus's address and
// certificate fingerprint, and the vault's public signing key.
func (v *Vault) showEnrolmentToken(caller, node string) (EnrolmentTokenAnswer, error) {
rec, held, err := v.Store.Read(enrolSecret(node))
if err != nil {
return EnrolmentTokenAnswer{}, err
}
if !held {
return EnrolmentTokenAnswer{}, fmt.Errorf("the vault holds no enrolment token of %s", node)
}
secret, err := openAtRest(rec, v)
if err != nil {
return EnrolmentTokenAnswer{}, fmt.Errorf("the token of %s does not open with the vault's key: %w", node, err)
}
operator, err := v.Store.OperatorKey()
if err != nil {
return EnrolmentTokenAnswer{}, err
}
if operator == "" {
return EnrolmentTokenAnswer{}, errors.New("the vault has no operator key to seal the token to (ADR 0113 §22)")
}
base, err := v.tokenBase()
if err != nil {
return EnrolmentTokenAnswer{}, err
}
keys, err := v.keys()
if err != nil {
return EnrolmentTokenAnswer{}, fmt.Errorf("the token carries the vault's signing key, which is not held yet: %w", err)
}
var tunnel *delivery.TokenTunnel
if trec, theld, err := v.Store.Read(enrolTunnel(node)); err != nil {
return EnrolmentTokenAnswer{}, err
} else if theld {
raw, err := openAtRest(trec, v)
if err != nil {
return EnrolmentTokenAnswer{}, fmt.Errorf("the first tunnel of %s does not open with the vault's key: %w", node, err)
}
tunnel = &delivery.TokenTunnel{}
if err := json.Unmarshal([]byte(raw), tunnel); err != nil {
return EnrolmentTokenAnswer{}, fmt.Errorf("the first tunnel of %s is not readable: %w", node, err)
}
}
signer := keys.key.Public().(ed25519.PublicKey)
tok := delivery.Token{Version: 1, Node: node, Broker: base.Broker, Fingerprint: base.Fingerprint,
Signer: []byte(signer), Secret: secret, Tunnel: tunnel}
if tunnel == nil {
return EnrolmentTokenAnswer{}, fmt.Errorf("the vault keeps no first tunnel for %s's token: issue it again with its tunnel", node)
}
text, err := tok.Encode()
if err != nil {
return EnrolmentTokenAnswer{}, err
}
sealed, err := operatorSeal(operator, []byte(text))
if err != nil {
return EnrolmentTokenAnswer{}, err
}
_ = v.recordHanding(Handing{To: caller, Name: enrolSecret(node), Generation: rec.Generation, Outcome: "shown",
Why: "sealed to the operator key"})
return EnrolmentTokenAnswer{Node: node, Broker: base.Broker, Fingerprint: base.Fingerprint,
Signer: base64.StdEncoding.EncodeToString(signer), Sealed: sealed}, nil
}
// tokenBase is what a token states of the bus, or why the vault knows none.
func (v *Vault) tokenBase() (TokenBase, error) {
if v.TokenBase == nil {
return TokenBase{}, errors.New("the vault does not know the bus's address and certificate fingerprint, so it makes no token to show")
}
return v.TokenBase()
}
// dropEnrolmentToken takes the token and its hash away, at every end of an enrolment (ADR 0587 §5). Nothing held is
// answered as dropped nothing.
func (v *Vault) dropEnrolmentToken(caller, node string) (EnrolmentTokenAnswer, error) {
name, user := enrolSecret(node), EnrolUserPrefix+node
out := EnrolmentTokenAnswer{Node: node}
a, err := v.Accounts.Get(user)
if err != nil {
return out, err
}
if a != nil {
if a.Kept != KeptEnrolment {
return out, fmt.Errorf("%s is no enrolment token's user: nothing was dropped", user)
}
if err := v.Accounts.Delete(user); err != nil {
return out, fmt.Errorf("the hash of %s could not be taken out of %s: %w", user, BusAccountsState, err)
}
out.Dropped = true
}
if _, held, err := v.Store.Read(name); err != nil {
return out, err
} else if held {
if err := v.Store.Delete(name); err != nil {
return out, fmt.Errorf("the token of %s could not be deleted: %w", node, err)
}
out.Dropped = true
}
if _, held, err := v.Store.Read(enrolTunnel(node)); err != nil {
return out, err
} else if held {
if err := v.Store.Delete(enrolTunnel(node)); err != nil {
return out, fmt.Errorf("the first tunnel of %s could not be deleted: %w", node, err)
}
}
if out.Dropped {
_ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "dropped", Why: "enrolment token of " + node})
v.log("dropped the enrolment token of %s and its hash, asked by %s", node, caller)
}
return out, nil
}
// saidHashed says account-hashed for a user whose key the vault wrote: never the hash or the value.
func (v *Vault) saidHashed(user, of string, generation int, fingerprint string, at time.Time) {
if v.Emit == nil {
return
}
if err := v.Emit(AccountHashedEvent, map[string]any{"user": user, "of": of, "generation": generation,
"fingerprint": fingerprint, "at": at.Format(time.RFC3339Nano)}); err != nil {
v.log("the hash of %s is written, but %s could not be said: %v", user, AccountHashedEvent, err)
}
}
// NodeAccountArgs are `node-account`'s arguments: the joining node, the sealing key its machine presented (raw X25519 in
// base64, or SPKI PEM), the digest of the keys it presented, and whether to retire the account instead.
type NodeAccountArgs struct {
Node string
SealingKey string
KeysDigest string
Retire bool
}
// NodeAccountAnswer is mesh-sdk's delivery.NodeAccountAnswer, the wire shape mesh-enrolment decodes and relays into the
// redeem answer: the login, and the account sealed to the presented key, as the node-engine opens it (the password,
// sealed anonymously to its raw X25519 key, in base64); or that it was retired.
type NodeAccountAnswer = delivery.NodeAccountAnswer
// NodeAccount answers caller's `node-account`: the joining node's account, minted once for a node and a set of keys and
// answered sealed to the key presented; or, with retire, taken away again at a revoke.
func (v *Vault) NodeAccount(caller string, a NodeAccountArgs) (NodeAccountAnswer, error) {
if err := v.fromEnrolment(caller, "node-account"); err != nil {
return NodeAccountAnswer{}, err
}
if err := enrolNode(a.Node); err != nil {
return NodeAccountAnswer{}, err
}
if v.Accounts == nil {
return NodeAccountAnswer{}, errors.New("the vault has no state of bus accounts to write to")
}
user := NodeEnginePrefix + a.Node
secret := loginSecret(a.Node, user)
v.hashMu.Lock()
defer v.hashMu.Unlock()
was, err := v.Accounts.Get(user)
if err != nil {
return NodeAccountAnswer{}, fmt.Errorf("%s %s could not be read: %w", BusAccountsState, user, err)
}
if a.Retire {
return v.retireNodeAccount(caller, a.Node, user, secret, was)
}
if was != nil && (was.Kept != KeptNode || was.Adopted) {
return NodeAccountAnswer{}, fmt.Errorf("%s already has an account the vault did not make for an enrolment: a node's account is never "+
"minted anew (novox/hq ADR 0561 §4)", user)
}
if _, held, err := v.Store.Read(enrolSecret(a.Node)); err != nil {
return NodeAccountAnswer{}, err
} else if !held {
return NodeAccountAnswer{}, fmt.Errorf("%s holds no enrolment token, so no enrolment is under way: no account is made", a.Node)
}
to, err := sealingRaw(a.SealingKey)
if err != nil {
return NodeAccountAnswer{}, err
}
if strings.TrimSpace(a.KeysDigest) == "" {
return NodeAccountAnswer{}, errors.New("node-account names the digest of the keys the machine presented (`keys_digest`)")
}
if was == nil {
if err := v.mintNodeAccount(caller, a.Node, user, secret, a.KeysDigest); err != nil {
return NodeAccountAnswer{}, err
}
} else if was.KeysDigest != a.KeysDigest {
return NodeAccountAnswer{}, fmt.Errorf("%s was made for another set of keys than the ones presented now: revoke the enrolment and issue it again", user)
}
rec, held, err := v.Store.Read(secret)
if err != nil || !held {
return NodeAccountAnswer{}, fmt.Errorf("the vault holds no value of %s to answer: %v", user, err)
}
value, err := openAtRest(rec, v)
if err != nil {
return NodeAccountAnswer{}, fmt.Errorf("%s does not open with the vault's key: %w", secret, err)
}
var acct struct {
Password string `json:"password"`
}
if err := json.Unmarshal([]byte(value), &acct); err != nil || acct.Password == "" {
return NodeAccountAnswer{}, fmt.Errorf("%s holds no password to answer", secret)
}
sealed, err := operatorSeal(to, []byte(acct.Password))
if err != nil {
return NodeAccountAnswer{}, err
}
_ = v.recordHanding(Handing{To: caller, Name: secret, Generation: rec.Generation, Outcome: "handed", Why: "node account of a joining node, sealed to its presented key"})
return NodeAccountAnswer{Node: a.Node, Login: user, Account: sealed}, nil
}
// mintNodeAccount makes a joining node's account: its value kept as the login's own secret, its hash in bus-accounts, and
// the node's key in node-logins naming it in use with nothing rotating.
func (v *Vault) mintNodeAccount(caller, node, user, secret, digest string) error {
value, password, err := v.accountValue(node, user)
if err != nil {
return err
}
c, err := v.keepWith(secret, value, "the vault", caller, OriginMade, keepOptions{quiet: true})
if err != nil {
return fmt.Errorf("the account of %s is not kept: %w", node, err)
}
hash, err := bcrypt.GenerateFromPassword([]byte(password), v.hashCost())
if err != nil {
return err
}
at := v.Now().UTC()
fp := Fingerprint(value)
if err := v.Accounts.Put(AccountHash{User: user, Hash: string(hash), Of: secret, Generation: c.Generation, Fingerprint: fp,
At: at, Kept: KeptNode, KeysDigest: digest}); err != nil {
return fmt.Errorf("the hash of %s could not be written to %s: %w", user, BusAccountsState, err)
}
v.saidHashed(user, secret, c.Generation, fp, at)
if err := v.putLogin(NodeLogin{Node: node, Login: user, Since: at}); err != nil {
return fmt.Errorf("the account of %s is made, and %s %s could not be written: %w", node, NodeLoginsState, node, err)
}
v.log("made the account %s of a joining node for %s", user, caller)
return nil
}
// accountValue is a node's login whole, in the shape the controller's accounts have (url, fingerprint, node, user,
// password), and its password.
func (v *Vault) accountValue(node, user string) (value, password string, err error) {
raw := make([]byte, 30)
if _, err := rand.Read(raw); err != nil {
return "", "", err
}
password = base64.RawURLEncoding.EncodeToString(raw)
acct := map[string]string{"node": node, "user": user, "password": password}
if v.TokenBase != nil {
if base, err := v.TokenBase(); err == nil {
acct["url"], acct["fingerprint"] = base.Broker, base.Fingerprint
}
}
out, err := json.Marshal(acct)
return string(out), password, err
}
// retireNodeAccount takes a joining node's account away: its hash, its value and its key in node-logins. Only one the
// vault made for an enrolment, and only while nothing rotates.
func (v *Vault) retireNodeAccount(caller, node, user, secret string, was *AccountHash) (NodeAccountAnswer, error) {
out := NodeAccountAnswer{Node: node, Login: user}
if was != nil && was.Kept != KeptNode {
return out, fmt.Errorf("%s is not an account the vault made for an enrolment: nothing was retired", user)
}
if nl, err := v.getLogin(node); err != nil {
return out, err
} else if nl != nil && nl.Rotating != "" {
return out, fmt.Errorf("%s is rotating its login: nothing was retired", node)
}
if was != nil {
if err := v.Accounts.Delete(user); err != nil {
return out, fmt.Errorf("the hash of %s could not be taken out of %s: %w", user, BusAccountsState, err)
}
out.Retired = true
}
if _, held, err := v.Store.Read(secret); err != nil {
return out, err
} else if held && was != nil {
if err := v.Store.Delete(secret); err != nil {
return out, fmt.Errorf("the value of %s could not be deleted: %w", user, err)
}
}
if out.Retired {
if err := v.deleteLogin(node); err != nil {
return out, err
}
_ = v.recordHanding(Handing{To: caller, Name: secret, Outcome: "retired", Why: "node account of an enrolment revoked"})
v.log("retired the account %s of a revoked enrolment, asked by %s", user, caller)
}
return out, nil
}
// sealingRaw takes the key a machine presents to be sealed to, as raw X25519 in base64 or SPKI PEM, and gives it raw in
// base64: the form the node-engine's sealing key opens an anonymous box with.
func sealingRaw(key string) (string, error) {
key = strings.TrimSpace(key)
if key == "" {
return "", errors.New("node-account names the sealing key the machine presented (`sealing_key`)")
}
if raw, err := base64.StdEncoding.DecodeString(key); err == nil && len(raw) == 32 {
return key, nil
}
block, _ := pem.Decode([]byte(key))
if block == nil {
return "", errors.New("the sealing key presented is neither a raw X25519 key in base64 nor SPKI PEM")
}
parsed, err := x509.ParsePKIXPublicKey(block.Bytes)
pub, ok := parsed.(*ecdh.PublicKey)
if err != nil || !ok || pub.Curve() != ecdh.X25519() {
return "", errors.New("the sealing key presented is not an X25519 key")
}
return base64.StdEncoding.EncodeToString(pub.Bytes()), nil
}
// sealingPEM takes the key a machine presents to be sealed to, as raw X25519 in base64 or SPKI PEM, and gives PEM.
func sealingPEM(key string) (string, error) {
key = strings.TrimSpace(key)
if key == "" {
return "", errors.New("node-account names the sealing key the machine presented (`sealing_key`)")
}
if strings.Contains(key, "BEGIN") {
return askerKey(key)
}
pem, err := publicPEM(key)
if err != nil {
return "", fmt.Errorf("the sealing key presented is not a usable key: %w", err)
}
return pem, nil
}
// TokenBaseFromFile reads what a token states of the bus from the vault's own account file (`{url, fingerprint}` as the
// controller's accounts have), in the directory of the vault's state. It answers an error when there is none.
func TokenBaseFromFile(stateDir string) func() (TokenBase, error) {
return func() (TokenBase, error) {
raw, err := os.ReadFile(filepath.Join(stateDir, "broker"))
if err != nil {
return TokenBase{}, fmt.Errorf("the vault's own account is not readable, so the bus's address is not known: %w", err)
}
var acct struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint"`
}
if json.Unmarshal(raw, &acct) != nil || acct.URL == "" || acct.Fingerprint == "" {
return TokenBase{}, errors.New("the vault's own account names no bus address and certificate fingerprint")
}
return TokenBase{Broker: acct.URL, Fingerprint: acct.Fingerprint}, nil
}
}
@@ -0,0 +1,503 @@
package main
import (
"crypto/ecdh"
"crypto/ed25519"
"crypto/rand"
"crypto/x509"
"encoding/base64"
"encoding/json"
"encoding/pem"
"fmt"
"slices"
"sort"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"golang.org/x/crypto/bcrypt"
"golang.org/x/crypto/nacl/box"
)
// loginsState is the vault's state node-logins in memory; like the runtime it refuses a value naming a credential.
type loginsState struct {
mu sync.Mutex
keys map[string]NodeLogin
}
func (l *loginsState) Get(node string) (*NodeLogin, error) {
l.mu.Lock()
defer l.mu.Unlock()
n, ok := l.keys[node]
if !ok {
return nil, nil
}
return &n, nil
}
func (l *loginsState) Put(n NodeLogin) error {
l.mu.Lock()
defer l.mu.Unlock()
raw, err := delivery.EncodeNodeLogin(n)
if err != nil {
return err
}
if field := credentialField(raw); field != "" {
return fmt.Errorf("%s carries a field %q, which names a credential", n.Node, field)
}
if l.keys == nil {
l.keys = map[string]NodeLogin{}
}
l.keys[n.Node] = n
return nil
}
func (l *loginsState) Delete(node string) error {
l.mu.Lock()
defer l.mu.Unlock()
delete(l.keys, node)
return nil
}
func (l *loginsState) Keys() ([]string, error) {
l.mu.Lock()
defer l.mu.Unlock()
var out []string
for k := range l.keys {
out = append(out, k)
}
sort.Strings(out)
return out, nil
}
// enrolling is a vault that mints, with the lease handed away, a loaded user list at revision 570, the bus on novox, the
// nodes g14, ace and novox placed, their existing logins adopted, and hand acts of person.jochen's.
type enrolling struct {
testVault
accounts *fakeAccounts
logins *loginsState
acts map[string]HandActSeen
}
func newEnrolling(t *testing.T) *enrolling {
t.Helper()
v := newVault(t, nil)
e := &enrolling{testVault: v, accounts: &fakeAccounts{keys: map[string]AccountHash{}}, logins: &loginsState{keys: map[string]NodeLogin{}}, acts: map[string]HandActSeen{}}
v.Accounts, v.Logins = e.accounts, e.logins
v.Placement = placementsOf(t, map[string][]string{"novox": {"nats"}, "g14": {"web"}, "ace": {"shop"}})
v.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil)
v.LoadedListHeard(570)
v.TokenBase = func() (TokenBase, error) {
return TokenBase{Broker: "tls://10.77.0.1:4222", Fingerprint: "sha256:bus"}, nil
}
v.ReadAct = func(id string) (*HandActSeen, error) {
if a, ok := e.acts[id]; ok {
return &a, nil
}
return nil, nil
}
for _, n := range []string{"g14", "ace", "novox"} {
e.accounts.keys["node."+n] = AccountHash{User: "node." + n, Hash: "$2a$04$adopted", Adopted: true, At: time.Now()}
pub, _ := aSealingKey(t)
if err := v.recordNodeKey(n, pub, "node."+n); err != nil {
t.Fatal(err)
}
}
return e
}
func (e *enrolling) act(id, by, node string) {
e.acts[id] = HandActSeen{ID: id, By: by, At: time.Now().Add(time.Hour), Verb: "rotate-node", Args: []string{node}}
}
// 774/S43/AC1-AC3, S44/AC1-AC3, S45/AC1 (novox/hq ADR 0587 §3, §5, §6, §8): the vault's enrolment verbs
// are called through their real tool handlers with the arguments mesh-enrolment sends, and each wire answer is decoded
// into mesh-sdk's delivery types the holder decodes (EnrolmentTokenAnswer, NodeAccountAnswer, SignAnswer and the signed
// parts of RedeemAnswer), with every field the holder reads filled. Only mesh-enrolment is answered; a token is issued
// only with its first tunnel whole, its hash is bcrypt of the kept secret, `sealed` answers it sealed to the operator key
// (a token whose Missing() is empty) and issues nothing; the account is the same for the same keys and opens with the
// sealing key presented, as the node-engine opens it; a joining node's certificate is signed for the serving key presented
// only when the keys have the digest node-account recorded; a retire and a drop take it all away.
func TestTheVaultsEnrolmentAnswersAreMeshSDKsWireShapes(t *testing.T) {
e := newEnrolling(t)
if _, err := e.EnsureSigning(); err != nil {
t.Fatal(err)
}
// new1 is placed with its private-network address, as mesh-enrolment writes it before it asks for the certificate.
values := map[string][]byte{}
for node, p := range map[string]delivery.Placement{"novox": {Node: "novox", Modules: []delivery.PlacedModule{{Module: "nats"}}},
"g14": {Node: "g14"}, "new1": {Node: "new1", Decided: delivery.NodeDecided{Address: "10.77.0.9"}}} {
p.V = delivery.Version
kv, err := delivery.PlacementValues(p)
if err != nil {
t.Fatal(err)
}
for k, raw := range kv {
values[k] = raw
}
_ = node
}
e.Placement = PlacementReader{Read: func(key string) ([]byte, error) { return values[key], nil },
Keys: func() ([]string, error) {
var out []string
for k := range values {
out = append(out, k)
}
return out, nil
}}
defs := definitions(&Env{Vault: e.Vault})
call := func(verb, caller string, args map[string]any) (json.RawMessage, error) {
out, err := toolNamed(t, defs, Seat+"."+verb).run(caller, args)
if err != nil {
return nil, err
}
return json.Marshal(out)
}
const holder = "novox.mesh-enrolment"
signer, _ := e.keys()
signerB64 := base64.StdEncoding.EncodeToString(signer.key.Public().(ed25519.PublicKey))
tunnel := map[string]any{"key": "machine-tunnel-key", "address": "10.77.0.9", "range": "10.77.0.0/24",
"hub_key": "hub-tunnel-key", "hub_endpoint": "203.0.113.1:51820"}
without := func(field string) map[string]any {
out := map[string]any{}
for k, x := range tunnel {
if k != field {
out[k] = x
}
}
return out
}
sealingKey, _ := ecdh.X25519().GenerateKey(rand.Reader)
sealingPub := base64.StdEncoding.EncodeToString(sealingKey.PublicKey().Bytes())
identity, _, _ := ed25519.GenerateKey(rand.Reader)
serving, _, _ := ed25519.GenerateKey(rand.Reader)
keys := map[string]any{"public_key": base64.StdEncoding.EncodeToString(identity), "overlay_key": "overlay-pub",
"sealing_key": sealingPub, "serving_key": base64.StdEncoding.EncodeToString(serving)}
digest := delivery.KeysDigest(keys["public_key"].(string), "overlay-pub", sealingPub, keys["serving_key"].(string))
with := func(base map[string]any, more map[string]any) map[string]any {
out := map[string]any{}
for k, x := range base {
out[k] = x
}
for k, x := range more {
out[k] = x
}
return out
}
reply := delivery.RedeemAnswer{Node: "new1", Login: "node.new1", Account: "sealed", Certificate: "pem", Signer: signerB64}
var issued, sealed, dropped delivery.EnrolmentTokenAnswer
var account, accountAgain, retired delivery.NodeAccountAnswer
var cert, replySig delivery.SignAnswer
var secret, accountHash string
for _, c := range []struct {
name, verb, caller string
args map[string]any
into any
filled []string // the json fields the holder reads, which must not be empty
refused string // a part of the refusal, when refused
}{
// Only the seat mesh-enrolment is answered: not the controller, a node, a person or a caller not proven.
{"a node asks a token", "enrolment-token", "node.g14", map[string]any{"node": "new1", "tunnel": tunnel}, nil, nil, "mesh-enrolment"},
{"the controller asks a token", "enrolment-token", "controller", map[string]any{"node": "new1", "tunnel": tunnel}, nil, nil, "mesh-enrolment"},
{"a person asks an account", "node-account", "person.jochen", map[string]any{"node": "new1"}, nil, nil, "mesh-enrolment"},
{"unproven asks an enrolment signed", "sign", "", map[string]any{"kind": delivery.KindEnrolment, "node": "new1", "reply": "AAEC"}, nil, nil, "proved"},
{"a grants part asks an enrolment signed", "sign", "novox.mesh-grants", map[string]any{"kind": delivery.KindEnrolment, "node": "new1", "reply": "AAEC"}, nil, nil, "mesh-enrolment"},
// Insight 2026-11-02: the first tunnel is whole at issue, or nothing is issued.
{"no tunnel", "enrolment-token", holder, map[string]any{"node": "new1"}, nil, nil, "first tunnel"},
{"a tunnel without the hub's key", "enrolment-token", holder, map[string]any{"node": "new1", "tunnel": without("hub_key")}, nil, nil, "hub's tunnel key"},
{"a tunnel without the hub's endpoint", "enrolment-token", holder, map[string]any{"node": "new1", "tunnel": without("hub_endpoint")}, nil, nil, "where the hub's tunnel is dialled"},
{"a tunnel without the address", "enrolment-token", holder, map[string]any{"node": "new1", "tunnel": without("address")}, nil, nil, "this machine's address"},
{"a tunnel without the machine's tunnel key", "enrolment-token", holder, map[string]any{"node": "new1", "tunnel": without("key")}, nil, nil, "the tunnel key it was issued for"},
{"a tunnel without the range", "enrolment-token", holder, map[string]any{"node": "new1", "tunnel": without("range")}, nil, nil, "the private network's range"},
{"a tunnel with a field it has not", "enrolment-token", holder, map[string]any{"node": "new1", "tunnel": with(tunnel, map[string]any{"port": 1})}, nil, nil, "not a first tunnel"},
// Issue answers what the machine must trust; sealed and drop as the holder sends them, the switch as a word.
{"issue", "enrolment-token", holder, map[string]any{"node": "new1", "expires": "2026-11-02T12:00:00Z", "tunnel": tunnel}, &issued, []string{"node", "broker", "fingerprint", "signer"}, ""},
{"sealed", "enrolment-token", holder, map[string]any{"node": "new1", "sealed": "true"}, &sealed, []string{"node", "sealed"}, ""},
// The account at the redeem, twice with the same keys; other keys and an adopted node's are refused.
{"node-account", "node-account", holder, map[string]any{"node": "new1", "sealing_key": sealingPub, "keys_digest": digest}, &account, []string{"node", "login", "account"}, ""},
{"node-account again", "node-account", holder, map[string]any{"node": "new1", "sealing_key": sealingPub, "keys_digest": digest}, &accountAgain, []string{"node", "login", "account"}, ""},
{"node-account for other keys", "node-account", holder, map[string]any{"node": "new1", "sealing_key": sealingPub, "keys_digest": "sha256:other"}, nil, nil, "another set of keys"},
{"node-account of an adopted node", "node-account", holder, map[string]any{"node": "g14", "sealing_key": sealingPub, "keys_digest": digest}, nil, nil, "never minted anew"},
// The certificate for the serving key presented, checked against node-account's digest; the reply signed.
{"a certificate without the keys", "sign", holder, map[string]any{"kind": delivery.KindNodeCertificate, "node": "new1", "serving_key": keys["serving_key"]}, nil, nil, "keys it presented"},
{"a certificate for another serving key", "sign", holder, with(keys, map[string]any{"kind": delivery.KindNodeCertificate, "node": "new1", "serving_key": base64.StdEncoding.EncodeToString(identity)}), nil, nil, "not the ones its account was made for"},
{"a certificate", "sign", holder, with(keys, map[string]any{"kind": delivery.KindNodeCertificate, "node": "new1"}), &cert, []string{"kind", "node", "certificate", "authority", "signer", "digest"}, ""},
{"the reply signed", "sign", holder, map[string]any{"kind": delivery.KindEnrolment, "node": "new1", "reply": base64.StdEncoding.EncodeToString(reply.SignedBytes())}, &replySig, []string{"kind", "node", "signature", "signer", "digest"}, ""},
{"a reply of a node with no token", "sign", holder, map[string]any{"kind": delivery.KindEnrolment, "node": "g14", "reply": "AAEC"}, nil, nil, "no enrolment token"},
// A revoke: the account retired, the token and its hash dropped.
{"retire", "node-account", holder, map[string]any{"node": "new1", "retire": "true"}, &retired, []string{"node", "retired"}, ""},
{"drop", "enrolment-token", holder, map[string]any{"node": "new1", "drop": "true"}, &dropped, []string{"node", "dropped"}, ""},
} {
raw, err := call(c.verb, c.caller, c.args)
if c.refused != "" {
if err == nil || !strings.Contains(err.Error(), c.refused) {
t.Errorf("%s: answered %s, %v; want a refusal saying %q", c.name, raw, err, c.refused)
}
continue
}
if err != nil {
t.Fatalf("%s: %v", c.name, err)
}
if err := json.Unmarshal(raw, c.into); err != nil {
t.Fatalf("%s: %s does not decode as mesh-sdk's shape: %v", c.name, raw, err)
}
var fields map[string]any
back, _ := json.Marshal(c.into)
_ = json.Unmarshal(back, &fields)
for _, f := range c.filled {
if x, ok := fields[f]; !ok || x == "" || x == false {
t.Errorf("%s: the wire answer %s leaves %q empty", c.name, raw, f)
}
}
if c.name == "issue" {
// What was kept: the secret, whose bcrypt hash is enrol.new1; listed, never handed out, never answered.
rec, held, _ := e.Store.Read("enrol/new1/token")
secret, _ = openAtRest(rec, e.Vault)
key := e.accounts.keys["enrol.new1"]
if !held || secret == "" || strings.Contains(string(raw), secret) || strings.Contains(strings.Join(e.bus.raw, " "), secret) ||
key.Kept != KeptEnrolment || bcrypt.CompareHashAndPassword([]byte(key.Hash), []byte(secret)) != nil {
t.Fatalf("the token is held %v, its hash %+v, or its secret is in an answer or an event: %s", held, key, raw)
}
if held, _ := e.EnrolmentsHeld(); len(held) != 1 || held[0].User != "enrol.new1" {
t.Errorf("secret_holders names %+v", held)
}
}
if c.name == "node-account" {
accountHash = e.accounts.keys["node.new1"].Hash
if _, err := e.Get(holder, "enrol/new1/token", pemOf(t, mustPub(t))); err == nil {
t.Error("an enrolment token was handed out by get")
}
}
}
if issued.Broker != "tls://10.77.0.1:4222" || issued.Fingerprint != "sha256:bus" || issued.Signer != signerB64 || issued.Sealed != "" {
t.Errorf("issue answered %+v", issued)
}
// sealed issued nothing new: the token it seals is the one whose hash is enrol.new1, with its tunnel, and usable whole.
tok, err := delivery.ParseToken(openOperator(t, e.operatorPublic, e.operatorPrivate, sealed.Sealed))
if err != nil || len(tok.Missing()) != 0 || tok.Node != "new1" || tok.Broker != issued.Broker || tok.Tunnel == nil ||
tok.Tunnel.HubEndpoint != "203.0.113.1:51820" || base64.StdEncoding.EncodeToString(tok.Signer) != signerB64 ||
tok.Secret != secret {
t.Errorf("the sealed token is %+v, %v", tok, err)
}
// The account opens with the sealing key presented, as the node-engine opens it, and is the same both times.
open := func(a delivery.NodeAccountAnswer) string {
blob, _ := base64.StdEncoding.DecodeString(a.Account)
var pub, priv [32]byte
copy(pub[:], sealingKey.PublicKey().Bytes())
copy(priv[:], sealingKey.Bytes())
out, _ := box.OpenAnonymous(nil, blob, &pub, &priv)
return string(out)
}
if p := open(account); p == "" || p != open(accountAgain) || account.Login != "node.new1" ||
bcrypt.CompareHashAndPassword([]byte(accountHash), []byte(p)) != nil {
t.Errorf("the account is %+v / %+v", account, accountAgain)
}
// The certificate certifies the serving key presented, for new1's internal name.
block, _ := pem.Decode([]byte(cert.Certificate))
if block == nil {
t.Fatalf("the certificate is %q", cert.Certificate)
}
parsed, err := x509.ParseCertificate(block.Bytes)
if err != nil || !ed25519.PublicKey(serving).Equal(parsed.PublicKey) {
t.Errorf("the certificate is of another key: %v", err)
}
ctx, _ := delivery.SignContext(delivery.KindEnrolment)
sig, _ := base64.StdEncoding.DecodeString(replySig.Signature)
if !ed25519.Verify(signer.key.Public().(ed25519.PublicKey), append([]byte(ctx), reply.SignedBytes()...), sig) {
t.Error("the redeem answer's signature does not verify under the enrolment context")
}
// After the retire and the drop nothing of the enrolment remains.
if _, ok := e.accounts.keys["node.new1"]; ok || e.logins.keys["new1"].Node != "" {
t.Error("the retired account's hash or node-logins key remains")
}
for _, name := range []string{"own/new1/mesh-host/node.new1", "enrol/new1/token", "enrol/new1/tunnel"} {
if _, held, _ := e.Store.Read(name); held {
t.Errorf("%s remains after the retire and the drop", name)
}
}
if _, ok := e.accounts.keys["enrol.new1"]; ok {
t.Error("the token's hash remains after a drop")
}
}
func mustPub(t *testing.T) string {
t.Helper()
pub, _ := aSealingKey(t)
return pub
}
// 774/S46/AC1-AC3, S47/AC1-AC2, S48/AC1, S49/AC1 (novox/hq ADR 0583 §1 to §7): a rotation is answered only for a hand act
// read back by its id; one runs at a time and the bus node waits for the rest; the new login is ready only when the loaded
// list holds it; a confirm from any caller but the new login retires nothing; a proven confirm removes the old hash and the
// rotation is done once the loaded list lacks it; an abandon keeps the old hash; and a node's login is its engine's alone.
func TestANodesLoginRotatesOneAtATimeAndEndsWithTheListWithoutTheOldOne(t *testing.T) {
e := newEnrolling(t)
ask := func(node, act string) (RotateNodeAnswer, error) {
return e.RotateNode("person.jochen", RotateNodeArgs{Node: node, Why: "a rotation under test", Act: act})
}
// AC1: no recorded act, another caller's act, and an old act are all refused.
e.acts["theirs"] = HandActSeen{ID: "theirs", By: "person.other", At: time.Now().Add(time.Hour), Verb: "rotate-node", Args: []string{"g14"}}
e.acts["old"] = HandActSeen{ID: "old", By: "person.jochen", At: time.Unix(0, 0), Verb: "rotate-node", Args: []string{"g14"}}
e.acts["elsewhere"] = HandActSeen{ID: "elsewhere", By: "person.jochen", At: time.Now().Add(time.Hour), Verb: "rotate-node", Args: []string{"ace"}}
e.acts["restart"] = HandActSeen{ID: "restart", By: "person.jochen", At: time.Now().Add(time.Hour), Verb: "restart", Args: []string{"g14"}}
for _, act := range []string{"", "missing", "theirs", "old", "elsewhere", "restart"} {
if _, err := ask("g14", act); err == nil {
t.Errorf("act %q warranted a rotation", act)
}
}
if len(e.logins.keys) != 0 {
t.Fatalf("a refused rotation wrote %+v", e.logins.keys)
}
e.act("a1", "person.jochen", "g14")
e.act("a2", "person.jochen", "ace")
e.act("a3", "person.jochen", "novox")
first, err := ask("g14", "a1")
if err != nil || first.Queued || first.Rotating != "node.g14--b" || first.Stage != StageBegun {
t.Fatalf("g14 began as %+v, %v", first, err)
}
// The second login has a hash of its own value, the first is marked rotating, and the value waits for the engine.
second := e.accounts.keys["node.g14--b"]
rec, held, _ := e.Store.Read("own/g14/mesh-host/node.g14--b")
value, _ := openAtRest(rec, e.Vault)
var acct struct{ User, Password string }
if !held || json.Unmarshal([]byte(value), &acct) != nil || acct.User != "node.g14--b" ||
bcrypt.CompareHashAndPassword([]byte(second.Hash), []byte(acct.Password)) != nil || e.accounts.keys["node.g14"].Rotating == nil {
t.Fatalf("g14's second login is held %v as %s, keyed %+v", held, value, second)
}
// AC2: one at a time, the bus node last.
ace, err := ask("ace", "a2")
if err != nil || !ace.Queued || strings.Join(ace.Behind, ",") != "g14" {
t.Fatalf("ace asked while g14 runs: %+v, %v", ace, err)
}
bus, err := ask("novox", "a3")
if err != nil || !bus.Queued || strings.Join(bus.Behind, ",") != "g14,ace" {
t.Fatalf("novox, the bus node, asked: %+v, %v", bus, err)
}
if _, err := ask("g14", "a1"); err == nil {
t.Error("g14 began a rotation twice")
}
// AC3: the loaded list lacks the second login, so nothing is ready; once it holds it, node-account-ready is said.
e.loadedListHeld("sha256:l1", map[string]string{"node.g14": "$2a$04$adopted"})
e.advanceRotations()
if said(e.bus, NodeAccountReadyEvent) != 0 || e.logins.keys["g14"].Stage != StageBegun {
t.Fatalf("ready without the login in the loaded list: %+v", e.logins.keys["g14"])
}
e.loadedListHeld("sha256:l2", map[string]string{"node.g14": "$2a$04$adopted", "node.g14--b": second.Hash})
e.advanceRotations()
if said(e.bus, NodeAccountReadyEvent) != 1 || e.logins.keys["g14"].Stage != StageReady {
t.Fatalf("not ready with the login in the loaded list: %+v", e.logins.keys["g14"])
}
// S49/AC1: the new login's value is g14's engine's alone, sealed to the key it stated.
stated, _, _ := e.Store.NodeKey("g14")
statedPEM, _ := publicPEM(stated)
if a, err := e.Get("node.g14", "own/g14/mesh-host/node.g14--b", statedPEM); err != nil || a.Sealed.Ct == "" {
t.Errorf("g14's engine was answered %+v, %v", a, err)
}
if _, err := e.Get("node.ace", "own/g14/mesh-host/node.g14--b", statedPEM); err == nil || !strings.Contains(err.Error(), "engine") {
t.Errorf("ace's engine asked for g14's login: %v", err)
}
if _, err := e.Get("node.g14", "own/g14/mesh-host/node.g14--b", pemOf(t, mustPub(t))); err == nil {
t.Error("g14's login was sealed to a key its engine did not state")
}
if own, err := e.OwnOfNode("node.ace"); err != nil || len(own.Own) != 0 {
t.Errorf("ace's engine was handed %+v, %v", own, err)
}
// S47/AC1: a confirm from the old login, from another node's login or from a person retires nothing.
for _, caller := range []string{"node.g14", "node.ace--b", "person.jochen", "controller"} {
if _, err := e.NodeAccountConfirm(caller); err == nil {
t.Errorf("%q confirmed g14's rotation", caller)
}
}
if _, ok := e.accounts.keys["node.g14"]; !ok || e.logins.keys["g14"].Stage != StageReady {
t.Fatalf("a refused confirm retired something: %+v", e.logins.keys["g14"])
}
// S47/AC2: the proven confirm removes the old hash; done only when the loaded list lacks it.
if c, err := e.NodeAccountConfirm("node.g14--b"); err != nil || c.Stage != StageConfirmed {
t.Fatalf("the confirm answered %+v, %v", c, err)
}
if _, ok := e.accounts.keys["node.g14"]; ok || said(e.bus, RotationDoneEvent) != 0 {
t.Fatal("the old hash remains, or the rotation was done while the loaded list still held it")
}
e.loadedListHeld("sha256:l3", map[string]string{"node.g14--b": second.Hash})
e.advanceRotations()
if nl := e.logins.keys["g14"]; said(e.bus, RotationDoneEvent) != 1 || nl.Login != "node.g14--b" || nl.Rotating != "" || nl.Stage != "" {
t.Fatalf("after the loaded list lacks the old login g14 is %+v", nl)
}
if _, held, _ := e.Store.Read("own/g14/mesh-host/node.g14"); held {
t.Error("the old login's value remains after the rotation is done")
}
if _, err := e.AbandonRotation("person.jochen", "g14"); err == nil {
t.Error("a finished rotation was abandoned")
}
// Blocker 5 (ADR 0583 §2): the act is recorded in g14's key and warrants no second rotation.
if acts := e.logins.keys["g14"].Acts; !slices.Equal(acts, []string{"a1"}) {
t.Errorf("g14's key records the acts %v", acts)
}
if _, err := ask("g14", "a1"); err == nil || !strings.Contains(err.Error(), "already") {
t.Errorf("a used hand act warranted another rotation: %v", err)
}
// The queue moves on, the bus node after every other: ace first; novox only once ace is done.
e.RotationPass()
if e.logins.keys["ace"].Rotating == "" || e.logins.keys["novox"].Rotating != "" {
t.Fatalf("after g14 the queue began ace=%+v novox=%+v", e.logins.keys["ace"], e.logins.keys["novox"])
}
e.RotationPass()
if e.logins.keys["novox"].Rotating != "" {
t.Fatal("the bus node began while ace still rotated")
}
// S48/AC1: an abandon drops the new login and its value, keeps the old hash and clears the rotating part.
if _, err := e.AbandonRotation("person.jochen", "ace"); err != nil {
t.Fatal(err)
}
if _, ok := e.accounts.keys["node.ace--b"]; ok || e.accounts.keys["node.ace"].Rotating != nil || e.accounts.keys["node.ace"].Hash == "" ||
e.logins.keys["ace"].Rotating != "" || e.logins.keys["ace"].Login != "node.ace" {
t.Errorf("an abandon left ace as %+v / %+v", e.accounts.keys["node.ace"], e.logins.keys["ace"])
}
if _, held, _ := e.Store.Read("own/ace/mesh-host/node.ace--b"); held {
t.Error("an abandoned login's value remains")
}
e.RotationPass()
if e.logins.keys["novox"].Rotating == "" {
t.Fatal("the bus node did not begin once every other was done")
}
}
// said counts the events of a kind the vault said.
func said(p *published, event string) int {
p.mu.Lock()
defer p.mu.Unlock()
n := 0
for _, e := range p.events {
if e["event"] == event {
n++
}
}
return n
}
// 774/S48/AC2 (novox/hq ADR 0583 §1 insight): the key of a node that leaves the placement is deleted, heard from the
// placement's watch or found by the minute pass; one that cannot be read deletes nothing.
func TestANodesLoginsKeyGoesWhenItLeavesThePlacement(t *testing.T) {
e := newEnrolling(t)
for _, n := range []string{"g14", "gone", "gone2"} {
e.logins.keys[n] = NodeLogin{Node: n, Login: "node." + n}
}
e.NodeLeft("g14")
if _, ok := e.logins.keys["g14"]; !ok {
t.Error("g14 is placed, and its key was deleted")
}
e.NodeLeft("gone")
e.LoginsPass()
if len(e.logins.keys) != 1 || e.logins.keys["g14"].Node == "" {
t.Errorf("the keys after a node left and a pass: %+v", e.logins.keys)
}
e.logins.keys["gone"] = NodeLogin{Node: "gone"}
e.Placement = PlacementReader{}
e.LoginsPass()
if _, ok := e.logins.keys["gone"]; !ok {
t.Error("a placement that is not read deleted a key")
}
}
@@ -0,0 +1,185 @@
package main
// The tools of enrolment and node rotation (novox/hq ADR 0587, ADR 0583): the seat's verbs the vault serves, and the
// tools that read what it holds of them. Arguments are strings and booleans the runtime hands; the caller is the one the
// runtime proved.
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// EnrolmentHeld is one enrolment token the vault holds, as secret_holders lists it: never the token or its hash.
type EnrolmentHeld struct {
Node string `json:"node"`
User string `json:"user"`
Generation int `json:"generation"`
Fingerprint string `json:"fingerprint"`
At time.Time `json:"at"`
// Account says the node's own account is made too (node-account answered).
Account bool `json:"account_made"`
}
// EnrolmentsHeld lists the enrolment tokens held, by node.
func (v *Vault) EnrolmentsHeld() ([]EnrolmentHeld, error) {
out := []EnrolmentHeld{}
if v.Accounts == nil {
return out, nil
}
users, err := v.Accounts.Users()
if err != nil {
return out, err
}
for _, u := range users {
a, err := v.Accounts.Get(u)
if err != nil || a == nil || a.Kept != KeptEnrolment {
continue
}
node := u[len(EnrolUserPrefix):]
n, _ := v.Accounts.Get(NodeEnginePrefix + node)
out = append(out, EnrolmentHeld{Node: node, User: u, Generation: a.Generation, Fingerprint: a.Fingerprint, At: a.At,
Account: n != nil && n.Kept == KeptNode})
}
sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
return out, nil
}
func enrolmentTools(env *Env) []toolDef {
node := str("the node's name")
return []toolDef{
{Name: Seat + ".enrolment-token",
Description: "The enrolment token of a joining machine, for the seat mesh-enrolment alone (novox/hq ADR 0587 §3, §5, " +
"§8). Issue (the default): takes the machine's first tunnel, makes the one-time secret, keeps both with the operator " +
"copy, writes the secret's hash as the bus user enrol.<node>, and answers the bus's address, its certificate's " +
"fingerprint and the vault's signing key, never the token. sealed: the whole token sealed to the operator key, " +
"for the operator to open at their terminal. drop: the token and its hash gone, at every end of an enrolment.",
Input: map[string]any{"node": node, "expires": str("when the token ends, as the seat states it"),
"tunnel": map[string]any{"type": "object", "description": "the machine's first tunnel at issue: hub_key, " +
"hub_endpoint, address (and key, range)"},
"sealed": map[string]any{"type": "boolean", "description": "answer the token sealed to the operator key"},
"drop": map[string]any{"type": "boolean", "description": "drop the token and its hash"}},
run: func(caller string, a map[string]any) (any, error) {
v, err := env.ready()
if err != nil {
return nil, err
}
tunnel, err := tunnelArg(a["tunnel"])
if err != nil {
return nil, err
}
return v.EnrolmentToken(caller, EnrolmentTokenArgs{Node: arg(a, "node"), Expires: arg(a, "expires"),
Tunnel: tunnel, Sealed: isTrue(a["sealed"]), Drop: isTrue(a["drop"])})
}},
{Name: Seat + ".node-account",
Description: "A joining node's account node.<node>, for the seat mesh-enrolment alone at the redeem (novox/hq ADR 0587 " +
"§3): made once for a node and the keys it presented, answered sealed to the sealing key presented, with the " +
"node's key in node-logins naming the login in use. Asked again for the same keys it answers the same account. " +
"retire: the account, its value and its key taken away at a revoke.",
Input: map[string]any{"node": node, "sealing_key": str("the machine's X25519 sealing key, raw in base64 or SPKI PEM"),
"keys_digest": str("the digest of the keys the machine presented"),
"retire": map[string]any{"type": "boolean", "description": "take the account away"}},
run: func(caller string, a map[string]any) (any, error) {
v, err := env.ready()
if err != nil {
return nil, err
}
return v.NodeAccount(caller, NodeAccountArgs{Node: arg(a, "node"), SealingKey: arg(a, "sealing_key"),
KeysDigest: arg(a, "keys_digest"), Retire: isTrue(a["retire"])})
}},
{Name: Seat + ".rotate-node",
Description: "Begin the rotation of a node's login (novox/hq ADR 0583 §2, §7): answered only for a call that carries a hand " +
"act recorded through mesh-hand-acts, read back by its id. One rotation at a time mesh-wide: a call asked while " +
"another runs is queued and begun when it is done, the bus node after every other. Makes the other login " +
"(node.<n>--b or node.<n>), keeps its value for the node's engine, and says rotation-begun.",
Input: map[string]any{"node": node, "why": str("why the login rotates"),
"act": str("the id of the hand act recorded through mesh-hand-acts that warrants it")},
run: func(caller string, a map[string]any) (any, error) {
v, err := env.ready()
if err != nil {
return nil, err
}
return v.RotateNode(caller, RotateNodeArgs{Node: arg(a, "node"), Why: arg(a, "why"), Act: arg(a, "act")})
}},
{Name: Seat + ".node-account-confirm",
Description: "A node-engine, dialled as its new login, says it connects with it (novox/hq ADR 0583 §3, §4): the runtime " +
"proves the login, the old login's hash is removed, and rotation-done follows when the loaded list lacks it.",
run: func(caller string, a map[string]any) (any, error) {
v, err := env.ready()
if err != nil {
return nil, err
}
return v.NodeAccountConfirm(caller)
}},
{Name: Seat + ".abandon",
Description: "Abandon a node's rotation before its confirmation (novox/hq ADR 0583 §5): the new login and its value are " +
"dropped, the old hash kept, the rotating part of the node's key cleared; a rotation only asked leaves the queue.",
Input: map[string]any{"node": node},
run: func(caller string, a map[string]any) (any, error) {
v, err := env.ready()
if err != nil {
return nil, err
}
return v.AbandonRotation(caller, arg(a, "node"))
}},
{Name: "node_logins",
Description: "Every node's key of the vault's state node-logins: the login its node-engine dials as and, while a rotation " +
"runs, the other login and its stage; with the rotations queued and the enrolment tokens held. Never a value or a hash.",
run: func(string, map[string]any) (any, error) {
v, err := env.ready()
if err != nil {
return nil, err
}
if v.Logins == nil {
return nil, errors.New("the vault has no state node-logins")
}
keys, err := v.Logins.Keys()
if err != nil {
return nil, err
}
sort.Strings(keys)
rows := []NodeLogin{}
for _, k := range keys {
if nl, err := v.Logins.Get(k); err == nil && nl != nil {
rows = append(rows, *nl)
}
}
held, _ := v.EnrolmentsHeld()
return map[string]any{"logins": rows, "rotation": v.RotationState(), "enrolments": held}, nil
}},
}
}
// tunnelArg reads enrolment-token's `tunnel`, an object or its JSON as a string, as mesh-sdk's TokenTunnel; nil when
// absent.
func tunnelArg(v any) (*delivery.TokenTunnel, error) {
var raw []byte
switch t := v.(type) {
case nil:
return nil, nil
case string:
if strings.TrimSpace(t) == "" {
return nil, nil
}
raw = []byte(t)
default:
b, err := json.Marshal(t)
if err != nil {
return nil, err
}
raw = b
}
var tt delivery.TokenTunnel
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&tt); err != nil {
return nil, fmt.Errorf("`tunnel` is not a first tunnel (hub_key, hub_endpoint, address, key, range): %w", err)
}
return &tt, nil
}
+48 -1
View File
@@ -60,7 +60,8 @@ func build() *Env {
Assignments: Mirror{Keys: assignmentKeys, Read: readAssignment}, Emit: emit, Ask: stdio.Ask, Assignments: Mirror{Keys: assignmentKeys, Read: readAssignment}, Emit: emit, Ask: stdio.Ask,
Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, LeaseRead: readLease, WantRead: readWant, Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, LeaseRead: readLease, WantRead: readWant,
HaveKeys: func() ([]string, error) { return stdio.State(HaveRead).Keys() }, Placement: placement, Signers: busSigners{}, HaveKeys: func() ([]string, error) { return stdio.State(HaveRead).Keys() }, Placement: placement, Signers: busSigners{},
AskCallers: askCallers(os.Getenv("MESH_VAULT_SECRET_ASK_CALLERS")), AskCallers: askCallers(os.Getenv("MESH_VAULT_SECRET_ASK_CALLERS")), EnrolCallers: askCallers(os.Getenv("MESH_VAULT_ENROLMENT_CALLERS")),
Logins: busNodeLogins{}, ReadAct: readHandAct, TokenBase: TokenBaseFromFile(os.Getenv("MESH_VAULT_STATE")),
Bound: Bound{ Bound: Bound{
Get: func(key string) ([]byte, error) { Get: func(key string) ([]byte, error) {
e, err := stdio.State(BoundState).Get(key) e, err := stdio.State(BoundState).Get(key)
@@ -86,6 +87,46 @@ func build() *Env {
return env return env
} }
// readHandAct reads a hand act back by its id through the seat mesh-hand-acts (the vault's manifest invokes it): nil when
// the log holds none.
func readHandAct(id string) (*HandActSeen, error) {
raw, err := stdio.Ask("mesh-hand-acts.hand-acts", map[string]any{"days": 2})
if err != nil {
return nil, err
}
var listing struct {
Acts []HandActSeen `json:"acts"`
}
if err := json.Unmarshal(raw, &listing); err != nil {
return nil, fmt.Errorf("mesh-hand-acts answered what is not its listing: %w", err)
}
for _, a := range listing.Acts {
if a.ID == id {
return &a, nil
}
}
return nil, nil
}
// watchLeavers deletes a node's key of node-logins when its node key of the placement is deleted (noderotation.go): a
// watch, never a poll. It answers whether the watch began; the minute pass (LoginsPass) catches what it missed.
func watchLeavers(env *Env) bool {
if env.Vault == nil {
return true
}
err := stdio.State(PlacementRead).Watch("*.node", func(c stdio.StateChange) error {
if c.Op == "delete" || c.Op == "purge" || len(c.Value) == 0 {
env.Vault.NodeLeft(strings.TrimSuffix(c.Key, ".node"))
}
return nil
})
if err != nil {
say("the placement's nodes are not watched for one that leaves; the minute pass finds it: %v", err)
return false
}
return true
}
// assigned is the assignments mirror as the vault's watch keeps it. // assigned is the assignments mirror as the vault's watch keeps it.
var assigned *WatchedAssignments var assigned *WatchedAssignments
@@ -280,7 +321,11 @@ func main() {
watched, watching := watchHandings(env), watchAssignments() watched, watching := watchHandings(env), watchAssignments()
lease := watchLease(env) lease := watchLease(env)
wants := watchWants(env) wants := watchWants(env)
leavers := watchLeavers(env)
for { for {
if !leavers {
leavers = watchLeavers(env)
}
if !watched { if !watched {
watched = watchHandings(env) watched = watchHandings(env)
} }
@@ -304,6 +349,8 @@ func main() {
env.Vault.HashOwed() env.Vault.HashOwed()
env.Vault.MintOwed() env.Vault.MintOwed()
env.Vault.RetirePass() env.Vault.RetirePass()
env.Vault.RotationPass()
env.Vault.LoginsPass()
} }
env.pairPass() env.pairPass()
if env.Custody != nil { if env.Custody != nil {
@@ -60,6 +60,8 @@ type player struct {
next int next int
pending map[string]chan wire pending map[string]chan wire
said []string said []string
// ask answers a mesh/ask (a call of another module's tool); nil answers none, as a call with no responder.
ask func(key string, body json.RawMessage) (any, error)
} }
// bucketOf is the bucket a state names: the vault's own by its local name, another's as `<owner>.<name>`. // bucketOf is the bucket a state names: the vault's own by its local name, another's as `<owner>.<name>`.
@@ -215,8 +217,20 @@ func (p *player) handle(m wire) {
p.said = append(p.said, e.Key+" "+string(e.Body)) p.said = append(p.said, e.Key+" "+string(e.Body))
p.mu.Unlock() p.mu.Unlock()
p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod.mesh-vault.event."+e.Key, e.Body)) p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod.mesh-vault.event."+e.Key, e.Body))
case "mesh/ask":
var ask struct {
Key string `json:"key"`
Body json.RawMessage `json:"body"`
}
_ = json.Unmarshal(m.Params, &ask)
if p.ask != nil {
result, err := p.ask(ask.Key, ask.Body)
p.answer(m.ID, result, err)
return
}
p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
default: default:
// mesh/ask and the rest: nothing answers on this bus, as a call with no responder. // the rest: nothing answers on this bus, as a call with no responder.
p.answer(m.ID, nil, errors.New("nats: no responders available for request")) p.answer(m.ID, nil, errors.New("nats: no responders available for request"))
} }
} }
@@ -0,0 +1,681 @@
package main
// A node's two logins and their rotation (novox/hq ADR 0583 §1 to §7 and its insights; issue 774).
//
// A node-engine dials the bus as `node.<n>` or as `node.<n>--b`, the two logins alternating. The vault's state
// `node-logins` holds one key per node, `<n>`: the login in use, and while a rotation runs the other login with its stage
// (begun, ready, confirmed). Each node-engine reads its own key alone (the bus grants it that read); the vault writes it
// alone.
//
// `rotate-node` begins a rotation: answered only for a call that carries a hand act recorded through mesh-hand-acts, read
// back by its id; one rotation at a time mesh-wide, a call asked meanwhile is queued and begun at the pass that finds the
// first done; the bus node last, after every other node asked. It makes the other login's value, keeps it as
// `own/<n>/mesh-host/<login>` (sealed, with the operator copy; the node-engine takes it through `own`, sealed to its own
// key), writes its hash as the key `<login>` of bus-accounts and marks the login in use as rotating, so the grants part
// composes both. `node-account-ready` is said only when the user list the bus node loaded holds the new login's hash.
// `node-account-confirm` is the node-engine's, dialled as the new login, which the runtime proves: the old login's hash
// is then removed, and `rotation-done` is said once the loaded list lacks it, when the old value is dropped and the
// rotating part of the key cleared. `abandon` drops the new login and its value before the confirmation and keeps the
// old hash. A key whose node leaves the placement is deleted.
import (
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"golang.org/x/crypto/bcrypt"
)
// The events a rotation says and the stages of a node's key are mesh-sdk's (delivery/node_logins.go); a rotation warranted
// by a hand act is warranted for a day.
const (
RotationBegunEvent = delivery.RotationBegunEvent
NodeAccountReadyEvent = delivery.NodeAccountReadyEvent
RotationDoneEvent = delivery.RotationDoneEvent
StageBegun = delivery.RotationBegun
StageReady = delivery.RotationReady
StageConfirmed = delivery.RotationConfirmed
// HandActWithin is how recent a hand act is that warrants a rotation.
HandActWithin = 24 * time.Hour
)
// NodeLogin is one key of the vault's state node-logins (mesh-sdk's contract): the login the node's node-engine dials as,
// and while a rotation runs the other login with its stage. Never a value or a hash.
type NodeLogin = delivery.NodeLogin
// NodeLoginStore is the vault's state `node-logins`, by node.
type NodeLoginStore interface {
Get(node string) (*NodeLogin, error)
Put(n NodeLogin) error
Delete(node string) error
Keys() ([]string, error)
}
// busNodeLogins is NodeLoginStore over the vault's own state through its runtime.
type busNodeLogins struct{}
func (busNodeLogins) Get(node string) (*NodeLogin, error) {
e, err := stdio.State(NodeLoginsState).Get(node)
if err != nil || e == nil {
return nil, err
}
n, err := delivery.DecodeNodeLogin(e.Value)
if err != nil {
return nil, fmt.Errorf("%s %s is not a node's login: %w", NodeLoginsState, node, err)
}
return &n, nil
}
func (busNodeLogins) Put(n NodeLogin) error {
raw, err := delivery.EncodeNodeLogin(n)
if err != nil {
return err
}
_, err = stdio.State(NodeLoginsState).Put(n.Node, json.RawMessage(raw))
return err
}
func (busNodeLogins) Delete(node string) error { return stdio.State(NodeLoginsState).Delete(node) }
func (busNodeLogins) Keys() ([]string, error) { return stdio.State(NodeLoginsState).Keys() }
// HandActSeen is a hand act as the vault reads it back from mesh-hand-acts by its id (mesh-sdk's handacts.Act): who
// recorded it, when, and what it was (its verb and arguments).
type HandActSeen struct {
ID string `json:"id"`
By string `json:"by"`
At time.Time `json:"at"`
Verb string `json:"verb"`
Args []string `json:"arguments,omitempty"`
}
// ActReader reads a hand act back by its id: nil when mesh-hand-acts holds none.
type ActReader func(id string) (*HandActSeen, error)
// queuedRotation is a `rotate-node` asked while another ran, begun at the pass that finds none running.
type queuedRotation struct {
Node string `json:"node"`
By string `json:"by"`
Why string `json:"why"`
Act string `json:"act"`
At time.Time `json:"at"`
}
// nodeRotation is what the vault keeps in memory of its rotations: the user list the bus node loaded (user to hash), and
// the rotations queued. The record is node-logins and bus-accounts; a restart forgets the queue, and the one who asked
// asks again.
type nodeRotation struct {
mu sync.Mutex
loaded map[string]string
loadedFrom string
queue []queuedRotation
}
func (v *Vault) getLogin(node string) (*NodeLogin, error) {
if v.Logins == nil {
return nil, errors.New("the vault has no state " + NodeLoginsState + " to read")
}
return v.Logins.Get(node)
}
// putLogin writes node's key; a key written without its hand acts keeps the ones recorded (ADR 0583 §2).
func (v *Vault) putLogin(n NodeLogin) error {
if v.Logins == nil {
return errors.New("the vault has no state " + NodeLoginsState + " to write")
}
if n.Acts == nil {
was, err := v.Logins.Get(n.Node)
if err != nil {
return err
}
if was != nil {
n.Acts = was.Acts
}
}
return v.Logins.Put(n)
}
func (v *Vault) deleteLogin(node string) error {
if v.Logins == nil {
return nil
}
return v.Logins.Delete(node)
}
// currentLogin is the login node dials as: its key's, or `node.<node>` before any key is written.
func (v *Vault) currentLogin(node string) (string, error) {
nl, err := v.getLogin(node)
if err != nil {
return "", err
}
if nl != nil && nl.Login != "" {
return nl.Login, nil
}
return NodeEnginePrefix + node, nil
}
// otherLogin is the login a node alternates to from login.
func otherLogin(login string) string {
if base, second := strings.CutSuffix(login, SecondLoginSuffix); second {
return base
}
return login + SecondLoginSuffix
}
// loginNode is the node whose login a bus user is: `node.<n>` or `node.<n>--b`.
func loginNode(user string) (string, bool) {
rest, ok := strings.CutPrefix(user, NodeEnginePrefix)
if !ok {
return "", false
}
node := strings.TrimSuffix(rest, SecondLoginSuffix)
return node, node != "" && enrolNode(node) == nil
}
// checkAct reads the hand act back by its id and refuses a rotation it does not warrant: none recorded, recorded by
// another caller, older than a day, or an act that is not rotate-node of this node.
func (v *Vault) checkAct(caller, node, id string) error {
if strings.TrimSpace(id) == "" {
return errors.New("rotate-node carries the hand act that warrants it (`act`): record it through mesh-hand-acts first (novox/hq ADR 0583 §2)")
}
if v.ReadAct == nil {
return errors.New("the vault reads no hand acts, so it answers no rotation (novox/hq ADR 0583 §2)")
}
act, err := v.ReadAct(id)
if err != nil {
return fmt.Errorf("the hand act %s could not be read back from mesh-hand-acts: %w", id, err)
}
switch {
case act == nil:
return fmt.Errorf("mesh-hand-acts holds no hand act %s: record one first (novox/hq ADR 0583 §2)", id)
case act.By != caller:
return fmt.Errorf("the hand act %s was recorded by %s, not by %s who asks: a rotation is warranted by its own caller's act", id, act.By, caller)
case v.Now().Sub(act.At) > HandActWithin:
return fmt.Errorf("the hand act %s is older than %s: record a new one", id, HandActWithin)
case act.Verb != delivery.RotateNodeVerb && !strings.HasSuffix(act.Verb, "."+delivery.RotateNodeVerb):
return fmt.Errorf("the hand act %s is %q, not %s: it warrants no rotation", id, act.Verb, delivery.RotateNodeVerb)
case !slices.Contains(act.Args, node):
return fmt.Errorf("the hand act %s is a rotation of %s, not of %s", id, strings.Join(act.Args, " "), node)
}
return nil
}
// rotatingNodes are the nodes whose key says a rotation runs, sorted.
func (v *Vault) rotatingNodes() ([]NodeLogin, error) {
if v.Logins == nil {
return nil, nil
}
keys, err := v.Logins.Keys()
if err != nil {
return nil, err
}
var out []NodeLogin
for _, k := range keys {
nl, err := v.Logins.Get(k)
if err != nil {
return nil, err
}
if nl != nil && nl.Rotating != "" {
out = append(out, *nl)
}
}
sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
return out, nil
}
// RotateNodeArgs are `rotate-node`'s arguments.
type RotateNodeArgs struct {
Node string
Why string
Act string
}
// RotateNodeAnswer says a rotation began, or that it waits for the ones before it. Never a value.
type RotateNodeAnswer struct {
Node string `json:"node"`
Login string `json:"login,omitempty"`
Rotating string `json:"rotating,omitempty"`
Stage string `json:"stage,omitempty"`
Revision uint64 `json:"list_revision,omitempty"`
Queued bool `json:"queued,omitempty"`
// Behind are the nodes whose rotation it waits for, running or asked before.
Behind []string `json:"behind,omitempty"`
}
// RotateNode answers caller's `rotate-node`.
func (v *Vault) RotateNode(caller string, a RotateNodeArgs) (RotateNodeAnswer, error) {
if err := fromAGrantedCaller(caller, "rotate-node"); err != nil {
return RotateNodeAnswer{}, err
}
if err := enrolNode(a.Node); err != nil {
return RotateNodeAnswer{}, err
}
if strings.TrimSpace(a.Why) == "" {
return RotateNodeAnswer{}, errors.New("rotate-node says why (`why`)")
}
if err := v.checkAct(caller, a.Node, a.Act); err != nil {
return RotateNodeAnswer{}, err
}
if v.Accounts == nil {
return RotateNodeAnswer{}, errors.New("the vault has no state of bus accounts to write to")
}
v.leaseAfresh()
if on, why := v.leaseHanded(); !on {
return RotateNodeAnswer{}, fmt.Errorf("no rotation begins: %s", why)
}
v.minting.mu.Lock()
revision := v.minting.loadedRevision
v.minting.mu.Unlock()
if revision == 0 {
return RotateNodeAnswer{}, errors.New("no rotation begins: the revision of the user list the bus loaded is not known yet")
}
bus, err := v.busNode()
if err != nil {
return RotateNodeAnswer{}, fmt.Errorf("no rotation is ordered: the node holding the bus is not known (the bus node rotates last): %w", err)
}
v.nr.mu.Lock()
defer v.nr.mu.Unlock()
if nl, err := v.getLogin(a.Node); err != nil {
return RotateNodeAnswer{}, err
} else if nl != nil && nl.Rotating != "" {
return RotateNodeAnswer{}, fmt.Errorf("%s is rotating already, since %s (second login %s)", a.Node, nl.Since.Format(time.RFC3339), nl.Rotating)
}
for _, q := range v.nr.queue {
if q.Node == a.Node {
return RotateNodeAnswer{}, fmt.Errorf("%s's rotation is asked already, by %s, and waits its turn", a.Node, q.By)
}
}
if err := v.recordAct(a.Node, a.Act); err != nil {
return RotateNodeAnswer{}, err
}
running, err := v.rotatingNodes()
if err != nil {
return RotateNodeAnswer{}, err
}
var behind []string
for _, r := range running {
behind = append(behind, r.Node)
}
if a.Node == bus {
for _, q := range v.nr.queue {
behind = append(behind, q.Node)
}
}
if len(behind) > 0 {
v.nr.queue = append(v.nr.queue, queuedRotation{Node: a.Node, By: caller, Why: a.Why, Act: a.Act, At: v.Now().UTC()})
v.log("the rotation of %s waits for %s (asked by %s): one rotation at a time, the bus node %s last", a.Node,
strings.Join(behind, ", "), caller, bus)
return RotateNodeAnswer{Node: a.Node, Queued: true, Behind: behind}, nil
}
nl, err := v.beginNodeRotation(caller, a.Node, a.Why, a.Act, revision)
if err != nil {
return RotateNodeAnswer{}, err
}
return RotateNodeAnswer{Node: a.Node, Login: nl.Login, Rotating: nl.Rotating, Stage: nl.Stage, Revision: revision}, nil
}
// recordAct keeps the hand act in node's key of node-logins, refusing one already used: an act warrants one rotation
// (ADR 0583 §2). Under nr.mu.
func (v *Vault) recordAct(node, act string) error {
if v.Logins == nil {
return errors.New("the vault has no state " + NodeLoginsState + " to record the rotation in")
}
nl, err := v.getLogin(node)
if err != nil {
return err
}
if nl != nil && slices.Contains(nl.Acts, act) {
return fmt.Errorf("the hand act %s warranted a rotation of %s already: record a new one", act, node)
}
if nl == nil {
cur, err := v.currentLogin(node)
if err != nil {
return err
}
nl = &NodeLogin{Node: node, Login: cur, Since: v.Now().UTC()}
}
nl.Acts = append(slices.Clone(nl.Acts), act)
if err := v.Logins.Put(*nl); err != nil {
return fmt.Errorf("the hand act %s could not be recorded in %s %s: %w", act, NodeLoginsState, node, err)
}
return nil
}
// beginNodeRotation makes the other login and records the rotation; under nr.mu.
func (v *Vault) beginNodeRotation(by, node, why, act string, revision uint64) (NodeLogin, error) {
cur, err := v.currentLogin(node)
if err != nil {
return NodeLogin{}, err
}
curKey, err := v.Accounts.Get(cur)
if err != nil {
return NodeLogin{}, fmt.Errorf("%s %s could not be read: %w", BusAccountsState, cur, err)
}
if curKey == nil {
return NodeLogin{}, fmt.Errorf("%s holds no key of %s, the login %s dials as: there is nothing to rotate", BusAccountsState, cur, node)
}
second := otherLogin(cur)
value, password, err := v.accountValue(node, second)
if err != nil {
return NodeLogin{}, err
}
secret := loginSecret(node, second)
c, err := v.keep(secret, value, "the vault", "", OriginMade)
if err != nil {
return NodeLogin{}, fmt.Errorf("the login %s is not kept: %w", second, err)
}
hash, err := bcrypt.GenerateFromPassword([]byte(password), v.hashCost())
if err != nil {
return NodeLogin{}, err
}
at := v.Now().UTC()
fp := Fingerprint(value)
if err := v.Accounts.Put(AccountHash{User: second, Hash: string(hash), Of: secret, Generation: c.Generation,
Fingerprint: fp, At: at, Kept: KeptNode}); err != nil {
return NodeLogin{}, fmt.Errorf("the hash of %s could not be written to %s: %w", second, BusAccountsState, err)
}
v.saidHashed(second, secret, c.Generation, fp, at)
marked := *curKey
marked.Rotating = &Rotating{Login: second, Generation: c.Generation, Since: at, Revision: revision}
if err := v.Accounts.Put(marked); err != nil {
return NodeLogin{}, fmt.Errorf("%s could not be marked rotating: %w", cur, err)
}
nl := NodeLogin{Node: node, Login: cur, Rotating: second, Stage: StageBegun, Since: at, ListRevision: revision}
if err := v.putLogin(nl); err != nil {
return NodeLogin{}, fmt.Errorf("%s %s could not be written: %w", NodeLoginsState, node, err)
}
v.sayRotation(RotationBegunEvent, nl, map[string]any{"why": why, "act": act, "by": by})
v.log("the rotation of %s begins at the loaded list's revision %d: %s alternates to %s (asked by %s, hand act %s)", node,
revision, cur, second, by, act)
return nl, nil
}
// sayRotation says an event of a rotation: the node, its logins, the stage; never a value or a hash.
func (v *Vault) sayRotation(event string, nl NodeLogin, extra map[string]any) {
if v.Emit == nil {
return
}
body := map[string]any{"node": nl.Node, "login": nl.Login, "rotating": nl.Rotating, "stage": nl.Stage,
"at": v.Now().UTC().Format(time.RFC3339Nano)}
for k, x := range extra {
body[k] = x
}
if err := v.Emit(event, body); err != nil {
v.log("the rotation of %s moved, but %s could not be said: %v", nl.Node, event, err)
}
}
// loadedListHeld keeps the users the bus node's loaded list holds, with their hashes (what its handing opened to).
func (v *Vault) loadedListHeld(from string, users map[string]string) {
v.nr.mu.Lock()
v.nr.loaded, v.nr.loadedFrom = users, from
v.nr.mu.Unlock()
}
// loadedHolds says the loaded list holds user with exactly hash; loadedLacks that it is known and holds no user.
func (v *Vault) loadedHolds(user, hash string) bool {
v.nr.mu.Lock()
defer v.nr.mu.Unlock()
return v.nr.loaded != nil && hash != "" && v.nr.loaded[user] == hash
}
func (v *Vault) loadedLacks(user string) bool {
v.nr.mu.Lock()
defer v.nr.mu.Unlock()
if len(v.nr.loaded) == 0 {
return false
}
_, held := v.nr.loaded[user]
return !held
}
// advanceRotations moves every rotation as far as the loaded list lets it: begun to ready when the list holds the new
// login, confirmed to done when it lacks the old one. At each adoption of the loaded list and at the minute pass.
func (v *Vault) advanceRotations() {
running, err := v.rotatingNodes()
if err != nil {
v.log("the rotations could not be read: %v", err)
return
}
for _, nl := range running {
switch nl.Stage {
case StageBegun:
k, err := v.Accounts.Get(nl.Rotating)
if err != nil || k == nil || !v.loadedHolds(nl.Rotating, k.Hash) {
continue
}
nl.Stage = StageReady
if err := v.putLogin(nl); err != nil {
v.log("%s is ready, and %s could not be written: %v", nl.Rotating, NodeLoginsState, err)
continue
}
v.sayRotation(NodeAccountReadyEvent, nl, nil)
v.log("the loaded user list holds %s: %s may dial as it", nl.Rotating, nl.Node)
case StageConfirmed:
if !v.loadedLacks(nl.Rotating) {
continue
}
v.finishRotation(nl)
}
}
}
// finishRotation ends a confirmed rotation whose old hash the loaded list lacks: the old value dropped, the rotating part
// of the key cleared, rotation-done said.
func (v *Vault) finishRotation(nl NodeLogin) {
old := nl.Rotating
if err := v.Store.Delete(loginSecret(nl.Node, old)); err != nil {
v.log("the old login %s of %s is out of the list, and its value could not be dropped: %v", old, nl.Node, err)
return
}
done := NodeLogin{Node: nl.Node, Login: nl.Login, Since: v.Now().UTC()}
if err := v.putLogin(done); err != nil {
v.log("the rotation of %s is done, and %s could not be written: %v", nl.Node, NodeLoginsState, err)
return
}
v.sayRotation(RotationDoneEvent, done, map[string]any{"dropped": old})
v.log("the rotation of %s is done: it dials as %s, and %s is gone from the list and the vault", nl.Node, nl.Login, old)
}
// ConfirmAnswer is what `node-account-confirm` answers.
type ConfirmAnswer struct {
Node string `json:"node"`
Login string `json:"login"`
Stage string `json:"stage"`
}
// NodeAccountConfirm answers `node-account-confirm`: the node-engine, dialled as the new login (which the runtime
// proves), says it connects with it. The old login's hash is then removed from bus-accounts; rotation-done follows when
// the loaded list lacks it.
func (v *Vault) NodeAccountConfirm(caller string) (ConfirmAnswer, error) {
if err := fromAGrantedCaller(caller, "node-account-confirm"); err != nil {
return ConfirmAnswer{}, err
}
node, ok := loginNode(caller)
if !ok {
return ConfirmAnswer{}, fmt.Errorf("%s is no node-engine's login: nothing is retired", caller)
}
nl, err := v.getLogin(node)
if err != nil {
return ConfirmAnswer{}, err
}
if nl == nil || nl.Rotating != caller || (nl.Stage != StageBegun && nl.Stage != StageReady) {
return ConfirmAnswer{}, fmt.Errorf("%s is not the new login of a rotation of %s under way: nothing is retired", caller, node)
}
old := nl.Login
if err := v.Accounts.Delete(old); err != nil {
return ConfirmAnswer{}, fmt.Errorf("the hash of %s could not be taken out of %s: %w", old, BusAccountsState, err)
}
confirmed := NodeLogin{Node: node, Login: caller, Rotating: old, Stage: StageConfirmed, Since: v.Now().UTC(), ListRevision: nl.ListRevision}
if err := v.putLogin(confirmed); err != nil {
return ConfirmAnswer{}, fmt.Errorf("the old hash of %s is out, and %s %s could not be written: %w", node, NodeLoginsState, node, err)
}
v.log("%s confirmed %s: the hash of %s is removed from %s", node, caller, old, BusAccountsState)
v.advanceRotations()
return ConfirmAnswer{Node: node, Login: caller, Stage: StageConfirmed}, nil
}
// AbandonAnswer says what `abandon` took away.
type AbandonAnswer struct {
Node string `json:"node"`
Login string `json:"login,omitempty"`
Dropped string `json:"dropped,omitempty"`
Queued bool `json:"was_queued,omitempty"`
}
// AbandonRotation answers `abandon`: the new login and its value are dropped, the old hash kept and the rotating part
// cleared; a rotation only asked is taken off the queue. Not after the confirmation, when the old hash is already gone.
func (v *Vault) AbandonRotation(caller, node string) (AbandonAnswer, error) {
if err := fromAGrantedCaller(caller, "abandon"); err != nil {
return AbandonAnswer{}, err
}
if err := enrolNode(node); err != nil {
return AbandonAnswer{}, err
}
v.nr.mu.Lock()
defer v.nr.mu.Unlock()
out := AbandonAnswer{Node: node}
if i := slices.IndexFunc(v.nr.queue, func(q queuedRotation) bool { return q.Node == node }); i >= 0 {
v.nr.queue = slices.Delete(v.nr.queue, i, i+1)
out.Queued = true
}
nl, err := v.getLogin(node)
if err != nil {
return out, err
}
if nl == nil || nl.Rotating == "" {
if out.Queued {
return out, nil
}
return out, fmt.Errorf("%s rotates nothing: nothing was abandoned", node)
}
if nl.Stage == StageConfirmed {
return out, fmt.Errorf("the rotation of %s is confirmed: the old login %s is out of the list, and %s is the login in use; rotate again to change it",
node, nl.Rotating, nl.Login)
}
second := nl.Rotating
if err := v.Accounts.Delete(second); err != nil {
return out, fmt.Errorf("the hash of %s could not be taken out of %s: %w", second, BusAccountsState, err)
}
if err := v.Store.Delete(loginSecret(node, second)); err != nil {
return out, fmt.Errorf("the value of %s could not be dropped: %w", second, err)
}
if k, err := v.Accounts.Get(nl.Login); err == nil && k != nil && k.Rotating != nil {
cleared := *k
cleared.Rotating = nil
if err := v.Accounts.Put(cleared); err != nil {
return out, fmt.Errorf("%s could not be marked as not rotating: %w", nl.Login, err)
}
}
if err := v.putLogin(NodeLogin{Node: node, Login: nl.Login, Since: v.Now().UTC()}); err != nil {
return out, fmt.Errorf("%s %s could not be written: %w", NodeLoginsState, node, err)
}
_ = v.recordHanding(Handing{To: caller, Name: loginSecret(node, second), Outcome: "dropped", Why: "rotation of " + node + " abandoned"})
v.log("the rotation of %s is abandoned by %s: %s and its value are dropped, %s keeps its hash", node, caller, second, nl.Login)
out.Login, out.Dropped = nl.Login, second
return out, nil
}
// RotationPass is the minute pass of rotations: each moves as far as the loaded list lets it, and the next queued one
// begins when none runs (the bus node after every other).
func (v *Vault) RotationPass() {
if v.Logins == nil || v.Accounts == nil {
return
}
v.advanceRotations()
running, err := v.rotatingNodes()
if err != nil || len(running) > 0 {
return
}
v.leaseAfresh()
if on, _ := v.leaseHanded(); !on {
return
}
bus, err := v.busNode()
if err != nil {
return
}
v.minting.mu.Lock()
revision := v.minting.loadedRevision
v.minting.mu.Unlock()
if revision == 0 {
return
}
v.nr.mu.Lock()
defer v.nr.mu.Unlock()
next := slices.IndexFunc(v.nr.queue, func(q queuedRotation) bool { return q.Node != bus })
if next < 0 {
next = slices.IndexFunc(v.nr.queue, func(q queuedRotation) bool { return q.Node == bus })
}
if next < 0 {
return
}
q := v.nr.queue[next]
v.nr.queue = slices.Delete(v.nr.queue, next, next+1)
if _, err := v.beginNodeRotation(q.By, q.Node, q.Why, q.Act, revision); err != nil {
v.log("the queued rotation of %s did not begin: %v", q.Node, err)
}
}
// RotationState is what vault_status says of rotations: the keys of node-logins that rotate, and the queue.
type RotationState struct {
Rotating []NodeLogin `json:"rotating"`
Queued []queuedRotation `json:"queued"`
}
// RotationState answers how the rotations stand.
func (v *Vault) RotationState() RotationState {
running, _ := v.rotatingNodes()
v.nr.mu.Lock()
defer v.nr.mu.Unlock()
return RotationState{Rotating: append([]NodeLogin{}, running...), Queued: append([]queuedRotation{}, v.nr.queue...)}
}
// LoginsPass deletes the key of every node that left the placement (ADR 0583 §1 insight). A placement that cannot be
// read, or holds no node, deletes nothing.
func (v *Vault) LoginsPass() {
if v.Logins == nil || v.Placement.Keys == nil {
return
}
placed, err := v.placements()
if err != nil || len(placed) == 0 {
return
}
keys, err := v.Logins.Keys()
if err != nil {
return
}
for _, k := range keys {
if _, in := placed[k]; !in {
v.nodeLeft(k)
}
}
}
// NodeLeft is heard when a node's key of the placement is deleted: its key of node-logins goes with it.
func (v *Vault) NodeLeft(node string) {
if v.Logins == nil || v.Placement.Read == nil {
return
}
if _, _, found, err := v.nodePlaced(node); err != nil || found {
return
}
v.nodeLeft(node)
}
func (v *Vault) nodeLeft(node string) {
if err := v.Logins.Delete(node); err != nil {
v.log("%s left the placement, and its key of %s could not be deleted: %v", node, NodeLoginsState, err)
return
}
v.log("%s left the placement: its key of %s is deleted", node, NodeLoginsState)
}
+4 -1
View File
@@ -118,7 +118,7 @@ func (v *Vault) goneAccounts() ([]string, int, error) {
if err != nil { if err != nil {
return nil, 0, err return nil, 0, err
} }
if a == nil || a.Adopted || a.Of == "" { if a == nil || a.Adopted || a.Of == "" || a.Kept != "" {
continue continue
} }
held++ held++
@@ -207,6 +207,9 @@ func (v *Vault) retireOne(user, why, by string) error {
if a == nil || a.Adopted || a.Of == "" { if a == nil || a.Adopted || a.Of == "" {
return fmt.Errorf("the vault holds no account of %s it made or was handed: an adopted hash is never retired here", user) return fmt.Errorf("the vault holds no account of %s it made or was handed: an adopted hash is never retired here", user)
} }
if a.Kept != "" {
return fmt.Errorf("%s is kept for an enrolment or a node's login, which ends with its own verb (enrolment-token, node-account, node-account-confirm, abandon): it is never retired as unwanted", user)
}
rec, held, err := v.Store.Read(a.Of) rec, held, err := v.Store.Read(a.Of)
if err != nil { if err != nil {
return err return err
@@ -0,0 +1,242 @@
//go:build integration
package main
// The vault whole against a real bus, rotating a node's login with a stand-in engine (novox/hq ADR 0583 §1 to §7, issue
// 774): the built binary played over its stdin and stdout as the node's tool runtime plays it, as in the mint test. The
// test is the engine and the bus node: it states its sealing key, takes the new login through `own`, and "reloads the
// list" by handing the bus-accounts hashes in the bus module's have, as the bus node's node-engine does. At every step at
// least one of the engine's logins authenticates against the list loaded last, and with the list never reloaded the vault
// never says node-account-ready.
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"golang.org/x/crypto/bcrypt"
"golang.org/x/crypto/nacl/box"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/seal"
"git.novox.be/novox/mesh-sdk/go/testbus"
)
// 774/S50/AC1.
func TestTheVaultWholeRotatesANodesLoginWithAStandInEngine(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
dir := t.TempDir()
bin := filepath.Join(dir, "mesh-vault")
build := exec.Command("go", "build", "-o", bin, ".")
build.Stderr = os.Stderr
if err := build.Run(); err != nil {
t.Fatalf("the vault does not build: %v", err)
}
nc, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
for _, b := range []string{"mesh-vault_bus-accounts", "mesh-vault_node-logins", "mesh-grants_accounts", "mesh-controller_lease",
"mesh-provisioner_plans", "node-engine_have", "mesh-delivery-assigner_placement"} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 2}); err != nil {
t.Fatal(err)
}
}
opPub, _, err := box.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
root := filepath.Join(dir, "root")
_ = os.MkdirAll(root, 0o700)
export, _ := json.Marshal(map[string]string{"operator-key": base64.StdEncoding.EncodeToString(opPub[:])})
if err := os.WriteFile(filepath.Join(root, "export.json"), export, 0o600); err != nil {
t.Fatal(err)
}
cmd := exec.CommandContext(ctx, bin)
cmd.Env = append(os.Environ(), "MESH_VAULT_STORE="+filepath.Join(dir, "store"), "MESH_VAULT_STATE="+filepath.Join(dir, "state"),
"MESH_VAULT_ROOT="+root, "MESH_VAULT_LEDGER="+filepath.Join(dir, "ledger"), "MESH_RECEIVES="+filepath.Join(dir, "mesh.json"),
"MESH_NODE=novox", "MESH_MODULE=mesh-vault")
stdin, _ := cmd.StdinPipe()
stdout, _ := cmd.StdoutPipe()
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() })
p := &player{t: t, ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}}
// mesh-hand-acts answers the one hand act the test recorded.
p.ask = func(key string, _ json.RawMessage) (any, error) {
if key != "mesh-hand-acts.hand-acts" {
return nil, errors.New("nats: no responders available for request")
}
return map[string]any{"acts": []HandActSeen{{ID: "act-1", By: "person.jochen", At: time.Now(), Verb: "rotate-node", Args: []string{"g14"}}}}, nil
}
go p.serve(stdout)
// The bus node (novox, running nats) and the node that rotates (g14); the lease handed away; g14's login adopted from
// the loaded list, its password known to the engine.
for node, modules := range map[string][]string{"novox": {"nats"}, "g14": {"web"}} {
pl := delivery.Placement{V: delivery.Version, Node: node}
for _, m := range modules {
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
}
values, err := delivery.PlacementValues(pl)
if err != nil {
t.Fatal(err)
}
for k, raw := range values {
kv, _ := p.kv("mesh-delivery-assigner.placement")
if _, err := kv.Put(ctx, k, raw); err != nil {
t.Fatal(err)
}
}
}
p.put(t, "mesh-controller.lease", LeaseKey, map[string]any{"handed-to": "the operator's hand-over"})
oldPassword := "the-old-password-of-g14"
oldHash, _ := bcrypt.GenerateFromPassword([]byte(oldPassword), bcrypt.MinCost)
p.put(t, "mesh-vault.bus-accounts", "node.g14", AccountHash{User: "node.g14", Hash: string(oldHash), Adopted: true})
// The vault's public key, which the bus node's engine seals its handing to.
var vaultKey seal.KeyPair
eventually(t, "the vault's key", func() bool {
raw, err := os.ReadFile(filepath.Join(dir, "state", OwnKeyFile))
return err == nil && json.Unmarshal(raw, &vaultKey) == nil && vaultKey.PublicKey != ""
})
// The bus node reloads the list from bus-accounts and hands the hashes of what it loaded in its have; the engine's logins
// are its passwords. An authentication is a bcrypt compare against the list loaded last.
passwords := map[string]string{"node.g14": oldPassword}
var loaded map[string]string
revision := uint64(0)
reload := func() {
kv, _ := p.kv("mesh-vault.bus-accounts")
keys, err := kv.Keys(ctx)
if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) {
t.Fatal(err)
}
sort.Strings(keys)
var accounts []delivery.HandedAccount
loaded = map[string]string{}
for _, k := range keys {
a := p.hashOf(t, k)
if a == nil || !delivery.IsBcrypt(a.Hash) {
continue
}
accounts = append(accounts, delivery.HandedAccount{User: k, Hash: a.Hash})
loaded[k] = a.Hash
}
revision++
list, _ := json.Marshal(accounts)
handed, err := delivery.HandAccounts(accounts, vaultKey.PublicKey, Fingerprint(string(list)), time.Now())
if err != nil {
t.Fatal(err)
}
raw, err := delivery.EncodeHave(delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK,
UserList: &delivery.UserListWritten{Source: "k", Revision: revision, Users: len(accounts), Handed: &handed}})
if err != nil {
t.Fatal(err)
}
hk, _ := p.kv("node-engine.have")
if _, err := hk.Put(ctx, "novox.nats", raw); err != nil {
t.Fatal(err)
}
}
authenticates := func(step string) {
t.Helper()
for user, password := range passwords {
if h, ok := loaded[user]; ok && bcrypt.CompareHashAndPassword([]byte(h), []byte(password)) == nil {
return
}
}
t.Fatalf("%s: none of the engine's logins %v authenticates against the loaded list %v", step, passwords, loaded)
}
said := func(event string) bool {
p.mu.Lock()
defer p.mu.Unlock()
return strings.Contains(strings.Join(p.said, "\n"), event+" ")
}
reload()
authenticates("before the rotation")
enginePub, enginePriv := aSealingKey(t)
if _, err := p.call(Seat+".node-key", "node.g14", map[string]any{"key": enginePub}); err != nil {
t.Fatal(err)
}
// The rotation begins once the vault has heard the loaded list's revision.
var begun RotateNodeAnswer
eventually(t, "the rotation begins", func() bool {
raw, err := p.call(Seat+".rotate-node", "person.jochen", map[string]any{"node": "g14", "why": "the weekly rotation", "act": "act-1"})
return err == nil && json.Unmarshal(raw, &begun) == nil && begun.Rotating == "node.g14--b"
})
authenticates("rotation begun")
// The list is never reloaded: node-account-ready is never said.
time.Sleep(time.Second)
if said(NodeAccountReadyEvent) {
t.Fatal("node-account-ready was said while the loaded list lacks the new login")
}
// The engine takes the new login through own, sealed to its key.
raw, err := p.call(Seat+".own", "node.g14", map[string]any{})
if err != nil {
t.Fatal(err)
}
var own OwnAnswer
if err := json.Unmarshal(raw, &own); err != nil || len(own.Own) != 1 || own.Own[0].Module != MeshHostModule || own.Own[0].Name != "node.g14--b" {
t.Fatalf("own answers %+v, %v", own.Own, err)
}
value, err := seal.Open(own.Own[0].Sealed, enginePriv)
if err != nil {
t.Fatal(err)
}
var account struct{ User, Password string }
if err := json.Unmarshal([]byte(value), &account); err != nil || account.User != "node.g14--b" {
t.Fatalf("the new login is %s, %v", value, err)
}
passwords["node.g14--b"] = account.Password
// The bus node reloads the list: the new login is in it, both authenticate, node-account-ready is said.
reload()
eventually(t, "node-account-ready", func() bool { return said(NodeAccountReadyEvent) })
authenticates("the list holds both logins")
if bcrypt.CompareHashAndPassword([]byte(loaded["node.g14--b"]), []byte(account.Password)) != nil {
t.Fatal("the loaded hash of the new login is not of the value the engine took")
}
// The engine, dialled as the new login, confirms; the vault removes the old hash, and the next reload drops it.
if _, err := p.call(Seat+".node-account-confirm", "node.g14--b", map[string]any{}); err != nil {
t.Fatal(err)
}
if h := p.hashOf(t, "node.g14"); h != nil {
t.Fatalf("the old login's hash remains after the confirm: %+v", h)
}
authenticates("the confirm, before the list reloads")
reload()
authenticates("the list without the old login")
eventually(t, "rotation-done", func() bool { return said(RotationDoneEvent) })
kv, _ := p.kv("mesh-vault.node-logins")
e, err := kv.Get(ctx, "g14")
if err != nil {
t.Fatal(err)
}
var nl NodeLogin
_ = json.Unmarshal(e.Value(), &nl)
if nl.Login != "node.g14--b" || nl.Rotating != "" || nl.Stage != "" {
t.Errorf("node-logins g14 is %+v", nl)
}
if h := p.hashOf(t, "node.g14--b"); h == nil || h.Rotating != nil {
t.Errorf("the new login's key is %+v", h)
}
}
+103 -2
View File
@@ -240,6 +240,15 @@ type SignArgs struct {
Kind string Kind string
Node string Node string
Reply string Reply string
// Keys are, for a joining node's certificate asked by mesh-enrolment, the public keys the machine presented at its
// redeem (delivery.Redeem's): the serving key is certified, the four are checked against the digest node-account
// recorded.
Keys PresentedKeys
}
// PresentedKeys are the public keys a joining machine presented at its redeem, as delivery.Redeem names them.
type PresentedKeys struct {
Public, Overlay, Sealing, Serving string
} }
// Sign answers caller's `sign`. The caller is the one the runtime proved; who may ask which kind is the vault's rule, // Sign answers caller's `sign`. The caller is the one the runtime proved; who may ask which kind is the vault's rule,
@@ -251,16 +260,98 @@ func (v *Vault) Sign(caller string, a SignArgs) (delivery.SignAnswer, error) {
} }
switch a.Kind { switch a.Kind {
case delivery.KindNodeCertificate: case delivery.KindNodeCertificate:
if v.fromEnrolment(caller, "sign") == nil {
return v.signRedeemingCertificate(caller, a.Node, a.Keys)
}
return v.signNodeCertificate(caller, a.Node) return v.signNodeCertificate(caller, a.Node)
case delivery.KindEnrolment: case delivery.KindEnrolment:
return delivery.SignAnswer{}, fmt.Errorf("%s may not ask for an enrolment's signature: no caller is named for it until "+ if err := v.fromEnrolment(caller, "sign an enrolment"); err != nil {
"adoption's record names the seat that adopts (novox/hq ADR 0561 §5)", caller) return delivery.SignAnswer{}, fmt.Errorf("%s may not ask for an enrolment's signature: only the seat %s, which "+
"decides who joins, is named for it (novox/hq ADR 0561 §5, ADR 0587 §6)", caller, EnrolmentModule)
}
return v.signEnrolment(caller, a.Node, a.Reply)
case delivery.KindAuthority: case delivery.KindAuthority:
return delivery.SignAnswer{}, fmt.Errorf("the authority is the vault's own, signed for itself; %s may not ask it", caller) return delivery.SignAnswer{}, fmt.Errorf("the authority is the vault's own, signed for itself; %s may not ask it", caller)
} }
return delivery.SignAnswer{}, fmt.Errorf("%q is not a kind the vault signs (%s)", a.Kind, strings.Join(delivery.SignKinds, ", ")) return delivery.SignAnswer{}, fmt.Errorf("%q is not a kind the vault signs (%s)", a.Kind, strings.Join(delivery.SignKinds, ", "))
} }
// redeeming says node has an enrolment token held: a machine joining, which the seat mesh-enrolment redeems.
func (v *Vault) redeeming(node string) error {
if err := enrolNode(node); err != nil {
return err
}
if _, held, err := v.Store.Read(enrolSecret(node)); err != nil {
return err
} else if !held {
return fmt.Errorf("%s holds no enrolment token: it is not a machine joining, so nothing is signed for it", node)
}
return nil
}
// signRedeemingCertificate signs the certificate of a machine that is joining, for the seat mesh-enrolment at the redeem
// (ADR 0587 §6). The machine has written no record yet, so the serving key is the one it presented at the redeem, which
// counts only when the four keys presented have the digest node-account recorded for its account; the name is its
// placement's, which mesh-enrolment writes before it asks.
func (v *Vault) signRedeemingCertificate(caller, node string, keys PresentedKeys) (delivery.SignAnswer, error) {
if err := v.redeeming(node); err != nil {
return delivery.SignAnswer{}, err
}
if v.Accounts == nil {
return delivery.SignAnswer{}, errors.New("the vault has no state of bus accounts, so it cannot read the keys a joining node presented")
}
acct, err := v.Accounts.Get(NodeEnginePrefix + node)
if err != nil {
return delivery.SignAnswer{}, fmt.Errorf("%s %s could not be read: %w", BusAccountsState, NodeEnginePrefix+node, err)
}
if acct == nil || acct.Kept != KeptNode || acct.KeysDigest == "" {
return delivery.SignAnswer{}, fmt.Errorf("%s has no account made at its redeem (node-account), so no keys are recorded to certify", node)
}
if keys.Serving == "" || keys.Public == "" || keys.Sealing == "" {
return delivery.SignAnswer{}, errors.New("a joining node's certificate is asked with the keys it presented at the redeem " +
"(`public_key`, `overlay_key`, `sealing_key`, `serving_key`), which are checked against node-account's digest")
}
if got := delivery.KeysDigest(keys.Public, keys.Overlay, keys.Sealing, keys.Serving); got != acct.KeysDigest {
return delivery.SignAnswer{}, fmt.Errorf("the keys presented for %s are not the ones its account was made for: nothing is signed", node)
}
pub, err := base64.StdEncoding.DecodeString(keys.Serving)
if err != nil || len(pub) != ed25519.PublicKeySize {
return delivery.SignAnswer{}, fmt.Errorf("%s's serving key is not an ed25519 public key in base64", node)
}
name, err := v.internalName(node)
if err != nil {
return delivery.SignAnswer{}, err
}
return v.signCertificateWith(caller, node, name, ed25519.PublicKey(pub))
}
// signEnrolment signs the bytes of an enrolment reply under the enrolment context, for a node that is joining; the
// signature is the vault's key's, and the answer carries it with the signer's fingerprint.
func (v *Vault) signEnrolment(caller, node, reply string) (delivery.SignAnswer, error) {
if err := v.redeeming(node); err != nil {
return delivery.SignAnswer{}, err
}
msg, err := base64.StdEncoding.DecodeString(reply)
if err != nil || len(msg) == 0 {
return delivery.SignAnswer{}, errors.New("an enrolment's signature is asked for the reply's bytes in base64 (`reply`)")
}
context, err := delivery.SignContext(delivery.KindEnrolment)
if err != nil {
return delivery.SignAnswer{}, err
}
k, err := v.keys()
if err != nil {
return delivery.SignAnswer{}, err
}
answer := delivery.SignAnswer{Kind: delivery.KindEnrolment, Node: node,
Signature: base64.StdEncoding.EncodeToString(delivery.SignMessage(k.key, context, msg)),
Signer: delivery.SignerFingerprint(k.key.Public().(ed25519.PublicKey)), Digest: delivery.DigestOf(msg)}
if err := v.saySigned(caller, SigningKeySecret, answer); err != nil {
return delivery.SignAnswer{}, err
}
return answer, nil
}
// signNodeCertificate signs the certificate of the calling machine's internal name for the serving key its record states. // signNodeCertificate signs the certificate of the calling machine's internal name for the serving key its record states.
func (v *Vault) signNodeCertificate(caller, asked string) (delivery.SignAnswer, error) { func (v *Vault) signNodeCertificate(caller, asked string) (delivery.SignAnswer, error) {
node, ok := nodeEngineOf(caller) node, ok := nodeEngineOf(caller)
@@ -271,6 +362,11 @@ func (v *Vault) signNodeCertificate(caller, asked string) (delivery.SignAnswer,
if asked != "" && asked != node { if asked != "" && asked != node {
return delivery.SignAnswer{}, fmt.Errorf("%s asks for the certificate of %s, and is %s's node-engine: a machine asks for its own alone", caller, asked, node) return delivery.SignAnswer{}, fmt.Errorf("%s asks for the certificate of %s, and is %s's node-engine: a machine asks for its own alone", caller, asked, node)
} }
return v.signCertificateFor(caller, node)
}
// signCertificateFor signs node's certificate from its placement's name and its record's serving key.
func (v *Vault) signCertificateFor(caller, node string) (delivery.SignAnswer, error) {
name, err := v.internalName(node) name, err := v.internalName(node)
if err != nil { if err != nil {
return delivery.SignAnswer{}, err return delivery.SignAnswer{}, err
@@ -279,6 +375,11 @@ func (v *Vault) signNodeCertificate(caller, asked string) (delivery.SignAnswer,
if err != nil { if err != nil {
return delivery.SignAnswer{}, err return delivery.SignAnswer{}, err
} }
return v.signCertificateWith(caller, node, name, serving)
}
// signCertificateWith signs the certificate of name for serving, for node.
func (v *Vault) signCertificateWith(caller, node, name string, serving ed25519.PublicKey) (delivery.SignAnswer, error) {
k, err := v.keys() k, err := v.keys()
if err != nil { if err != nil {
return delivery.SignAnswer{}, err return delivery.SignAnswer{}, err
@@ -157,6 +157,19 @@ func (s *Store) Write(r Record) error {
return writeWhole(s.pathOf(r.Name), append(raw, '\n')) return writeWhole(s.pathOf(r.Name), append(raw, '\n'))
} }
// Delete takes one secret out of the store: for the enrolment token and a joining node's account, which end with
// their enrolment (novox/hq ADR 0587 §5), never for a value the mesh keeps. Nothing held is no error.
func (s *Store) Delete(name string) error {
if err := validName(name); err != nil {
return err
}
path := s.pathOf(name)
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
return syncDir(filepath.Dir(path))
}
// Empty says whether the store holds no secret at all. // Empty says whether the store holds no secret at all.
func (s *Store) Empty() (bool, error) { func (s *Store) Empty() (bool, error) {
all, err := s.List() all, err := s.List()
+19 -8
View File
@@ -75,7 +75,7 @@ func str(description string) map[string]any {
} }
func definitions(env *Env) []toolDef { func definitions(env *Env) []toolDef {
return []toolDef{ return append(enrolmentTools(env), []toolDef{
{Name: Seat + ".get", {Name: Seat + ".get",
Description: "One secret, sealed to the public key the asker sends (the sealed box of ADR 0206), with its " + Description: "One secret, sealed to the public key the asker sends (the sealed box of ADR 0206), with its " +
"generation; answered only to an asker the vault allows, as the runtime proved it (ADR 0349 §11). " + "generation; answered only to an asker the vault allows, as the runtime proved it (ADR 0349 §11). " +
@@ -395,8 +395,13 @@ func definitions(env *Env) []toolDef {
"granted, how many times it has been rotated and when, and the fingerprint of the current value. " + "granted, how many times it has been rotated and when, and the fingerprint of the current value. " +
"Fingerprints only — the value is never returned.", "Fingerprints only — the value is never returned.",
run: func(string, map[string]any) (any, error) { run: func(string, map[string]any) (any, error) {
enrolments := []EnrolmentHeld{}
if env.Vault != nil {
enrolments, _ = env.Vault.EnrolmentsHeld()
}
if env.Ledger == nil { if env.Ledger == nil {
return map[string]any{"holders": []any{}, "count": 0, "error": "the vault has no ledger (MESH_VAULT_LEDGER is not set)"}, nil return map[string]any{"holders": []any{}, "count": 0, "enrolments": enrolments,
"error": "the vault has no ledger (MESH_VAULT_LEDGER is not set)"}, nil
} }
var asked []Contribution var asked []Contribution
if env.Receives != "" { if env.Receives != "" {
@@ -415,7 +420,7 @@ func definitions(env *Env) []toolDef {
} }
holders = append(holders, row) holders = append(holders, row)
} }
return map[string]any{"holders": holders, "count": len(holders)}, nil return map[string]any{"holders": holders, "count": len(holders), "enrolments": enrolments}, nil
}}, }},
{Name: "secret_verify", {Name: "secret_verify",
Description: "Check one holder's secret without seeing it: the fingerprint the vault recorded against the " + Description: "Check one holder's secret without seeing it: the fingerprint the vault recorded against the " +
@@ -581,14 +586,20 @@ func definitions(env *Env) []toolDef {
"adoption's record (ADR 0561 §5). Answers the certificate, the authority, the signer's fingerprint and the " + "adoption's record (ADR 0561 §5). Answers the certificate, the authority, the signer's fingerprint and the " +
"digest, and says signed. Never a key.", "digest, and says signed. Never a key.",
Input: map[string]any{"kind": str("node-certificate or enrolment"), Input: map[string]any{"kind": str("node-certificate or enrolment"),
"node": str("the machine, for a node certificate: the caller's own (left out, the caller's)"), "node": str("the machine, for a node certificate: the caller's own (left out, the caller's)"),
"reply": str("the enrolment reply's bytes in base64, for an enrolment")}, "reply": str("the enrolment reply's bytes in base64, for an enrolment"),
"public_key": str("a joining node's identity key, as presented at its redeem (mesh-enrolment alone)"),
"overlay_key": str("a joining node's overlay key, as presented at its redeem"),
"sealing_key": str("a joining node's sealing key, as presented at its redeem"),
"serving_key": str("a joining node's serving key to certify, as presented at its redeem")},
run: func(caller string, a map[string]any) (any, error) { run: func(caller string, a map[string]any) (any, error) {
v, err := env.ready() v, err := env.ready()
if err != nil { if err != nil {
return nil, err return nil, err
} }
return v.Sign(caller, SignArgs{Kind: arg(a, "kind"), Node: arg(a, "node"), Reply: arg(a, "reply")}) return v.Sign(caller, SignArgs{Kind: arg(a, "kind"), Node: arg(a, "node"), Reply: arg(a, "reply"),
Keys: PresentedKeys{Public: arg(a, "public_key"), Overlay: arg(a, "overlay_key"), Sealing: arg(a, "sealing_key"),
Serving: arg(a, "serving_key")}})
}}, }},
{Name: "vault_signers", {Name: "vault_signers",
Description: "The vault's signing key and authority (novox/hq ADR 0550 §2): whether each is held with its operator " + Description: "The vault's signing key and authority (novox/hq ADR 0550 §2): whether each is held with its operator " +
@@ -647,9 +658,9 @@ func definitions(env *Env) []toolDef {
key, _ := env.Vault.Store.OperatorKey() key, _ := env.Vault.Store.OperatorKey()
return map[string]any{"ready": true, "operator_key": key, "copies_on_an_earlier_key": left, return map[string]any{"ready": true, "operator_key": key, "copies_on_an_earlier_key": left,
"operator_key_refused": env.keyRefused(), "handed_rolled": env.Vault.Store.Rolled(), "operator_key_refused": env.keyRefused(), "handed_rolled": env.Vault.Store.Rolled(),
"minting": env.Vault.MintState(), "retiring": env.Vault.RetireState(), "signing": env.Vault.SigningState(), "ok": left == 0 && env.keyRefused() == "" && key != ""}, nil "minting": env.Vault.MintState(), "retiring": env.Vault.RetireState(), "signing": env.Vault.SigningState(), "ok": left == 0 && env.keyRefused() == "" && key != "", "rotation": env.Vault.RotationState()}, nil
}}, }},
} }...)
} }
// actInput is the arguments of an act on retirement, as the SDK's harness takes them. // actInput is the arguments of an act on retirement, as the SDK's harness takes them.
+30 -3
View File
@@ -76,6 +76,17 @@ type Vault struct {
// Signers is the vault's state `signers`, where it publishes its signing halves (signing.go, ADR 0550 §2): nil writes // Signers is the vault's state `signers`, where it publishes its signing halves (signing.go, ADR 0550 §2): nil writes
// none. // none.
Signers SignersStore Signers SignersStore
// EnrolCallers are the callers of the enrolment verbs `enrolment-token` and `node-account` and of `sign` for an
// enrolment: the seat mesh-enrolment (enrolment.go). Nil: that module alone.
EnrolCallers stdio.Callers
// TokenBase says where the bus is and its certificate's fingerprint, for the token shown (enrolment.go): nil shows none.
TokenBase func() (TokenBase, error)
// Logins is the vault's state node-logins (noderotation.go): nil writes none.
Logins NodeLoginStore
// ReadAct reads a hand act back from mesh-hand-acts by its id, which a rotation carries (noderotation.go): nil answers none.
ReadAct ActReader
// nr is what the vault keeps in memory of its node rotations (noderotation.go).
nr nodeRotation
// signingWhy is why the last signing pass did not finish, under mu (signing.go). // signingWhy is why the last signing pass did not finish, under mu (signing.go).
signingWhy string signingWhy string
// callsMu guards pending: the keys of the calls open now (intake.go), never written anywhere. // callsMu guards pending: the keys of the calls open now (intake.go), never written anywhere.
@@ -341,14 +352,30 @@ func (v *Vault) Get(caller, name, publicKey string) (Answer, error) {
if signingSecret(name) { if signingSecret(name) {
return Answer{}, fmt.Errorf("%s is the vault's own signing secret and is handed to nobody: ask the verb sign", name) return Answer{}, fmt.Errorf("%s is the vault's own signing secret and is handed to nobody: ask the verb sign", name)
} }
// A value issued for one holder is that holder's alone, whatever else would allow it (issue 749). // An enrolment token is shown to the operator sealed to the operator key, and handed to nobody (ADR 0587 §8).
if why := notTheHolder(caller, name); why != "" { if enrolSecretName(name) {
return Answer{}, fmt.Errorf("%s is an enrolment token and is handed to nobody: the operator opens it sealed to the operator key", name)
}
// A node's login is its node-engine's alone, sealed to the key that machine stated (ADR 0583 §3 insight).
isLogin, why := v.loginRefusal(caller, name)
if isLogin {
if why == "" {
why = v.loginKeyRefusal(caller, publicKey)
}
if why != "" {
v.log("refused %s to %s: %s", name, caller, why)
_ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "refused", Why: why})
return Answer{}, fmt.Errorf("%s may not ask for %s: %s", caller, name, why)
}
} else if why := notTheHolder(caller, name); why != "" {
v.log("refused %s to %s: %s", name, caller, why) v.log("refused %s to %s: %s", name, caller, why)
_ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "refused", Why: why}) _ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "refused", Why: why})
return Answer{}, fmt.Errorf("%s may not ask for %s: %s", caller, name, why) return Answer{}, fmt.Errorf("%s may not ask for %s: %s", caller, name, why)
} }
// The core's own credentials, to the controller alone, sealed only to their machine's key (ADR 0349 §16). // The core's own credentials, to the controller alone, sealed only to their machine's key (ADR 0349 §16).
if caller == ControllerCaller { if isLogin {
// Judged above: the node's own engine, sealed to its own key.
} else if caller == ControllerCaller {
if why := v.coreRefusal(name, publicKey); why != "" { if why := v.coreRefusal(name, publicKey); why != "" {
v.log("refused %s to %s: %s", name, caller, why) v.log("refused %s to %s: %s", name, caller, why)
_ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "refused", Why: why}) _ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "refused", Why: why})
@@ -146,10 +146,10 @@ func TestNoBucketHoldsASecret(t *testing.T) {
if err := json.Unmarshal(raw, &manifest); err != nil { if err := json.Unmarshal(raw, &manifest); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if s, _ := json.Marshal(manifest["state"]); string(s) != `[{"history":2,"name":"bus-accounts"},"signers","bound"]` { if s, _ := json.Marshal(manifest["state"]); string(s) != `[{"history":2,"name":"bus-accounts"},"signers","bound","node-logins"]` {
t.Errorf("the manifest declares buckets %s: a secret is in no bucket, sealed or not (ADR 0201 §13), and the "+ t.Errorf("the manifest declares buckets %s: a secret is in no bucket, sealed or not (ADR 0201 §13), and the "+
"buckets are the accounts' hashes (ADR 0547 clause 9), the signing keys' public halves (ADR 0550 §2) and the "+ "buckets are the accounts' hashes (ADR 0547 clause 9), the signing keys' public halves (ADR 0550 §2) and the "+
"credentials' outcomes, names and generations alone (ADR 0570 clauses 3, 6)", s) "credentials' outcomes, names and generations alone (ADR 0570 clauses 3, 6) and each node's login in use (ADR 0583)", s)
} }
if r, _ := json.Marshal(manifest["reads"]); string(r) != `["node-engine.have","mesh-grants.accounts","mesh-controller.lease","mesh-delivery-assigner.placement","mesh-provisioner.plans","mesh-provisioner.provisions"]` { if r, _ := json.Marshal(manifest["reads"]); string(r) != `["node-engine.have","mesh-grants.accounts","mesh-controller.lease","mesh-delivery-assigner.placement","mesh-provisioner.plans","mesh-provisioner.provisions"]` {
t.Errorf("the manifest reads %s: the vault reads the seat mesh-provisioner's plans and provisions, the haves, the "+ t.Errorf("the manifest reads %s: the vault reads the seat mesh-provisioner's plans and provisions, the haves, the "+
@@ -173,6 +173,13 @@ func TestNoBucketHoldsASecret(t *testing.T) {
code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Keys(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Keys(", "", 1))
code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Delete(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Delete(", "", 1))
} }
// Each node's login in use and its rotation's stage, in noderotation.go alone: one read, one write, its keys and a
// delete of its own bucket (ADR 0583 §1 insight); never a secret.
if f == "noderotation.go" {
for _, call := range []string{"Get(", "Put(", "Delete(", "Keys("} {
code = []byte(strings.Replace(string(code), "stdio.State(NodeLoginsState)."+call, "", -1))
}
}
// The signing keys' public halves, in signing.go alone: one read and one write of its own bucket (ADR 0550 §2). // The signing keys' public halves, in signing.go alone: one read and one write of its own bucket (ADR 0550 §2).
if f == "signing.go" { if f == "signing.go" {
code = []byte(strings.Replace(string(code), "stdio.State(delivery.SignersState).Get(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(delivery.SignersState).Get(", "", 1))
@@ -194,6 +201,8 @@ func TestNoBucketHoldsASecret(t *testing.T) {
// And the seat's placement, where modules stand (placed.go): its keys and one key, read. // And the seat's placement, where modules stand (placed.go): its keys and one key, read.
code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Keys(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Keys(", "", 1))
code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Get(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Get(", "", 1))
// And one watch of its nodes' keys, for a node that leaves: its node-logins key goes with it (noderotation.go).
code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Watch(", "", 1))
// And its outcome entries (novox/hq ADR 0570 clauses 3, 6): a value's name and generation, never a value; and // And its outcome entries (novox/hq ADR 0570 clauses 3, 6): a value's name and generation, never a value; and
// one read of the seat's provisions, which hold none. // one read of the seat's provisions, which hold none.
for _, call := range []string{"Get(", "Put(", "Delete(", "Keys("} { for _, call := range []string{"Get(", "Put(", "Delete(", "Keys("} {
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-vault
go 1.26.0 go 1.26.0
require ( require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21
github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0 github.com/nats-io/nats.go v1.54.0
golang.org/x/crypto v0.57.0 golang.org/x/crypto v0.57.0
+2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915 h1:x8k7IeMirMTxUo8BH+xUO+MDnHYgU+636jl1NIw1wvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022206-8af6e7266915/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+19 -5
View File
@@ -21,7 +21,13 @@
"account-hashed", "account-hashed",
"own-replaced", "own-replaced",
"account-issued", "account-issued",
"signed" "signed",
"rotation-begun",
"node-account-ready",
"rotation-done"
],
"invokes": [
"seat:mesh-hand-acts.hand-acts"
], ],
"reads": [ "reads": [
"node-engine.have", "node-engine.have",
@@ -37,7 +43,8 @@
"history": 2 "history": 2
}, },
"signers", "signers",
"bound" "bound",
"node-logins"
], ],
"receives": { "receives": {
"secret": "${dir:grants}/mesh.json" "secret": "${dir:grants}/mesh.json"
@@ -156,7 +163,8 @@
"MESH_VAULT_ROOT": "${dir:root}", "MESH_VAULT_ROOT": "${dir:root}",
"MESH_VAULT_STORE": "${dir:store}", "MESH_VAULT_STORE": "${dir:store}",
"MESH_VAULT_STATE": "${dir:state}", "MESH_VAULT_STATE": "${dir:state}",
"MESH_VAULT_SECRET_ASK_CALLERS": "node-tools" "MESH_VAULT_SECRET_ASK_CALLERS": "node-tools",
"MESH_VAULT_ENROLMENT_CALLERS": "mesh-enrolment"
} }
} }
] ]
@@ -182,7 +190,8 @@
"own_take_switch", "own_take_switch",
"vault_signers", "vault_signers",
"vault_signed", "vault_signed",
"signers_reassert" "signers_reassert",
"node_logins"
], ],
"claims": [ "claims": [
{ {
@@ -199,7 +208,12 @@
"move-asks", "move-asks",
"own", "own",
"issue-account", "issue-account",
"sign" "sign",
"enrolment-token",
"node-account",
"rotate-node",
"node-account-confirm",
"abandon"
] ]
} }
] ]