diff --git a/cmd/mesh-control/main.go b/cmd/mesh-control/main.go index abc8f49..dbfbbfd 100644 --- a/cmd/mesh-control/main.go +++ b/cmd/mesh-control/main.go @@ -1005,6 +1005,22 @@ func planFor(ctx context.Context, inv *inventory.Inventory, nodeName string) (ca return catalogue.Resolution{}, nil, err } + // The credential for each thing this node takes from elsewhere. Made once and kept, so the + // password a provider is told to create is the one its consumer was given — and sealed to + // this node before it was ever written down, so nothing between here and there can read it. + for i, n := range resolved.Needs { + secret, err := inv.SecretFor(ctx, n.Name, nodeName, n.From) + if err != nil { + // Said rather than skipped. A machine that resolves cleanly and receives no + // credential is one that will fail to authenticate at some later, less obvious + // moment. + return catalogue.Resolution{}, nil, fmt.Errorf( + "%s needs %s from %s and no credential could be made for it: %w", + nodeName, n.Name, n.From, err) + } + resolved.Needs[i].Sealed = secret.ForConsumer + } + // Settings for everything that resolved, including modules nobody assigned directly: a // requirement pulled in by something else is still configurable, and finding out that it is // not only when you try would be an arbitrary line nobody could predict. @@ -1179,6 +1195,51 @@ func whereEveryoneIs(ctx context.Context, inv *inventory.Inventory, return out, nil } +// declarationFor is everything a node would be sent. +// +// One place, because there were three and one of them was written before credentials existed and +// silently produced a declaration missing them — a difference between what `plan` showed and what +// `plan --json` handed to anything reading it. +func declarationFor(ctx context.Context, inv *inventory.Inventory, node string, + plan catalogue.Resolution, settings catalogue.SettingsBy) ([]map[string]any, error) { + gens, err := generators(ctx, inv) + if err != nil { + return nil, err + } + return declarationWith(ctx, inv, node, plan, settings, gens) +} + +// declarationWith is the same, for a caller that has already worked out the generators once and +// is about to use them for every node. +func declarationWith(ctx context.Context, inv *inventory.Inventory, node string, + plan catalogue.Resolution, settings catalogue.SettingsBy, + gens map[string]catalogue.Generator) ([]map[string]any, error) { + grants, err := grantsFor(ctx, inv, node) + if err != nil { + return nil, err + } + return plan.Declaration( + catalogue.Rendering{Settings: settings, Generators: gens, Grants: grants}) +} + +// grantsFor is every credential this node must create, because something elsewhere uses it. +// +// The mirror of what a consumer is given, and the half that makes the credential real: a password +// nothing was told to create is a password that authenticates nowhere. Sealed to this node, so +// the mesh hands over something it cannot itself use. +func grantsFor(ctx context.Context, inv *inventory.Inventory, node string) ([]catalogue.Grant, error) { + issued, err := inv.SecretsFrom(ctx, node) + if err != nil { + return nil, err + } + out := make([]catalogue.Grant, 0, len(issued)) + for _, s := range issued { + out = append(out, catalogue.Grant{ + Provision: s.Name, Consumer: s.Consumer, Sealed: s.ForProvider}) + } + return out, nil +} + func planCommand(ctx context.Context, args []string) error { set := flag.NewFlagSet("plan", flag.ContinueOnError) // Because "one resource" does not tell you whether the settings landed. Being able to read @@ -1211,12 +1272,7 @@ func planCommand(ctx context.Context, args []string) error { return nil } if *asJSON { - gens, err := generators(ctx, inv) - if err != nil { - return err - } - resources, err := plan.Declaration( - catalogue.Rendering{Settings: settings, Generators: gens}) + resources, err := declarationFor(ctx, inv, args[0], plan, settings) if err != nil { return err } @@ -1242,11 +1298,7 @@ func planCommand(ctx context.Context, args []string) error { for _, n := range plan.Needs { fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For) } - gens, err := generators(ctx, inv) - if err != nil { - return err - } - resources, err := plan.Declaration(catalogue.Rendering{Settings: settings, Generators: gens}) + resources, err := declarationFor(ctx, inv, args[0], plan, settings) if err != nil { return err } @@ -1331,7 +1383,7 @@ func pushCommand(ctx context.Context, args []string) error { // The private network is in here with everything else. It used to be composed separately // and prepended, which meant every machine with an address was on it and no machine could // be kept off. It is a module now, so it arrives the way a module does. - resources, err := plan.Declaration(catalogue.Rendering{Settings: settings, Generators: gens}) + resources, err := declarationWith(ctx, inv, n.Name, plan, settings, gens) if err != nil { refusals = append(refusals, fmt.Sprintf("%s:\n%v", n.Name, err)) continue diff --git a/go.mod b/go.mod index c8188a4..2ff6a81 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,8 @@ require ( github.com/jackc/pgx/v5 v5.10.0 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/rabbitmq/amqp091-go v1.14.0 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/text v0.29.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect ) diff --git a/go.sum b/go.sum index 6806b42..c927634 100644 --- a/go.sum +++ b/go.sum @@ -13,9 +13,17 @@ github.com/rabbitmq/amqp091-go v1.14.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMu github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index b7b6e91..ccba53a 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -178,8 +178,33 @@ type Manifest struct { // and knows nothing about provisions, which is what keeps this from needing anything new // down there. Binds map[string]string `json:"binds,omitempty"` + + // Secrets is where this module wants the credential for something it requires, per + // requirement. The file holds the value and nothing else, so a program can read it without + // parsing anything. + // + // **Its own file, separate from Binds, because the mesh cannot compose a document containing + // it.** The value was sealed to this node when it was made and the plaintext discarded — so + // there is nothing to interpolate into a larger file, and that is the property worth keeping + // rather than an inconvenience to work around. It also means the readable half stays readable + // in the declaration, and the secret half changes only when the secret does, which is what + // makes `restart-on` precise. + Secrets map[string]string `json:"secrets,omitempty"` + + // Grants is a directory this module wants the credentials of its consumers written into, per + // provision it offers — one file per consumer, named for it, holding the value alone. + // + // A directory rather than one document for the same reason as above: each value is sealed + // separately and the mesh cannot open any of them to build a list. + Grants map[string]string `json:"grants,omitempty"` } +// SecretID is the resource identity of the file a module is given a credential in. +func SecretID(requirement string) string { return "secret-" + requirement } + +// GrantID is the resource identity of one consumer's credential on the providing machine. +func GrantID(provision, consumer string) string { return "grant-" + provision + "-" + consumer } + // BoundID is the resource identity of the file a module is told about a provision in. func BoundID(requirement string) string { return "bound-" + requirement } @@ -312,6 +337,39 @@ func ParseManifest(raw []byte) (Manifest, error) { "%s binds %q and does not require it", m.Module, to)) } } + for to, where := range m.Secrets { + if !strings.HasPrefix(where, "/") { + problems = append(problems, fmt.Sprintf( + "%s keeps the credential for %q at %q, which is not an absolute path", + m.Module, to, where)) + } + var wanted bool + for _, w := range m.Wants() { + if w == to { + wanted = true + } + } + if !wanted { + problems = append(problems, fmt.Sprintf( + "%s wants the credential for %q and does not require it", m.Module, to)) + } + } + for to, where := range m.Grants { + if !strings.HasPrefix(where, "/") { + problems = append(problems, fmt.Sprintf( + "%s grants %q into %q, which is not an absolute path", m.Module, to, where)) + } + var offered bool + for _, o := range m.Offers() { + if o == to { + offered = true + } + } + if !offered { + problems = append(problems, fmt.Sprintf( + "%s grants %q to its consumers and does not provide it", m.Module, to)) + } + } for to, where := range m.Receives { if !name.MatchString(to) { problems = append(problems, fmt.Sprintf("%q is not a usable name to receive", to)) diff --git a/internal/catalogue/resolve.go b/internal/catalogue/resolve.go index ae7a38c..bb0d8a7 100644 --- a/internal/catalogue/resolve.go +++ b/internal/catalogue/resolve.go @@ -98,6 +98,9 @@ type Needed struct { At string // Serves is what the providing module said a consumer needs to know. Serves map[string]any + // Sealed is the credential, closed to this node. Filled in after resolving, because whose + // credential it is only becomes answerable once which node answers has been settled. + Sealed string // For is the module that wanted it. For string } @@ -453,10 +456,24 @@ type Generator interface { Resources(node string) ([]map[string]any, bool, error) } +// Grant is one consumer's credential, on the machine that must create it. +type Grant struct { + // Provision is what was required. + Provision string + // Consumer is the node that will use it, which is also what names the file. + Consumer string + // Sealed is the credential, closed to the providing node. + Sealed string +} + // Rendering is everything needed to turn a resolution into the declaration a node is sent. type Rendering struct { Settings SettingsBy Generators map[string]Generator + // Grants are the credentials this node must create, for the provisions it offers. Passed in + // rather than resolved, because who consumes a node is a fact about the rest of the mesh and + // resolution answers questions about one machine. + Grants []Grant } // Declaration is everything the resolved modules put on the node, with settings applied. @@ -473,6 +490,37 @@ func (r Resolution) Declaration(with Rendering) ([]map[string]any, error) { var out []map[string]any for _, m := range r.Modules { resources := m.Resources + for _, to := range sortedKeys(m.Secrets) { + var found *Needed + for i, n := range r.Needs { + if n.Name == to { + found = &r.Needs[i] + } + } + if found == nil || found.Sealed == "" { + // Answered on this machine, or answered by a node the mesh could not seal to. + // Nothing to write either way, and writing an empty credential file would be + // worse than none: something would read it and fail authenticating. + continue + } + resources = append(append([]map[string]any{}, resources...), map[string]any{ + "id": SecretID(to), "type": "file", "path": m.Secrets[to], + "sealed": found.Sealed, + }) + } + for _, to := range sortedKeys(m.Grants) { + for _, g := range with.Grants { + if g.Provision != to { + continue + } + resources = append(append([]map[string]any{}, resources...), map[string]any{ + "id": GrantID(to, g.Consumer), + "type": "file", + "path": strings.TrimRight(m.Grants[to], "/") + "/" + g.Consumer, + "sealed": g.Sealed, + }) + } + } for _, to := range sortedKeys(m.Binds) { var found *Needed for i, n := range r.Needs { diff --git a/internal/inventory/migrations/0009-secrets.sql b/internal/inventory/migrations/0009-secrets.sql new file mode 100644 index 0000000..93e974b --- /dev/null +++ b/internal/inventory/migrations/0009-secrets.sql @@ -0,0 +1,44 @@ +-- The key a node's secrets are sealed to, and the sealed secrets themselves. +-- +-- The arrangement is the opposite of encrypting a credential column. There, the control plane can +-- read every secret it stores, so a copy of its database is a copy of every credential in the +-- mesh, and encryption at rest only means somebody needs the process rather than the file. Here +-- the value is sealed to the node that will use it before it is written, so **this table holds +-- nothing usable** -- which is what makes novox/hq ADR 0004's "compromise of a node is compromise +-- of that node" true of secrets and not only of identity. +-- +-- It also costs the ability to audit by value, and that is the right trade rather than an +-- oversight: a `where value like ...` over an encrypted column returns zero rows and proves +-- nothing, so the audit was never real. What is answerable here is which node holds what, which +-- is the question rotation actually asks. + +alter table node add column sealing_key text; + +create table secret ( + -- What it is for. The provision as required -- `database` -- not the module answering it. + name text not null, + consumer uuid not null references node(id) on delete cascade, + provider uuid not null references node(id) on delete cascade, + + -- The same value, sealed twice: once to each end. Two blobs rather than one shared key, + -- because a key both ends hold is a key the mesh must also hold to distribute. + -- + -- The plaintext is never written. It exists for the length of one function call, is sealed to + -- both recipients, and is discarded -- so rotation means generating a new one rather than + -- reading the old one back, which is the only version of rotation that is honest about what + -- the mesh knows. + for_consumer text not null, + for_provider text not null, + + -- Which key each was sealed to. A node that regenerates its sealing key can no longer open + -- what was sealed to the old one, and this is what lets that be reported rather than + -- discovered as a service that will not start. + consumer_key text not null, + provider_key text not null, + + created_at timestamptz not null default now(), + + primary key (name, consumer, provider) +); + +create index secret_by_provider on secret (provider); diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index 1d6dbe4..de9a19b 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -322,6 +322,34 @@ type Overlay struct { // closed firewall (novox/hq ADR 0007). func (o Overlay) Reachable() bool { return strings.TrimSpace(o.Endpoint) != "" } +// RecordSealingKey keeps the public half of the key this node's secrets are sealed to. +// +// Replacing whatever was there. A node that rejoins has generated a new one, and everything +// sealed to the old key is unreadable to it -- which is why this does not merge and why what it +// invalidates is reported rather than repaired silently. +func (i *Inventory) RecordSealingKey(ctx context.Context, node, key string) error { + if key == "" { + return nil + } + _, err := i.store.Pool().Exec(ctx, + `update node set sealing_key = $2 where id = $1`, node, key) + return err +} + +// SealingKeyOf is the key to seal something to for a node, empty if it has none. +func (i *Inventory) SealingKeyOf(ctx context.Context, name string) (string, error) { + var key *string + err := i.store.Pool().QueryRow(ctx, + `select sealing_key from node where name = $1`, name).Scan(&key) + if err != nil { + return "", err + } + if key == nil { + return "", nil + } + return *key, nil +} + // RecordOverlayKey keeps the public half a node generated. func (i *Inventory) RecordOverlayKey(ctx context.Context, node, key string) error { if strings.TrimSpace(key) == "" { diff --git a/internal/inventory/secrets.go b/internal/inventory/secrets.go new file mode 100644 index 0000000..3aaf307 --- /dev/null +++ b/internal/inventory/secrets.go @@ -0,0 +1,135 @@ +package inventory + +import ( + "context" + + "github.com/novox/mesh-control/internal/secrets" +) + +// Where sealed secrets live. +// +// The table holds nothing usable — see the migration and internal/secrets for why that is the +// design rather than an inconvenience. + +// Secret is one provision's credential, sealed to each end. +type Secret struct { + Name string + Consumer string + Provider string + ForConsumer string + ForProvider string + ConsumerKey string + ProviderKey string +} + +// SecretFor is the credential for one provision between two nodes, making one the first time. +// +// **Made once and kept**, rather than regenerated whenever it is asked for. A secret that changed +// on every declaration would restart both ends on every push and would mean the password a +// provider was told to create never matches the one a consumer was given — which is a mesh that +// reports success and cannot connect. +// +// **Remade when either end's sealing key changes.** A node that rejoined generated a new key and +// can no longer open what was sealed to the old one, so keeping the blob would deliver something +// unreadable for ever. The new secret reaches both ends in the same push, which is the only +// moment they can be changed together. +func (i *Inventory) SecretFor(ctx context.Context, name, consumer, provider string) (Secret, error) { + consumerKey, err := i.SealingKeyOf(ctx, consumer) + if err != nil { + return Secret{}, err + } + providerKey, err := i.SealingKeyOf(ctx, provider) + if err != nil { + return Secret{}, err + } + + consumerNode, err := i.NodeByName(ctx, consumer) + if err != nil { + return Secret{}, err + } + providerNode, err := i.NodeByName(ctx, provider) + if err != nil { + return Secret{}, err + } + + var held Secret + err = i.store.Pool().QueryRow(ctx, + `select for_consumer, for_provider, consumer_key, provider_key from secret + where name = $1 and consumer = $2 and provider = $3`, + name, consumerNode.ID, providerNode.ID). + Scan(&held.ForConsumer, &held.ForProvider, &held.ConsumerKey, &held.ProviderKey) + if err == nil && held.ConsumerKey == consumerKey && held.ProviderKey == providerKey { + held.Name, held.Consumer, held.Provider = name, consumer, provider + return held, nil + } + + made, err := secrets.Make(consumerKey, providerKey) + if err != nil { + return Secret{}, err + } + _, err = i.store.Pool().Exec(ctx, + `insert into secret (name, consumer, provider, for_consumer, for_provider, + consumer_key, provider_key) + values ($1, $2, $3, $4, $5, $6, $7) + on conflict (name, consumer, provider) do update set + for_consumer = excluded.for_consumer, for_provider = excluded.for_provider, + consumer_key = excluded.consumer_key, provider_key = excluded.provider_key, + created_at = now()`, + name, consumerNode.ID, providerNode.ID, + made.ForConsumer, made.ForProvider, made.ConsumerKey, made.ProviderKey) + if err != nil { + return Secret{}, err + } + return Secret{Name: name, Consumer: consumer, Provider: provider, + ForConsumer: made.ForConsumer, ForProvider: made.ForProvider, + ConsumerKey: made.ConsumerKey, ProviderKey: made.ProviderKey}, nil +} + +// RotateSecret discards what was there, so the next declaration carries a new one. +// +// Only a delete. Nothing reads the old value first, because nothing can — and making the +// replacement here rather than on the next read would be a second path to the same act, which is +// how two ends come to hold different passwords. +// +// The new secret then reaches both ends on the same push, together, which is what makes rotation +// a single event rather than a fanout with a window where half the mesh holds a dead credential. +func (i *Inventory) RotateSecret(ctx context.Context, name, consumer, provider string) error { + consumerNode, err := i.NodeByName(ctx, consumer) + if err != nil { + return err + } + providerNode, err := i.NodeByName(ctx, provider) + if err != nil { + return err + } + _, err = i.store.Pool().Exec(ctx, + `delete from secret where name = $1 and consumer = $2 and provider = $3`, + name, consumerNode.ID, providerNode.ID) + return err +} + +// SecretsFrom is every credential a provider node was issued, so it can be told what to create. +func (i *Inventory) SecretsFrom(ctx context.Context, provider string) ([]Secret, error) { + providerNode, err := i.NodeByName(ctx, provider) + if err != nil { + return nil, err + } + rows, err := i.store.Pool().Query(ctx, + `select s.name, c.name, s.for_provider from secret s + join node c on c.id = s.consumer + where s.provider = $1 order by s.name, c.name`, providerNode.ID) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []Secret + for rows.Next() { + s := Secret{Provider: provider} + if err := rows.Scan(&s.Name, &s.Consumer, &s.ForProvider); err != nil { + return nil, err + } + out = append(out, s) + } + return out, rows.Err() +} diff --git a/internal/inventory/secrets_test.go b/internal/inventory/secrets_test.go new file mode 100644 index 0000000..9a70661 --- /dev/null +++ b/internal/inventory/secrets_test.go @@ -0,0 +1,232 @@ +package inventory + +import ( + "context" + "crypto/ecdh" + "crypto/rand" + "encoding/base64" + "strings" + "testing" + + "golang.org/x/crypto/nacl/box" +) + +// aSealingKey is a node's key, keeping the private half so a test can open what was sealed — the +// only assertion that actually distinguishes "the right blob" from "a blob". +func aSealingKey(t *testing.T) (string, func(string) ([]byte, bool)) { + t.Helper() + k, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + var pub, priv [32]byte + copy(pub[:], k.PublicKey().Bytes()) + copy(priv[:], k.Bytes()) + return base64.StdEncoding.EncodeToString(k.PublicKey().Bytes()), + func(sealed string) ([]byte, bool) { + blob, err := base64.StdEncoding.DecodeString(sealed) + if err != nil { + return nil, false + } + return box.OpenAnonymous(nil, blob, &pub, &priv) + } +} + +func twoNodesWithKeys(t *testing.T) (*Inventory, context.Context) { + t.Helper() + inv := fresh(t) + ctx := context.Background() + for _, n := range []string{"consumer", "provider"} { + node, err := inv.AddNode(ctx, n) + if err != nil { + t.Fatal(err) + } + key, _ := aSealingKey(t) + if err := inv.RecordSealingKey(ctx, node.ID, key); err != nil { + t.Fatal(err) + } + } + return inv, ctx +} + +func TestASecretIsMadeOnceAndKept(t *testing.T) { + // Regenerating on every declaration would restart both ends on every push, and — worse — the + // password a provider was told to create would never be the one its consumer was given. + inv, ctx := twoNodesWithKeys(t) + first, err := inv.SecretFor(ctx, "database", "consumer", "provider") + if err != nil { + t.Fatal(err) + } + second, err := inv.SecretFor(ctx, "database", "consumer", "provider") + if err != nil { + t.Fatal(err) + } + if first.ForConsumer != second.ForConsumer || first.ForProvider != second.ForProvider { + t.Fatal("asking twice produced two different credentials") + } +} + +func TestTheStoredSecretIsNotTheSecret(t *testing.T) { + // The whole point. A copy of this database is not a copy of the mesh's credentials — which is + // what an encrypted column does not achieve, because whoever runs the control plane can read + // through it. + inv, ctx := twoNodesWithKeys(t) + got, err := inv.SecretFor(ctx, "database", "consumer", "provider") + if err != nil { + t.Fatal(err) + } + var columns []string + rows, err := inv.store.Pool().Query(ctx, + `select column_name from information_schema.columns where table_name = 'secret'`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + for rows.Next() { + var c string + if err := rows.Scan(&c); err != nil { + t.Fatal(err) + } + columns = append(columns, c) + } + for _, c := range columns { + if strings.Contains(c, "password") || strings.Contains(c, "value") || + strings.Contains(c, "plain") { + t.Fatalf("the table has a column called %q, which suggests it holds the thing", c) + } + } + if got.ForConsumer == got.ForProvider { + t.Fatal("both ends were given the identical blob, so the storage reveals they match") + } +} + +func TestANewSealingKeyMeansANewSecret(t *testing.T) { + // A node that rejoined generated a new key and can no longer open what was sealed to the old + // one. Keeping the blob would deliver something unreadable for ever, reported as configured. + inv, ctx := twoNodesWithKeys(t) + before, err := inv.SecretFor(ctx, "database", "consumer", "provider") + if err != nil { + t.Fatal(err) + } + node, err := inv.NodeByName(ctx, "consumer") + if err != nil { + t.Fatal(err) + } + fresh, _ := aSealingKey(t) + if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil { + t.Fatal(err) + } + after, err := inv.SecretFor(ctx, "database", "consumer", "provider") + if err != nil { + t.Fatal(err) + } + if after.ForConsumer == before.ForConsumer { + t.Fatal("the node was handed a credential sealed to a key it no longer has") + } + // And the provider's copy changed too, in the same breath. Otherwise the two ends hold + // different passwords — which is the fanout window that makes rotation dangerous elsewhere. + if after.ForProvider == before.ForProvider { + t.Fatal("only one end was rotated, so the two now disagree") + } +} + +func TestRotatingReachesBothEnds(t *testing.T) { + inv, ctx := twoNodesWithKeys(t) + before, err := inv.SecretFor(ctx, "database", "consumer", "provider") + if err != nil { + t.Fatal(err) + } + if err := inv.RotateSecret(ctx, "database", "consumer", "provider"); err != nil { + t.Fatal(err) + } + after, err := inv.SecretFor(ctx, "database", "consumer", "provider") + if err != nil { + t.Fatal(err) + } + if after.ForConsumer == before.ForConsumer || after.ForProvider == before.ForProvider { + t.Fatal("rotation left one of the ends holding what it had") + } +} + +func TestAProviderIsToldEveryCredentialItMustCreate(t *testing.T) { + // The half that makes a credential real. A password nothing was told to create authenticates + // nowhere, and the mesh cannot tell the provider what it is in any other way — it cannot read + // it either. + inv, ctx := twoNodesWithKeys(t) + // The provider's own key, kept, so this asserts it can *open* what it was handed rather than + // that the field is non-empty. Without that, selecting the wrong column reads the same both + // ways and the test proves nothing — which it did, until the check was removed and it passed. + providerKey, openProvider := aSealingKey(t) + provider, err := inv.NodeByName(ctx, "provider") + if err != nil { + t.Fatal(err) + } + if err := inv.RecordSealingKey(ctx, provider.ID, providerKey); err != nil { + t.Fatal(err) + } + + other, err := inv.AddNode(ctx, "second-consumer") + if err != nil { + t.Fatal(err) + } + secondKey, _ := aSealingKey(t) + if err := inv.RecordSealingKey(ctx, other.ID, secondKey); err != nil { + t.Fatal(err) + } + for _, who := range []string{"consumer", "second-consumer"} { + if _, err := inv.SecretFor(ctx, "database", who, "provider"); err != nil { + t.Fatal(err) + } + } + issued, err := inv.SecretsFrom(ctx, "provider") + if err != nil { + t.Fatal(err) + } + if len(issued) != 2 { + t.Fatalf("the provider was told about %d of 2", len(issued)) + } + for _, s := range issued { + if _, ok := openProvider(s.ForProvider); !ok { + t.Fatalf("the provider cannot open the credential it was given for %s", s.Consumer) + } + if s.ForConsumer != "" { + // It has no business holding the other end's copy, and handing it out would put a + // second readable-by-someone-else copy into circulation. + t.Fatalf("the provider was handed the consumer's own copy of %s", s.Name) + } + } +} + +func TestANodeWithNoSealingKeyCannotBeGivenASecret(t *testing.T) { + inv := fresh(t) + ctx := context.Background() + for _, n := range []string{"consumer", "provider"} { + if _, err := inv.AddNode(ctx, n); err != nil { + t.Fatal(err) + } + } + _, err := inv.SecretFor(ctx, "database", "consumer", "provider") + if err == nil { + t.Fatal("a credential was made for nodes that cannot open one") + } + if !strings.Contains(err.Error(), "sealing key") { + t.Fatalf("the refusal does not say what is missing: %v", err) + } +} + +func TestSecretsGoWhenANodeLeaves(t *testing.T) { + inv, ctx := twoNodesWithKeys(t) + if _, err := inv.SecretFor(ctx, "database", "consumer", "provider"); err != nil { + t.Fatal(err) + } + if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'consumer'`); err != nil { + t.Fatal(err) + } + var left int + if err := inv.store.Pool().QueryRow(ctx, `select count(*) from secret`).Scan(&left); err != nil { + t.Fatal(err) + } + if left != 0 { + t.Fatalf("%d credential(s) outlived the machine they were for", left) + } +} diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index 5705a2b..3b0c23f 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -88,6 +88,16 @@ func (e Enrolment) Enrol(ctx context.Context, request EnrolRequest) (EnrolReply, // receive, and without this key the mesh cannot compose one. A node enrolled with no overlay // key is a node the graph skips — an ordinary in-between state, and one worth leaving as // briefly as possible. + // And the key its secrets are sealed to. Same reasoning as the overlay key below and one step + // stronger: without it the mesh cannot send this node a credential at all, and a node that + // enrolled without one will be refused a sealed file rather than quietly given none. + if request.SealingKey != "" { + if err := e.Inventory.RecordSealingKey(ctx, node.ID, request.SealingKey); err != nil { + return EnrolReply{}, fmt.Errorf( + "the token was spent and %s's sealing key could not be recorded: %w", + node.Name, err) + } + } if request.OverlayKey != "" { if err := e.Inventory.RecordOverlayKey(ctx, node.ID, request.OverlayKey); err != nil { return EnrolReply{}, fmt.Errorf( diff --git a/internal/link/protocol.go b/internal/link/protocol.go index 13c5067..d39cecb 100644 --- a/internal/link/protocol.go +++ b/internal/link/protocol.go @@ -45,6 +45,11 @@ type EnrolRequest struct { // from PublicKey, and the mesh only ever sees this half. OverlayKey string `json:"overlay_key,omitempty"` + // SealingKey is the public half of the key this node's secrets are sealed to. A third key, + // and the reasoning is the same one twice over: the mesh must be able to send this node + // something nothing else can read, and it must never be able to read it either. + SealingKey string `json:"sealing_key,omitempty"` + // Profile is what this machine can be asked to do. The control plane cannot decide what a // node should run without it, so it arrives with enrolment rather than being asked for after. Profile map[string]any `json:"profile,omitempty"` diff --git a/internal/secrets/seal.go b/internal/secrets/seal.go new file mode 100644 index 0000000..cff6eb0 --- /dev/null +++ b/internal/secrets/seal.go @@ -0,0 +1,91 @@ +package secrets + +import ( + "crypto/rand" + "encoding/base64" + "fmt" + + "golang.org/x/crypto/nacl/box" +) + +// Secrets the mesh delivers and cannot read. +// +// **What this is not.** The obvious arrangement is a credentials column, encrypted at rest. It +// has been built, in another mesh, and that mesh's own tooling records what it bought: a query +// against the encrypted column returns zero rows and proves nothing, so auditing moved to the +// decrypted copies on the nodes; and the tool for finding a secret has to search **by value** +// rather than by name, because the same password sits in the provisions table, in the environment +// table, in each node's environment file in plain text, and inside every connection string +// composed from it — copies its own documentation calls "often the only copies actually in use". +// +// Two faults there, and encryption at rest addresses neither. **The control plane can read what +// it stores**, so a copy of its database is a copy of every credential in the mesh. And **one +// secret has many homes with nothing tracking them.** +// +// So here the value is sealed to the node that will use it before it is stored, with a key that +// node generated and whose private half the mesh has never seen. What gets written is unusable by +// whoever holds it, the mesh included. And nothing is composed centrally — a connection string is +// assembled on the machine that needs one, so the mesh never mints a second copy in a shape +// nothing tracks. + +// Sealed is one value, closed to both ends of a provision. +// +// Two blobs of the same secret rather than one shared key: a key both ends hold is a key the mesh +// would have to distribute, which is this problem again one level down. +type Sealed struct { + ForConsumer string + ForProvider string + // Which key each was sealed to, kept so a node that regenerated its key can be told what it + // can no longer open rather than discovering it as a service that will not start. + ConsumerKey string + ProviderKey string +} + +// Make generates a secret and seals it to both ends, keeping no readable copy. +// +// The plaintext exists for the length of this call. Rotation is therefore generating a new one +// rather than reading the old one back — the only version of rotation that is honest about what +// the mesh knows. +func Make(consumerKey, providerKey string) (Sealed, error) { + if consumerKey == "" || providerKey == "" { + // Sealing to an empty key would produce a blob nobody can open, stored as though it were + // a working credential. The caller knows which node is which and says so. + return Sealed{}, fmt.Errorf("both ends need a sealing key before a secret can be made") + } + + value := make([]byte, 32) + if _, err := rand.Read(value); err != nil { + return Sealed{}, err + } + // Base64 without padding, because it lands in a configuration file something else parses and + // a password containing a newline or a quote is a support call. + password := base64.RawURLEncoding.EncodeToString(value) + + forConsumer, err := Seal(consumerKey, []byte(password)) + if err != nil { + return Sealed{}, err + } + forProvider, err := Seal(providerKey, []byte(password)) + if err != nil { + return Sealed{}, err + } + return Sealed{ + ForConsumer: forConsumer, ForProvider: forProvider, + ConsumerKey: consumerKey, ProviderKey: providerKey, + }, nil +} + +// Seal closes a value to a node's public sealing key. +func Seal(publicKey string, value []byte) (string, error) { + public, err := base64.StdEncoding.DecodeString(publicKey) + if err != nil || len(public) != 32 { + return "", fmt.Errorf("%q is not a sealing key", publicKey) + } + var pub [32]byte + copy(pub[:], public) + sealed, err := box.SealAnonymous(nil, value, &pub, rand.Reader) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(sealed), nil +} diff --git a/internal/secrets/seal_test.go b/internal/secrets/seal_test.go new file mode 100644 index 0000000..17cfde9 --- /dev/null +++ b/internal/secrets/seal_test.go @@ -0,0 +1,156 @@ +package secrets + +import ( + "crypto/ecdh" + "crypto/rand" + "encoding/base64" + "strings" + "testing" + + "golang.org/x/crypto/nacl/box" +) + +// A node's key, as the node would generate it and report the public half. +func nodeKey(t *testing.T) (public string, open func(string) ([]byte, error)) { + t.Helper() + private, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + var pub, priv [32]byte + copy(pub[:], private.PublicKey().Bytes()) + copy(priv[:], private.Bytes()) + return base64.StdEncoding.EncodeToString(private.PublicKey().Bytes()), + func(sealed string) ([]byte, error) { + blob, err := base64.StdEncoding.DecodeString(sealed) + if err != nil { + return nil, err + } + out, ok := box.OpenAnonymous(nil, blob, &pub, &priv) + if !ok { + return nil, errNotForYou + } + return out, nil + } +} + +var errNotForYou = ¬ForYou{} + +type notForYou struct{} + +func (*notForYou) Error() string { return "not sealed to this node" } + +func TestBothEndsGetTheSameSecretAndTheMeshGetsNeither(t *testing.T) { + // The whole arrangement in one test. The provider must create the credential the consumer was + // given, or the mesh reports success and nothing can connect — and neither blob is readable + // by whoever is holding them, which is the part encrypting a column does not achieve. + consumerPub, openConsumer := nodeKey(t) + providerPub, openProvider := nodeKey(t) + + sealed, err := Make(consumerPub, providerPub) + if err != nil { + t.Fatal(err) + } + forConsumer, err := openConsumer(sealed.ForConsumer) + if err != nil { + t.Fatal(err) + } + forProvider, err := openProvider(sealed.ForProvider) + if err != nil { + t.Fatal(err) + } + if string(forConsumer) != string(forProvider) { + t.Fatalf("the two ends were given different passwords: %q and %q", + forConsumer, forProvider) + } + if len(forConsumer) < 32 { + t.Fatalf("the password is %d characters, which is not a password", len(forConsumer)) + } + // Neither can open the other's, which is what makes two blobs different from one shared key. + if _, err := openConsumer(sealed.ForProvider); err == nil { + t.Fatal("the consumer opened the provider's copy") + } +} + +func TestTheSealedFormLooksNothingLikeTheSecret(t *testing.T) { + consumerPub, openConsumer := nodeKey(t) + providerPub, _ := nodeKey(t) + sealed, err := Make(consumerPub, providerPub) + if err != nil { + t.Fatal(err) + } + password, err := openConsumer(sealed.ForConsumer) + if err != nil { + t.Fatal(err) + } + if strings.Contains(sealed.ForConsumer, string(password)) { + t.Fatal("the secret is visible inside what is stored") + } + if sealed.ForConsumer == sealed.ForProvider { + // Sealed boxes are randomised, so an observer cannot tell the two ends hold the same + // value — nor that a rotation changed nothing. + t.Fatal("the two blobs are identical, so the storage says they hold the same value") + } +} + +func TestAPasswordIsSafeToPutInAFile(t *testing.T) { + // It lands in a file something else reads. A newline or a quote in it is a support call. + consumerPub, openConsumer := nodeKey(t) + providerPub, _ := nodeKey(t) + for i := 0; i < 50; i++ { + sealed, err := Make(consumerPub, providerPub) + if err != nil { + t.Fatal(err) + } + password, err := openConsumer(sealed.ForConsumer) + if err != nil { + t.Fatal(err) + } + if strings.ContainsAny(string(password), "\n\r\t \"'\\$`") { + t.Fatalf("a password needs quoting: %q", password) + } + } +} + +func TestEverySecretIsDifferent(t *testing.T) { + consumerPub, openConsumer := nodeKey(t) + providerPub, _ := nodeKey(t) + seen := map[string]bool{} + for i := 0; i < 50; i++ { + sealed, _ := Make(consumerPub, providerPub) + password, _ := openConsumer(sealed.ForConsumer) + if seen[string(password)] { + t.Fatalf("the same password came out twice: %q", password) + } + seen[string(password)] = true + } +} + +func TestAnEndWithNoSealingKeyIsRefused(t *testing.T) { + // Sealing to nothing would produce a blob nobody can open, stored as though it were a working + // credential — which is the failure this whole design exists to make impossible. + // + // Asserted on the message, not merely on failing. Seal refuses an empty key anyway, so a test + // that only checked for an error passed with this check removed and proved nothing about it. + // What the check adds is a reason a person can act on: the remedy is on the node, not here. + public, _ := nodeKey(t) + for _, pair := range [][2]string{{public, ""}, {"", public}} { + _, err := Make(pair[0], pair[1]) + if err == nil { + t.Fatal("a secret was made for a node with no sealing key") + } + if !strings.Contains(err.Error(), "both ends need a sealing key") { + t.Fatalf("the refusal does not say what is missing: %v", err) + } + } +} + +func TestSomethingThatIsNotAKeyIsRefused(t *testing.T) { + if _, err := Seal("not-a-key", []byte("x")); err == nil { + t.Fatal("a secret was sealed to nonsense") + } + short := base64.StdEncoding.EncodeToString([]byte("too short")) + if _, err := Seal(short, []byte("x")); err == nil { + t.Fatal("a secret was sealed to a key of the wrong length") + } +}