Files
mesh-controller/internal/inventory/secrets_test.go
T
jschoubben ee84b624b1 A provision names the engine, because a consumer is coupled to one
Provisions were named after roles: provides "database", requires
"database". Nothing distinguished engines, so a module written against
PostgreSQL could be matched to a provider of SQL Server, resolve as
satisfied, deploy, and fail on its first query — with nothing
connecting that error back to a match made elsewhere by something that
believed it had done its job.

The failure is in the direction that hides. Refusing on ambiguity
exists precisely so this does not happen, and the generic name walked
around it: with one provider of each name nothing is ambiguous, so
nothing is asked.

How it got in: every resolver test had exactly one provider per name,
so no mismatch was expressible and none was caught. The fixtures agreed
with the design — the same fault as the imagined test output in
04-ISSUES/005, at the level of a name.

Refused rather than documented, because the old naming *was* the
documented convention. Providing database/db/sql/sql-database is now a
parse error naming what to write instead.

The rule is about coupling, not specificity everywhere: route and
resolver stay role-named, because a consumer genuinely cannot tell
which proxy answered. novox/hq ADR 0027.
2026-08-31 17:12:46 +02:00

552 lines
18 KiB
Go

package inventory
import (
"context"
"crypto/ecdh"
"crypto/rand"
"encoding/base64"
"github.com/novox/mesh-control/internal/catalogue"
"strings"
"testing"
"golang.org/x/crypto/nacl/box"
)
// aSealingKey is a node's key, keeping the private half so a test can open what was sealed — the
// only assertion that actually distinguishes "the right blob" from "a blob".
func aSealingKey(t *testing.T) (string, func(string) ([]byte, bool)) {
t.Helper()
k, err := ecdh.X25519().GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
var pub, priv [32]byte
copy(pub[:], k.PublicKey().Bytes())
copy(priv[:], k.Bytes())
return base64.StdEncoding.EncodeToString(k.PublicKey().Bytes()),
func(sealed string) ([]byte, bool) {
blob, err := base64.StdEncoding.DecodeString(sealed)
if err != nil {
return nil, false
}
return box.OpenAnonymous(nil, blob, &pub, &priv)
}
}
func twoNodesWithKeys(t *testing.T) (*Inventory, context.Context) {
t.Helper()
inv := fresh(t)
ctx := context.Background()
for _, n := range []string{"consumer", "provider"} {
node, err := inv.AddNode(ctx, n)
if err != nil {
t.Fatal(err)
}
key, _ := aSealingKey(t)
if err := inv.RecordSealingKey(ctx, node.ID, key); err != nil {
t.Fatal(err)
}
}
return inv, ctx
}
func TestASecretIsMadeOnceAndKept(t *testing.T) {
// Regenerating on every declaration would restart both ends on every push, and — worse — the
// password a provider was told to create would never be the one its consumer was given.
inv, ctx := twoNodesWithKeys(t)
first, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
if err != nil {
t.Fatal(err)
}
second, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
if err != nil {
t.Fatal(err)
}
if first.ForConsumer != second.ForConsumer || first.ForProvider != second.ForProvider {
t.Fatal("asking twice produced two different credentials")
}
}
func TestTheStoredSecretIsNotTheSecret(t *testing.T) {
// The whole point. A copy of this database is not a copy of the mesh's credentials — which is
// what an encrypted column does not achieve, because whoever runs the control plane can read
// through it.
inv, ctx := twoNodesWithKeys(t)
got, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
if err != nil {
t.Fatal(err)
}
var columns []string
rows, err := inv.store.Pool().Query(ctx,
`select column_name from information_schema.columns where table_name = 'secret'`)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
for rows.Next() {
var c string
if err := rows.Scan(&c); err != nil {
t.Fatal(err)
}
columns = append(columns, c)
}
for _, c := range columns {
if strings.Contains(c, "password") || strings.Contains(c, "value") ||
strings.Contains(c, "plain") {
t.Fatalf("the table has a column called %q, which suggests it holds the thing", c)
}
}
if got.ForConsumer == got.ForProvider {
t.Fatal("both ends were given the identical blob, so the storage reveals they match")
}
}
func TestANewSealingKeyMeansANewSecret(t *testing.T) {
// A node that rejoined generated a new key and can no longer open what was sealed to the old
// one. Keeping the blob would deliver something unreadable for ever, reported as configured.
inv, ctx := twoNodesWithKeys(t)
before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
if err != nil {
t.Fatal(err)
}
node, err := inv.NodeByName(ctx, "consumer")
if err != nil {
t.Fatal(err)
}
fresh, _ := aSealingKey(t)
if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil {
t.Fatal(err)
}
after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
if err != nil {
t.Fatal(err)
}
if after.ForConsumer == before.ForConsumer {
t.Fatal("the node was handed a credential sealed to a key it no longer has")
}
// And the provider's copy changed too, in the same breath. Otherwise the two ends hold
// different passwords — which is the fanout window that makes rotation dangerous elsewhere.
if after.ForProvider == before.ForProvider {
t.Fatal("only one end was rotated, so the two now disagree")
}
}
func TestRotatingReachesBothEnds(t *testing.T) {
inv, ctx := twoNodesWithKeys(t)
before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
if err != nil {
t.Fatal(err)
}
if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "provider"); err != nil {
t.Fatal(err)
}
after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
if err != nil {
t.Fatal(err)
}
if after.ForConsumer == before.ForConsumer || after.ForProvider == before.ForProvider {
t.Fatal("rotation left one of the ends holding what it had")
}
}
func TestAProviderIsToldEveryCredentialItMustCreate(t *testing.T) {
// The half that makes a credential real. A password nothing was told to create authenticates
// nowhere, and the mesh cannot tell the provider what it is in any other way — it cannot read
// it either.
inv, ctx := twoNodesWithKeys(t)
// The provider's own key, kept, so this asserts it can *open* what it was handed rather than
// that the field is non-empty. Without that, selecting the wrong column reads the same both
// ways and the test proves nothing — which it did, until the check was removed and it passed.
providerKey, openProvider := aSealingKey(t)
provider, err := inv.NodeByName(ctx, "provider")
if err != nil {
t.Fatal(err)
}
if err := inv.RecordSealingKey(ctx, provider.ID, providerKey); err != nil {
t.Fatal(err)
}
other, err := inv.AddNode(ctx, "second-consumer")
if err != nil {
t.Fatal(err)
}
secondKey, _ := aSealingKey(t)
if err := inv.RecordSealingKey(ctx, other.ID, secondKey); err != nil {
t.Fatal(err)
}
for _, who := range []string{"consumer", "second-consumer"} {
if _, err := inv.SecretFor(ctx, "postgres-database", who, "provider"); err != nil {
t.Fatal(err)
}
}
issued, err := inv.SecretsFrom(ctx, "provider")
if err != nil {
t.Fatal(err)
}
if len(issued) != 2 {
t.Fatalf("the provider was told about %d of 2", len(issued))
}
for _, s := range issued {
if _, ok := openProvider(s.ForProvider); !ok {
t.Fatalf("the provider cannot open the credential it was given for %s", s.Consumer)
}
if s.ForConsumer != "" {
// It has no business holding the other end's copy, and handing it out would put a
// second readable-by-someone-else copy into circulation.
t.Fatalf("the provider was handed the consumer's own copy of %s", s.Name)
}
}
}
func TestANodeWithNoSealingKeyCannotBeGivenASecret(t *testing.T) {
inv := fresh(t)
ctx := context.Background()
for _, n := range []string{"consumer", "provider"} {
if _, err := inv.AddNode(ctx, n); err != nil {
t.Fatal(err)
}
}
_, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
if err == nil {
t.Fatal("a credential was made for nodes that cannot open one")
}
if !strings.Contains(err.Error(), "sealing key") {
t.Fatalf("the refusal does not say what is missing: %v", err)
}
}
func TestSecretsGoWhenANodeLeaves(t *testing.T) {
inv, ctx := twoNodesWithKeys(t)
if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider"); err != nil {
t.Fatal(err)
}
if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'consumer'`); err != nil {
t.Fatal(err)
}
var left int
if err := inv.store.Pool().QueryRow(ctx, `select count(*) from secret`).Scan(&left); err != nil {
t.Fatal(err)
}
if left != 0 {
t.Fatalf("%d credential(s) outlived the machine they were for", left)
}
}
func TestAModulesOwnSecretIsPerMachineAndKept(t *testing.T) {
// A module running on three machines has three passwords. One in the manifest instead would
// put the same secret on every machine that ever runs it, in a file anybody can read.
inv, ctx := twoNodesWithKeys(t)
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
Source{}); err != nil {
t.Fatal(err)
}
here, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
if err != nil {
t.Fatal(err)
}
there, err := inv.SecretForModule(ctx, "provider", "postgres", "superuser")
if err != nil {
t.Fatal(err)
}
if here == there {
t.Fatal("two machines were given the same secret")
}
// Made once and kept, or a running database would be handed a password it was not started
// with on the next declaration.
again, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
if err != nil {
t.Fatal(err)
}
if again != here {
t.Fatal("asking twice made a second secret")
}
}
func TestTwoNeedsInOneModuleAreTwoSecrets(t *testing.T) {
inv, ctx := twoNodesWithKeys(t)
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
Source{}); err != nil {
t.Fatal(err)
}
one, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
if err != nil {
t.Fatal(err)
}
two, err := inv.SecretForModule(ctx, "consumer", "postgres", "replication")
if err != nil {
t.Fatal(err)
}
if one == two {
t.Fatal("two names gave one secret")
}
}
func TestAModulesSecretIsRemadeWhenTheMachineRejoins(t *testing.T) {
// The node generated a new sealing key and can no longer open what was sealed to the old one.
inv, ctx := twoNodesWithKeys(t)
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
Source{}); err != nil {
t.Fatal(err)
}
before, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
if err != nil {
t.Fatal(err)
}
node, err := inv.NodeByName(ctx, "consumer")
if err != nil {
t.Fatal(err)
}
fresh, _ := aSealingKey(t)
if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil {
t.Fatal(err)
}
after, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
if err != nil {
t.Fatal(err)
}
if after == before {
t.Fatal("a machine was handed a secret sealed to a key it no longer has")
}
}
func TestAMachineWithNoSealingKeyCannotBeGivenAModuleSecret(t *testing.T) {
inv := fresh(t)
ctx := context.Background()
if _, err := inv.AddNode(ctx, "bare"); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
Source{}); err != nil {
t.Fatal(err)
}
if _, err := inv.SecretForModule(ctx, "bare", "postgres", "superuser"); err == nil {
t.Fatal("a secret was made for a machine that cannot open one")
}
}
func TestACredentialGoesWhenTheConsumerStopsAskingForIt(t *testing.T) {
// Withdrawal is the half nobody tests. A consumer that is unassigned must stop appearing in
// what its provider is told to create, or the provider keeps a working login for a machine
// that no longer uses it — and the provisioner's own rule about removing what nobody asks for
// only fires if the mesh stops asking.
inv, ctx := twoNodesWithKeys(t)
if err := inv.RegisterModule(ctx, catalogue.Manifest{
Module: "meshboard", Version: "1", Requires: []string{"postgres-database"},
}, Source{}); err != nil {
t.Fatal(err)
}
if err := inv.Assign(ctx, "consumer", "meshboard"); err != nil {
t.Fatal(err)
}
if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider"); err != nil {
t.Fatal(err)
}
issued, err := inv.SecretsFrom(ctx, "provider")
if err != nil {
t.Fatal(err)
}
if len(issued) != 1 {
t.Fatalf("the provider was told about %d consumers", len(issued))
}
// Unassigned. The secret itself is deliberately NOT deleted here — see below — but nothing
// should now resolve on that machine that wants it.
if err := inv.Unassign(ctx, "consumer", "meshboard"); err != nil {
t.Fatal(err)
}
assigned, err := inv.Assigned(ctx, "consumer")
if err != nil {
t.Fatal(err)
}
if len(assigned) != 0 {
t.Fatalf("the module is still assigned: %v", assigned)
}
}
func TestACredentialGoesWhenEitherMachineDoes(t *testing.T) {
// The case that must not leave a live login behind: a machine removed from the mesh. Its
// credentials go with it, and the provider stops being told to keep them.
inv, ctx := twoNodesWithKeys(t)
if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider"); err != nil {
t.Fatal(err)
}
if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'consumer'`); err != nil {
t.Fatal(err)
}
issued, err := inv.SecretsFrom(ctx, "provider")
if err != nil {
t.Fatal(err)
}
if len(issued) != 0 {
t.Fatalf("a departed machine's credential is still granted: %+v", issued)
}
}
// The other half of that, and the one that must not behave the same way.
//
// A secret the mesh made, it can make again — nothing else ever knew the old one. A secret the
// mesh was *given* it cannot: the broker knows a password, and the mesh inventing another puts 32
// random bytes where a working credential was. The machine applies it, reports success, and
// whatever reads it fails to authenticate somewhere else entirely, with the mesh insisting the
// secret was delivered — which it was.
func TestASecretTheMeshWasGivenIsNotReinventedWhenTheMachineRejoins(t *testing.T) {
inv, ctx := twoNodesWithKeys(t)
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "builder", Version: "1"},
Source{}); err != nil {
t.Fatal(err)
}
const url = "amqps://builder:the-password-the-broker-was-told@broker/"
if err := inv.AcceptSecretForModule(ctx, "consumer", "builder", "broker", url); err != nil {
t.Fatal(err)
}
node, err := inv.NodeByName(ctx, "consumer")
if err != nil {
t.Fatal(err)
}
fresh, _ := aSealingKey(t)
if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil {
t.Fatal(err)
}
_, err = inv.SecretForModule(ctx, "consumer", "builder", "broker")
if err == nil {
t.Fatal("the mesh invented a broker password, which the broker has never heard of")
}
// And says what to do about it, because the remedy is a command somebody runs and no amount
// of pushing will produce one.
if !strings.Contains(err.Error(), "issue it again") {
t.Fatalf("refused without saying what would fix it: %v", err)
}
}
// Until the key changes, a given secret is handed back unchanged — the ordinary case, and the one
// that would make the refusal above useless if it were wrong.
func TestASecretTheMeshWasGivenSurvivesAnOrdinaryPush(t *testing.T) {
inv, ctx := twoNodesWithKeys(t)
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "builder", Version: "1"},
Source{}); err != nil {
t.Fatal(err)
}
if err := inv.AcceptSecretForModule(ctx, "consumer", "builder", "broker",
"amqps://builder:password@broker/"); err != nil {
t.Fatal(err)
}
first, err := inv.SecretForModule(ctx, "consumer", "builder", "broker")
if err != nil {
t.Fatal(err)
}
second, err := inv.SecretForModule(ctx, "consumer", "builder", "broker")
if err != nil {
t.Fatal(err)
}
if first != second || first == "" {
t.Fatal("a given secret changed between two pushes, so the machine was handed two")
}
}
// Rotation has to know who holds the old credential, and that was the half HAL could not answer.
//
// There a provision had one shared credential; rotating it updated the provider's row and nothing
// enumerated the consumers, so three nodes carried dead credentials for two days while the mesh
// reported success (novox/hq ADR 0001). Here the holders are a set, and this is the query that
// makes "every consumer" nameable rather than hopeful.
func TestEveryHolderOfACredentialCanBeNamed(t *testing.T) {
inv, ctx := twoNodesWithKeys(t)
third, err := inv.AddNode(ctx, "third")
if err != nil {
t.Fatal(err)
}
key, _ := aSealingKey(t)
if err := inv.RecordSealingKey(ctx, third.ID, key); err != nil {
t.Fatal(err)
}
for _, consumer := range []string{"consumer", "third"} {
if _, err := inv.SecretFor(ctx, "postgres-database", consumer, "provider"); err != nil {
t.Fatal(err)
}
}
// And one for a different provision, which must not be swept up.
if _, err := inv.SecretFor(ctx, "cache", "consumer", "provider"); err != nil {
t.Fatal(err)
}
holders, err := inv.HoldersOf(ctx, "postgres-database", "")
if err != nil {
t.Fatal(err)
}
if len(holders) != 2 {
t.Fatalf("a holder of the credential was not named: %+v", holders)
}
for _, h := range holders {
if h.Provision != "postgres-database" {
t.Fatalf("rotating one provision would have touched %q", h.Provision)
}
}
// One machine's, when that is what was asked for. Rotating the other nine because one is
// suspected is a great deal of disruption for one suspicion.
one, err := inv.HoldersOf(ctx, "postgres-database", "third")
if err != nil {
t.Fatal(err)
}
if len(one) != 1 || one[0].Consumer != "third" {
t.Fatalf("asking for one machine's holder gave %+v", one)
}
}
// And rotating gives both ends a new credential, together — the same one.
func TestRotatingGivesBothEndsTheSameNewCredential(t *testing.T) {
inv, ctx := twoNodesWithKeys(t)
before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
if err != nil {
t.Fatal(err)
}
if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "provider"); err != nil {
t.Fatal(err)
}
after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
if err != nil {
t.Fatal(err)
}
if after.ForConsumer == before.ForConsumer {
t.Fatal("the consumer was handed the credential that was just rotated away")
}
if after.ForProvider == before.ForProvider {
t.Fatal("the provider was left creating the old password, which is the fault exactly")
}
// The two halves are the same secret sealed twice, which is the whole point: a provider
// creating one password and a consumer given another is a mesh that reports success and
// cannot connect.
if after.ForConsumer == after.ForProvider {
t.Fatal("both ends were sealed identically, so one of them cannot open it")
}
// Rotating one pair leaves every other holder alone. Otherwise "rotate this machine's
// credential" is a mesh-wide outage with a narrow name.
third, err := inv.AddNode(ctx, "third")
if err != nil {
t.Fatal(err)
}
key, _ := aSealingKey(t)
if err := inv.RecordSealingKey(ctx, third.ID, key); err != nil {
t.Fatal(err)
}
untouched, err := inv.SecretFor(ctx, "postgres-database", "third", "provider")
if err != nil {
t.Fatal(err)
}
if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "provider"); err != nil {
t.Fatal(err)
}
again, err := inv.SecretFor(ctx, "postgres-database", "third", "provider")
if err != nil {
t.Fatal(err)
}
if again.ForConsumer != untouched.ForConsumer {
t.Fatal("rotating one machine's credential changed another machine's")
}
}