mesh-controller/internal/inventory/secrets_test.go

package inventory

import (
	"context"
	"crypto/ecdh"
	"crypto/rand"
	"encoding/base64"
	"github.com/novox/mesh-control/internal/catalogue"
	"strings"
	"testing"

	"golang.org/x/crypto/nacl/box"
)

// aSealingKey is a node's key, keeping the private half so a test can open what was sealed — the
// only assertion that actually distinguishes "the right blob" from "a blob".
func aSealingKey(t *testing.T) (string, func(string) ([]byte, bool)) {
	t.Helper()
	k, err := ecdh.X25519().GenerateKey(rand.Reader)
	if err != nil {
		t.Fatal(err)
	}
	var pub, priv [32]byte
	copy(pub[:], k.PublicKey().Bytes())
	copy(priv[:], k.Bytes())
	return base64.StdEncoding.EncodeToString(k.PublicKey().Bytes()),
		func(sealed string) ([]byte, bool) {
			blob, err := base64.StdEncoding.DecodeString(sealed)
			if err != nil {
				return nil, false
			}
			return box.OpenAnonymous(nil, blob, &pub, &priv)
		}
}

func twoNodesWithKeys(t *testing.T) (*Inventory, context.Context) {
	t.Helper()
	inv := fresh(t)
	ctx := context.Background()
	for _, n := range []string{"consumer", "provider"} {
		node, err := inv.AddNode(ctx, n)
		if err != nil {
			t.Fatal(err)
		}
		key, _ := aSealingKey(t)
		if err := inv.RecordSealingKey(ctx, node.ID, key); err != nil {
			t.Fatal(err)
		}
	}
	return inv, ctx
}

func TestASecretIsMadeOnceAndKept(t *testing.T) {
	// Regenerating on every declaration would restart both ends on every push, and — worse — the
	// password a provider was told to create would never be the one its consumer was given.
	inv, ctx := twoNodesWithKeys(t)
	first, err := inv.SecretFor(ctx, "database", "consumer", "provider")
	if err != nil {
		t.Fatal(err)
	}
	second, err := inv.SecretFor(ctx, "database", "consumer", "provider")
	if err != nil {
		t.Fatal(err)
	}
	if first.ForConsumer != second.ForConsumer || first.ForProvider != second.ForProvider {
		t.Fatal("asking twice produced two different credentials")
	}
}

func TestTheStoredSecretIsNotTheSecret(t *testing.T) {
	// The whole point. A copy of this database is not a copy of the mesh's credentials — which is
	// what an encrypted column does not achieve, because whoever runs the control plane can read
	// through it.
	inv, ctx := twoNodesWithKeys(t)
	got, err := inv.SecretFor(ctx, "database", "consumer", "provider")
	if err != nil {
		t.Fatal(err)
	}
	var columns []string
	rows, err := inv.store.Pool().Query(ctx,
		`select column_name from information_schema.columns where table_name = 'secret'`)
	if err != nil {
		t.Fatal(err)
	}
	defer rows.Close()
	for rows.Next() {
		var c string
		if err := rows.Scan(&c); err != nil {
			t.Fatal(err)
		}
		columns = append(columns, c)
	}
	for _, c := range columns {
		if strings.Contains(c, "password") || strings.Contains(c, "value") ||
			strings.Contains(c, "plain") {
			t.Fatalf("the table has a column called %q, which suggests it holds the thing", c)
		}
	}
	if got.ForConsumer == got.ForProvider {
		t.Fatal("both ends were given the identical blob, so the storage reveals they match")
	}
}

func TestANewSealingKeyMeansANewSecret(t *testing.T) {
	// A node that rejoined generated a new key and can no longer open what was sealed to the old
	// one. Keeping the blob would deliver something unreadable for ever, reported as configured.
	inv, ctx := twoNodesWithKeys(t)
	before, err := inv.SecretFor(ctx, "database", "consumer", "provider")
	if err != nil {
		t.Fatal(err)
	}
	node, err := inv.NodeByName(ctx, "consumer")
	if err != nil {
		t.Fatal(err)
	}
	fresh, _ := aSealingKey(t)
	if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil {
		t.Fatal(err)
	}
	after, err := inv.SecretFor(ctx, "database", "consumer", "provider")
	if err != nil {
		t.Fatal(err)
	}
	if after.ForConsumer == before.ForConsumer {
		t.Fatal("the node was handed a credential sealed to a key it no longer has")
	}
	// And the provider's copy changed too, in the same breath. Otherwise the two ends hold
	// different passwords — which is the fanout window that makes rotation dangerous elsewhere.
	if after.ForProvider == before.ForProvider {
		t.Fatal("only one end was rotated, so the two now disagree")
	}
}

func TestRotatingReachesBothEnds(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	before, err := inv.SecretFor(ctx, "database", "consumer", "provider")
	if err != nil {
		t.Fatal(err)
	}
	if err := inv.RotateSecret(ctx, "database", "consumer", "provider"); err != nil {
		t.Fatal(err)
	}
	after, err := inv.SecretFor(ctx, "database", "consumer", "provider")
	if err != nil {
		t.Fatal(err)
	}
	if after.ForConsumer == before.ForConsumer || after.ForProvider == before.ForProvider {
		t.Fatal("rotation left one of the ends holding what it had")
	}
}

func TestAProviderIsToldEveryCredentialItMustCreate(t *testing.T) {
	// The half that makes a credential real. A password nothing was told to create authenticates
	// nowhere, and the mesh cannot tell the provider what it is in any other way — it cannot read
	// it either.
	inv, ctx := twoNodesWithKeys(t)
	// The provider's own key, kept, so this asserts it can *open* what it was handed rather than
	// that the field is non-empty. Without that, selecting the wrong column reads the same both
	// ways and the test proves nothing — which it did, until the check was removed and it passed.
	providerKey, openProvider := aSealingKey(t)
	provider, err := inv.NodeByName(ctx, "provider")
	if err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordSealingKey(ctx, provider.ID, providerKey); err != nil {
		t.Fatal(err)
	}

	other, err := inv.AddNode(ctx, "second-consumer")
	if err != nil {
		t.Fatal(err)
	}
	secondKey, _ := aSealingKey(t)
	if err := inv.RecordSealingKey(ctx, other.ID, secondKey); err != nil {
		t.Fatal(err)
	}
	for _, who := range []string{"consumer", "second-consumer"} {
		if _, err := inv.SecretFor(ctx, "database", who, "provider"); err != nil {
			t.Fatal(err)
		}
	}
	issued, err := inv.SecretsFrom(ctx, "provider")
	if err != nil {
		t.Fatal(err)
	}
	if len(issued) != 2 {
		t.Fatalf("the provider was told about %d of 2", len(issued))
	}
	for _, s := range issued {
		if _, ok := openProvider(s.ForProvider); !ok {
			t.Fatalf("the provider cannot open the credential it was given for %s", s.Consumer)
		}
		if s.ForConsumer != "" {
			// It has no business holding the other end's copy, and handing it out would put a
			// second readable-by-someone-else copy into circulation.
			t.Fatalf("the provider was handed the consumer's own copy of %s", s.Name)
		}
	}
}

func TestANodeWithNoSealingKeyCannotBeGivenASecret(t *testing.T) {
	inv := fresh(t)
	ctx := context.Background()
	for _, n := range []string{"consumer", "provider"} {
		if _, err := inv.AddNode(ctx, n); err != nil {
			t.Fatal(err)
		}
	}
	_, err := inv.SecretFor(ctx, "database", "consumer", "provider")
	if err == nil {
		t.Fatal("a credential was made for nodes that cannot open one")
	}
	if !strings.Contains(err.Error(), "sealing key") {
		t.Fatalf("the refusal does not say what is missing: %v", err)
	}
}

func TestSecretsGoWhenANodeLeaves(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	if _, err := inv.SecretFor(ctx, "database", "consumer", "provider"); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'consumer'`); err != nil {
		t.Fatal(err)
	}
	var left int
	if err := inv.store.Pool().QueryRow(ctx, `select count(*) from secret`).Scan(&left); err != nil {
		t.Fatal(err)
	}
	if left != 0 {
		t.Fatalf("%d credential(s) outlived the machine they were for", left)
	}
}

func TestAModulesOwnSecretIsPerMachineAndKept(t *testing.T) {
	// A module running on three machines has three passwords. One in the manifest instead would
	// put the same secret on every machine that ever runs it, in a file anybody can read.
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
		Source{}); err != nil {
		t.Fatal(err)
	}

	here, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
	if err != nil {
		t.Fatal(err)
	}
	there, err := inv.SecretForModule(ctx, "provider", "postgres", "superuser")
	if err != nil {
		t.Fatal(err)
	}
	if here == there {
		t.Fatal("two machines were given the same secret")
	}

	// Made once and kept, or a running database would be handed a password it was not started
	// with on the next declaration.
	again, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
	if err != nil {
		t.Fatal(err)
	}
	if again != here {
		t.Fatal("asking twice made a second secret")
	}
}

func TestTwoNeedsInOneModuleAreTwoSecrets(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
		Source{}); err != nil {
		t.Fatal(err)
	}
	one, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
	if err != nil {
		t.Fatal(err)
	}
	two, err := inv.SecretForModule(ctx, "consumer", "postgres", "replication")
	if err != nil {
		t.Fatal(err)
	}
	if one == two {
		t.Fatal("two names gave one secret")
	}
}

func TestAModulesSecretIsRemadeWhenTheMachineRejoins(t *testing.T) {
	// The node generated a new sealing key and can no longer open what was sealed to the old one.
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
		Source{}); err != nil {
		t.Fatal(err)
	}
	before, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
	if err != nil {
		t.Fatal(err)
	}
	node, err := inv.NodeByName(ctx, "consumer")
	if err != nil {
		t.Fatal(err)
	}
	fresh, _ := aSealingKey(t)
	if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil {
		t.Fatal(err)
	}
	after, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
	if err != nil {
		t.Fatal(err)
	}
	if after == before {
		t.Fatal("a machine was handed a secret sealed to a key it no longer has")
	}
}

func TestAMachineWithNoSealingKeyCannotBeGivenAModuleSecret(t *testing.T) {
	inv := fresh(t)
	ctx := context.Background()
	if _, err := inv.AddNode(ctx, "bare"); err != nil {
		t.Fatal(err)
	}
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
		Source{}); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.SecretForModule(ctx, "bare", "postgres", "superuser"); err == nil {
		t.Fatal("a secret was made for a machine that cannot open one")
	}
}

func TestACredentialGoesWhenTheConsumerStopsAskingForIt(t *testing.T) {
	// Withdrawal is the half nobody tests. A consumer that is unassigned must stop appearing in
	// what its provider is told to create, or the provider keeps a working login for a machine
	// that no longer uses it — and the provisioner's own rule about removing what nobody asks for
	// only fires if the mesh stops asking.
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.RegisterModule(ctx, catalogue.Manifest{
		Module: "meshboard", Version: "1", Requires: []string{"database"},
	}, Source{}); err != nil {
		t.Fatal(err)
	}
	if err := inv.Assign(ctx, "consumer", "meshboard"); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.SecretFor(ctx, "database", "consumer", "provider"); err != nil {
		t.Fatal(err)
	}

	issued, err := inv.SecretsFrom(ctx, "provider")
	if err != nil {
		t.Fatal(err)
	}
	if len(issued) != 1 {
		t.Fatalf("the provider was told about %d consumers", len(issued))
	}

	// Unassigned. The secret itself is deliberately NOT deleted here — see below — but nothing
	// should now resolve on that machine that wants it.
	if err := inv.Unassign(ctx, "consumer", "meshboard"); err != nil {
		t.Fatal(err)
	}
	assigned, err := inv.Assigned(ctx, "consumer")
	if err != nil {
		t.Fatal(err)
	}
	if len(assigned) != 0 {
		t.Fatalf("the module is still assigned: %v", assigned)
	}
}

func TestACredentialGoesWhenEitherMachineDoes(t *testing.T) {
	// The case that must not leave a live login behind: a machine removed from the mesh. Its
	// credentials go with it, and the provider stops being told to keep them.
	inv, ctx := twoNodesWithKeys(t)
	if _, err := inv.SecretFor(ctx, "database", "consumer", "provider"); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'consumer'`); err != nil {
		t.Fatal(err)
	}
	issued, err := inv.SecretsFrom(ctx, "provider")
	if err != nil {
		t.Fatal(err)
	}
	if len(issued) != 0 {
		t.Fatalf("a departed machine's credential is still granted: %+v", issued)
	}
}

// The other half of that, and the one that must not behave the same way.
//
// A secret the mesh made, it can make again — nothing else ever knew the old one. A secret the
// mesh was *given* it cannot: the broker knows a password, and the mesh inventing another puts 32
// random bytes where a working credential was. The machine applies it, reports success, and
// whatever reads it fails to authenticate somewhere else entirely, with the mesh insisting the
// secret was delivered — which it was.
func TestASecretTheMeshWasGivenIsNotReinventedWhenTheMachineRejoins(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "builder", Version: "1"},
		Source{}); err != nil {
		t.Fatal(err)
	}
	const url = "amqps://builder:the-password-the-broker-was-told@broker/"
	if err := inv.AcceptSecretForModule(ctx, "consumer", "builder", "broker", url); err != nil {
		t.Fatal(err)
	}
	node, err := inv.NodeByName(ctx, "consumer")
	if err != nil {
		t.Fatal(err)
	}
	fresh, _ := aSealingKey(t)
	if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil {
		t.Fatal(err)
	}

	_, err = inv.SecretForModule(ctx, "consumer", "builder", "broker")
	if err == nil {
		t.Fatal("the mesh invented a broker password, which the broker has never heard of")
	}
	// And says what to do about it, because the remedy is a command somebody runs and no amount
	// of pushing will produce one.
	if !strings.Contains(err.Error(), "issue it again") {
		t.Fatalf("refused without saying what would fix it: %v", err)
	}
}

// Until the key changes, a given secret is handed back unchanged — the ordinary case, and the one
// that would make the refusal above useless if it were wrong.
func TestASecretTheMeshWasGivenSurvivesAnOrdinaryPush(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "builder", Version: "1"},
		Source{}); err != nil {
		t.Fatal(err)
	}
	if err := inv.AcceptSecretForModule(ctx, "consumer", "builder", "broker",
		"amqps://builder:password@broker/"); err != nil {
		t.Fatal(err)
	}
	first, err := inv.SecretForModule(ctx, "consumer", "builder", "broker")
	if err != nil {
		t.Fatal(err)
	}
	second, err := inv.SecretForModule(ctx, "consumer", "builder", "broker")
	if err != nil {
		t.Fatal(err)
	}
	if first != second || first == "" {
		t.Fatal("a given secret changed between two pushes, so the machine was handed two")
	}
}