mesh-controller/internal/inventory/secrets_test.go

package inventory

import (
	"context"
	"crypto/ecdh"
	"crypto/rand"
	"encoding/base64"
	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/secrets"
	"strings"
	"testing"

	"golang.org/x/crypto/nacl/box"
)

// aSealingKey is a node's key, keeping the private half so a test can open what was sealed — the
// only assertion that actually distinguishes "the right blob" from "a blob".
func aSealingKey(t *testing.T) (string, func(string) ([]byte, bool)) {
	t.Helper()
	k, err := ecdh.X25519().GenerateKey(rand.Reader)
	if err != nil {
		t.Fatal(err)
	}
	var pub, priv [32]byte
	copy(pub[:], k.PublicKey().Bytes())
	copy(priv[:], k.Bytes())
	return base64.StdEncoding.EncodeToString(k.PublicKey().Bytes()),
		func(sealed string) ([]byte, bool) {
			blob, err := base64.StdEncoding.DecodeString(sealed)
			if err != nil {
				return nil, false
			}
			return box.OpenAnonymous(nil, blob, &pub, &priv)
		}
}

func twoNodesWithKeys(t *testing.T) (*Inventory, context.Context) {
	t.Helper()
	inv := fresh(t)
	ctx := context.Background()
	for _, n := range []string{"consumer", "provider"} {
		node, err := inv.AddNode(ctx, n)
		if err != nil {
			t.Fatal(err)
		}
		key, _ := aSealingKey(t)
		if err := inv.RecordSealingKey(ctx, node.ID, key); err != nil {
			t.Fatal(err)
		}
	}
	// A credential belongs to a module on a machine, so the modules holding one must exist
	// before it can (novox/hq 04-ISSUES/022). Two of them, because "two consumers on one node"
	// is the case that key exists for.
	for _, m := range []string{"gitea", "keycloak"} {
		// Each requires what a test delivers to it: a pair credential is refused for a
		// requirement the module does not have (novox/hq 04-ISSUES/078).
		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1",
			Requires: []string{"secret", "postgres-database", "object-store"},
			Secrets:  map[string]string{"secret": "/run/secret", "postgres-database": "/run/pg"}}, Source{}); err != nil {
			t.Fatal(err)
		}
	}
	return inv, ctx
}

func TestASecretIsMadeOnceAndKept(t *testing.T) {
	// Regenerating on every declaration would restart both ends on every push, and — worse — the
	// password a provider was told to create would never be the one its consumer was given.
	inv, ctx := twoNodesWithKeys(t)
	first, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider", "")
	if err != nil {
		t.Fatal(err)
	}
	second, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider", "")
	if err != nil {
		t.Fatal(err)
	}
	if first.ForConsumer != second.ForConsumer || first.ForProvider != second.ForProvider {
		t.Fatal("asking twice produced two different credentials")
	}
}

func TestTheStoredSecretIsNotTheSecret(t *testing.T) {
	// The whole point. A copy of this database is not a copy of the mesh's credentials — which is
	// what an encrypted column does not achieve, because whoever runs the control plane can read
	// through it.
	inv, ctx := twoNodesWithKeys(t)
	got, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider", "")
	if err != nil {
		t.Fatal(err)
	}
	var columns []string
	rows, err := inv.store.Pool().Query(ctx,
		`select column_name from information_schema.columns where table_name = 'secret'`)
	if err != nil {
		t.Fatal(err)
	}
	defer rows.Close()
	for rows.Next() {
		var c string
		if err := rows.Scan(&c); err != nil {
			t.Fatal(err)
		}
		columns = append(columns, c)
	}
	for _, c := range columns {
		if strings.Contains(c, "password") || strings.Contains(c, "value") ||
			strings.Contains(c, "plain") {
			t.Fatalf("the table has a column called %q, which suggests it holds the thing", c)
		}
	}
	if got.ForConsumer == got.ForProvider {
		t.Fatal("both ends were given the identical blob, so the storage reveals they match")
	}
}

func TestANewSealingKeyMeansANewSecret(t *testing.T) {
	// A node that rejoined generated a new key and can no longer open what was sealed to the old
	// one. Keeping the blob would deliver something unreadable for ever, reported as configured.
	inv, ctx := twoNodesWithKeys(t)
	before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider", "")
	if err != nil {
		t.Fatal(err)
	}
	node, err := inv.NodeByName(ctx, "consumer")
	if err != nil {
		t.Fatal(err)
	}
	fresh, _ := aSealingKey(t)
	if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil {
		t.Fatal(err)
	}
	after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider", "")
	if err != nil {
		t.Fatal(err)
	}
	if after.ForConsumer == before.ForConsumer {
		t.Fatal("the node was handed a credential sealed to a key it no longer has")
	}
	// And the provider's copy changed too, in the same breath. Otherwise the two ends hold
	// different passwords — which is the fanout window that makes rotation dangerous elsewhere.
	if after.ForProvider == before.ForProvider {
		t.Fatal("only one end was rotated, so the two now disagree")
	}
}

func TestRotatingReachesBothEnds(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider", "")
	if err != nil {
		t.Fatal(err)
	}
	if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "gitea", "provider", ""); err != nil {
		t.Fatal(err)
	}
	after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider", "")
	if err != nil {
		t.Fatal(err)
	}
	if after.ForConsumer == before.ForConsumer || after.ForProvider == before.ForProvider {
		t.Fatal("rotation left one of the ends holding what it had")
	}
}

func TestAProviderIsToldEveryCredentialItMustCreate(t *testing.T) {
	// The half that makes a credential real. A password nothing was told to create authenticates
	// nowhere, and the mesh cannot tell the provider what it is in any other way — it cannot read
	// it either.
	inv, ctx := twoNodesWithKeys(t)
	// The provider's own key, kept, so this asserts it can *open* what it was handed rather than
	// that the field is non-empty. Without that, selecting the wrong column reads the same both
	// ways and the test proves nothing — which it did, until the check was removed and it passed.
	providerKey, openProvider := aSealingKey(t)
	provider, err := inv.NodeByName(ctx, "provider")
	if err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordSealingKey(ctx, provider.ID, providerKey); err != nil {
		t.Fatal(err)
	}

	other, err := inv.AddNode(ctx, "second-consumer")
	if err != nil {
		t.Fatal(err)
	}
	secondKey, _ := aSealingKey(t)
	if err := inv.RecordSealingKey(ctx, other.ID, secondKey); err != nil {
		t.Fatal(err)
	}
	for _, who := range []string{"consumer", "second-consumer"} {
		if _, err := inv.SecretFor(ctx, "postgres-database", who, "gitea", "provider", ""); err != nil {
			t.Fatal(err)
		}
	}
	issued, err := inv.SecretsFrom(ctx, "provider")
	if err != nil {
		t.Fatal(err)
	}
	if len(issued) != 2 {
		t.Fatalf("the provider was told about %d of 2", len(issued))
	}
	for _, s := range issued {
		if _, ok := openProvider(s.ForProvider); !ok {
			t.Fatalf("the provider cannot open the credential it was given for %s", s.Consumer)
		}
		if s.ForConsumer != "" {
			// It has no business holding the other end's copy, and handing it out would put a
			// second readable-by-someone-else copy into circulation.
			t.Fatalf("the provider was handed the consumer's own copy of %s", s.Name)
		}
	}
}

func TestANodeWithNoSealingKeyCannotBeGivenASecret(t *testing.T) {
	inv := fresh(t)
	ctx := context.Background()
	for _, n := range []string{"consumer", "provider"} {
		if _, err := inv.AddNode(ctx, n); err != nil {
			t.Fatal(err)
		}
	}
	_, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider", "")
	if err == nil {
		t.Fatal("a credential was made for nodes that cannot open one")
	}
	if !strings.Contains(err.Error(), "sealing key") {
		t.Fatalf("the refusal does not say what is missing: %v", err)
	}
}

func TestSecretsGoWhenANodeLeaves(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider", ""); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'consumer'`); err != nil {
		t.Fatal(err)
	}
	var left int
	if err := inv.store.Pool().QueryRow(ctx, `select count(*) from secret`).Scan(&left); err != nil {
		t.Fatal(err)
	}
	if left != 0 {
		t.Fatalf("%d credential(s) outlived the machine they were for", left)
	}
}

func TestAModulesOwnSecretIsPerMachineAndKept(t *testing.T) {
	// A module running on three machines has three passwords. One in the manifest instead would
	// put the same secret on every machine that ever runs it, in a file anybody can read.
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
		Source{}); err != nil {
		t.Fatal(err)
	}

	here, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
	if err != nil {
		t.Fatal(err)
	}
	there, err := inv.SecretForModule(ctx, "provider", "postgres", "superuser")
	if err != nil {
		t.Fatal(err)
	}
	if here == there {
		t.Fatal("two machines were given the same secret")
	}

	// Made once and kept, or a running database would be handed a password it was not started
	// with on the next declaration.
	again, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
	if err != nil {
		t.Fatal(err)
	}
	if again != here {
		t.Fatal("asking twice made a second secret")
	}
}

func TestTwoNeedsInOneModuleAreTwoSecrets(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
		Source{}); err != nil {
		t.Fatal(err)
	}
	one, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
	if err != nil {
		t.Fatal(err)
	}
	two, err := inv.SecretForModule(ctx, "consumer", "postgres", "replication")
	if err != nil {
		t.Fatal(err)
	}
	if one == two {
		t.Fatal("two names gave one secret")
	}
}

func TestAModulesSecretIsRemadeWhenTheMachineRejoins(t *testing.T) {
	// The node generated a new sealing key and can no longer open what was sealed to the old one.
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
		Source{}); err != nil {
		t.Fatal(err)
	}
	before, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
	if err != nil {
		t.Fatal(err)
	}
	node, err := inv.NodeByName(ctx, "consumer")
	if err != nil {
		t.Fatal(err)
	}
	fresh, _ := aSealingKey(t)
	if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil {
		t.Fatal(err)
	}
	after, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser")
	if err != nil {
		t.Fatal(err)
	}
	if after == before {
		t.Fatal("a machine was handed a secret sealed to a key it no longer has")
	}
}

func TestAMachineWithNoSealingKeyCannotBeGivenAModuleSecret(t *testing.T) {
	inv := fresh(t)
	ctx := context.Background()
	if _, err := inv.AddNode(ctx, "bare"); err != nil {
		t.Fatal(err)
	}
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"},
		Source{}); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.SecretForModule(ctx, "bare", "postgres", "superuser"); err == nil {
		t.Fatal("a secret was made for a machine that cannot open one")
	}
}

func TestACredentialGoesWhenTheConsumerStopsAskingForIt(t *testing.T) {
	// Withdrawal is the half nobody tests. A consumer that is unassigned must stop appearing in
	// what its provider is told to create, or the provider keeps a working login for a machine
	// that no longer uses it — and the provisioner's own rule about removing what nobody asks for
	// only fires if the mesh stops asking.
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.RegisterModule(ctx, catalogue.Manifest{
		Module: "meshboard", Version: "1", Requires: []string{"postgres-database"},
	}, Source{}); err != nil {
		t.Fatal(err)
	}
	if err := inv.Assign(ctx, "consumer", "meshboard"); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider", ""); err != nil {
		t.Fatal(err)
	}

	issued, err := inv.SecretsFrom(ctx, "provider")
	if err != nil {
		t.Fatal(err)
	}
	if len(issued) != 1 {
		t.Fatalf("the provider was told about %d consumers", len(issued))
	}

	// Unassigned. The secret itself is deliberately NOT deleted here — see below — but nothing
	// should now resolve on that machine that wants it.
	if err := inv.Unassign(ctx, "consumer", "meshboard"); err != nil {
		t.Fatal(err)
	}
	assigned, err := inv.Assigned(ctx, "consumer")
	if err != nil {
		t.Fatal(err)
	}
	if len(assigned) != 0 {
		t.Fatalf("the module is still assigned: %v", assigned)
	}
}

func TestACredentialGoesWhenEitherMachineDoes(t *testing.T) {
	// The case that must not leave a live login behind: a machine removed from the mesh. Its
	// credentials go with it, and the provider stops being told to keep them.
	inv, ctx := twoNodesWithKeys(t)
	if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider", ""); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'consumer'`); err != nil {
		t.Fatal(err)
	}
	issued, err := inv.SecretsFrom(ctx, "provider")
	if err != nil {
		t.Fatal(err)
	}
	if len(issued) != 0 {
		t.Fatalf("a departed machine's credential is still granted: %+v", issued)
	}
}

// The other half of that, and the one that must not behave the same way.
//
// A secret the mesh made, it can make again — nothing else ever knew the old one. A secret the
// mesh was *given* it cannot: the broker knows a password, and the mesh inventing another puts 32
// random bytes where a working credential was. The machine applies it, reports success, and
// whatever reads it fails to authenticate somewhere else entirely, with the mesh insisting the
// secret was delivered — which it was.
func TestASecretTheMeshWasGivenIsNotReinventedWhenTheMachineRejoins(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "builder", Version: "1",
		OwnSecrets: map[string]string{"broker": "/run/broker"}}, Source{}); err != nil {
		t.Fatal(err)
	}
	const url = "amqps://builder:the-password-the-broker-was-told@broker/"
	if err := inv.AcceptSecretForModule(ctx, "consumer", "builder", "broker", url); err != nil {
		t.Fatal(err)
	}
	node, err := inv.NodeByName(ctx, "consumer")
	if err != nil {
		t.Fatal(err)
	}
	fresh, _ := aSealingKey(t)
	if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil {
		t.Fatal(err)
	}

	_, err = inv.SecretForModule(ctx, "consumer", "builder", "broker")
	if err == nil {
		t.Fatal("the mesh invented a broker password, which the broker has never heard of")
	}
	// And says what to do about it, because the remedy is a command somebody runs and no amount
	// of pushing will produce one.
	if !strings.Contains(err.Error(), "issue it again") {
		t.Fatalf("refused without saying what would fix it: %v", err)
	}
}

// Until the key changes, a given secret is handed back unchanged — the ordinary case, and the one
// that would make the refusal above useless if it were wrong.
func TestASecretTheMeshWasGivenSurvivesAnOrdinaryPush(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "builder", Version: "1",
		OwnSecrets: map[string]string{"broker": "/run/broker"}}, Source{}); err != nil {
		t.Fatal(err)
	}
	if err := inv.AcceptSecretForModule(ctx, "consumer", "builder", "broker",
		"amqps://builder:password@broker/"); err != nil {
		t.Fatal(err)
	}
	first, err := inv.SecretForModule(ctx, "consumer", "builder", "broker")
	if err != nil {
		t.Fatal(err)
	}
	second, err := inv.SecretForModule(ctx, "consumer", "builder", "broker")
	if err != nil {
		t.Fatal(err)
	}
	if first != second || first == "" {
		t.Fatal("a given secret changed between two pushes, so the machine was handed two")
	}
}

// Rotation has to know who holds the old credential, and that was the half HAL could not answer.
//
// There a provision had one shared credential; rotating it updated the provider's row and nothing
// enumerated the consumers, so three nodes carried dead credentials for two days while the mesh
// reported success (novox/hq ADR 0001). Here the holders are a set, and this is the query that
// makes "every consumer" nameable rather than hopeful.
func TestEveryHolderOfACredentialCanBeNamed(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	third, err := inv.AddNode(ctx, "third")
	if err != nil {
		t.Fatal(err)
	}
	key, _ := aSealingKey(t)
	if err := inv.RecordSealingKey(ctx, third.ID, key); err != nil {
		t.Fatal(err)
	}
	for _, consumer := range []string{"consumer", "third"} {
		if _, err := inv.SecretFor(ctx, "postgres-database", consumer, "gitea", "provider", ""); err != nil {
			t.Fatal(err)
		}
	}
	// And one for a different provision, which must not be swept up.
	if _, err := inv.SecretFor(ctx, "cache", "consumer", "gitea", "provider", ""); err != nil {
		t.Fatal(err)
	}

	holders, err := inv.HoldersOf(ctx, "postgres-database", "")
	if err != nil {
		t.Fatal(err)
	}
	if len(holders) != 2 {
		t.Fatalf("a holder of the credential was not named: %+v", holders)
	}
	for _, h := range holders {
		if h.Provision != "postgres-database" {
			t.Fatalf("rotating one provision would have touched %q", h.Provision)
		}
	}

	// One machine's, when that is what was asked for. Rotating the other nine because one is
	// suspected is a great deal of disruption for one suspicion.
	one, err := inv.HoldersOf(ctx, "postgres-database", "third")
	if err != nil {
		t.Fatal(err)
	}
	if len(one) != 1 || one[0].Consumer != "third" {
		t.Fatalf("asking for one machine's holder gave %+v", one)
	}
}

// And rotating gives both ends a new credential, together — the same one.
func TestRotatingGivesBothEndsTheSameNewCredential(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider", "")
	if err != nil {
		t.Fatal(err)
	}
	if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "gitea", "provider", ""); err != nil {
		t.Fatal(err)
	}
	after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider", "")
	if err != nil {
		t.Fatal(err)
	}
	if after.ForConsumer == before.ForConsumer {
		t.Fatal("the consumer was handed the credential that was just rotated away")
	}
	if after.ForProvider == before.ForProvider {
		t.Fatal("the provider was left creating the old password, which is the fault exactly")
	}
	// The two halves are the same secret sealed twice, which is the whole point: a provider
	// creating one password and a consumer given another is a mesh that reports success and
	// cannot connect.
	if after.ForConsumer == after.ForProvider {
		t.Fatal("both ends were sealed identically, so one of them cannot open it")
	}

	// Rotating one pair leaves every other holder alone. Otherwise "rotate this machine's
	// credential" is a mesh-wide outage with a narrow name.
	third, err := inv.AddNode(ctx, "third")
	if err != nil {
		t.Fatal(err)
	}
	key, _ := aSealingKey(t)
	if err := inv.RecordSealingKey(ctx, third.ID, key); err != nil {
		t.Fatal(err)
	}
	untouched, err := inv.SecretFor(ctx, "postgres-database", "third", "gitea", "provider", "")
	if err != nil {
		t.Fatal(err)
	}
	if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "gitea", "provider", ""); err != nil {
		t.Fatal(err)
	}
	again, err := inv.SecretFor(ctx, "postgres-database", "third", "gitea", "provider", "")
	if err != nil {
		t.Fatal(err)
	}
	if again.ForConsumer != untouched.ForConsumer {
		t.Fatal("rotating one machine's credential changed another machine's")
	}
}

// **A person can deliver a pair credential** (novox/hq 04-ISSUES/070, ADR 0092): the vault's
// third species, a value for something outside the mesh. It is sealed to both ends like a made
// one; what differs is that the mesh will neither replace it with one of its own nor rotate it,
// because it cannot make the replacement.
func TestAnAcceptedPairCredentialIsKeptAndNeverRemade(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.AcceptSecretForPair(ctx, "secret", "consumer", "gitea", "provider", "", "hunter2"); err != nil {
		t.Fatal(err)
	}
	got, err := inv.SecretFor(ctx, "secret", "consumer", "gitea", "provider", "")
	if err != nil {
		t.Fatal(err)
	}
	if got.Origin != OriginAccepted {
		t.Fatalf("an accepted credential reads back as %q", got.Origin)
	}
	again, err := inv.SecretFor(ctx, "secret", "consumer", "gitea", "provider", "")
	if err != nil {
		t.Fatal(err)
	}
	if again.ForConsumer != got.ForConsumer || again.ForProvider != got.ForProvider {
		t.Fatal("reading an accepted credential twice produced two different values")
	}

	// Rotation is refused, and says what to do instead.
	err = inv.RotateSecret(ctx, "secret", "consumer", "gitea", "provider", "")
	if err == nil || !strings.Contains(err.Error(), "secret accept") {
		t.Fatalf("rotating an accepted credential was not refused with the remedy: %v", err)
	}
	// And a made one still rotates.
	if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider", ""); err != nil {
		t.Fatal(err)
	}
	if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "gitea", "provider", ""); err != nil {
		t.Fatalf("a made credential no longer rotates: %v", err)
	}
}

// A key that changed at either end makes the accepted value unreadable there, and the mesh cannot
// re-seal what it does not hold: refused aloud, never quietly replaced by a minted one.
func TestAnAcceptedPairCredentialIsNotRemadeWhenAKeyChanges(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.AcceptSecretForPair(ctx, "secret", "consumer", "gitea", "provider", "", "hunter2"); err != nil {
		t.Fatal(err)
	}
	node, err := inv.NodeByName(ctx, "consumer")
	if err != nil {
		t.Fatal(err)
	}
	fresh, _ := aSealingKey(t)
	if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil {
		t.Fatal(err)
	}
	_, err = inv.SecretFor(ctx, "secret", "consumer", "gitea", "provider", "")
	if err == nil || !strings.Contains(err.Error(), "accept it again") {
		t.Fatalf("an accepted credential was remade, or refused without the remedy: %v", err)
	}
	// Accepting it again is the remedy, and it works.
	if err := inv.AcceptSecretForPair(ctx, "secret", "consumer", "gitea", "provider", "", "hunter3"); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.SecretFor(ctx, "secret", "consumer", "gitea", "provider", ""); err != nil {
		t.Fatal(err)
	}
}

// Two secrets from one provider to one module are two credentials (novox/hq 04-ISSUES/069, ADR
// 0094): keyed on the local name, made and rotated apart, and listed apart for the provider.
func TestTwoLocalNamesAreTwoCredentials(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	key, err := inv.SecretFor(ctx, "secret", "consumer", "gitea", "provider", "root-key")
	if err != nil {
		t.Fatal(err)
	}
	pass, err := inv.SecretFor(ctx, "secret", "consumer", "gitea", "provider", "root-pass")
	if err != nil {
		t.Fatal(err)
	}
	if key.ForConsumer == pass.ForConsumer {
		t.Fatal("two local names were given one credential")
	}
	if err := inv.RotateSecret(ctx, "secret", "consumer", "gitea", "provider", "root-key"); err != nil {
		t.Fatal(err)
	}
	keyAgain, err := inv.SecretFor(ctx, "secret", "consumer", "gitea", "provider", "root-key")
	if err != nil {
		t.Fatal(err)
	}
	passAgain, err := inv.SecretFor(ctx, "secret", "consumer", "gitea", "provider", "root-pass")
	if err != nil {
		t.Fatal(err)
	}
	if keyAgain.ForConsumer == key.ForConsumer || passAgain.ForConsumer != pass.ForConsumer {
		t.Fatal("rotating one local name touched the other, or neither")
	}
	holders, err := inv.HoldersOf(ctx, "secret", "")
	if err != nil {
		t.Fatal(err)
	}
	var locals []string
	for _, h := range holders {
		locals = append(locals, h.Local)
	}
	if strings.Join(locals, ",") != "root-key,root-pass" {
		t.Fatalf("the holders are listed apart, by local name: %v", holders)
	}
	from, err := inv.SecretsFrom(ctx, "provider")
	if err != nil {
		t.Fatal(err)
	}
	if len(from) != 2 || from[0].Local == from[1].Local {
		t.Fatalf("the provider is told two credentials to create: %+v", from)
	}
}

// The operator can recover either of two local names apart (review C3).
func TestTheOperatorRecoversEachLocalNameApart(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	pub, _, err := secrets.Keypair()
	if err != nil {
		t.Fatal(err)
	}
	if _, err := inv.SetOperatorKey(ctx, pub); err != nil {
		t.Fatal(err)
	}
	for _, local := range []string{"root-key", "root-pass"} {
		if _, err := inv.SecretFor(ctx, "secret", "consumer", "gitea", "provider", local); err != nil {
			t.Fatal(err)
		}
	}
	key, err := inv.KeptSecret(ctx, "consumer", "gitea", "secret", "provider", "root-key")
	if err != nil {
		t.Fatal(err)
	}
	pass, err := inv.KeptSecret(ctx, "consumer", "gitea", "secret", "provider", "root-pass")
	if err != nil {
		t.Fatal(err)
	}
	if key.Local != "root-key" || pass.Local != "root-pass" || key.Kind != "pair" {
		t.Fatalf("recovery does not tell the two apart: %+v %+v", key, pass)
	}
	kept, _, _, err := inv.KeptForOperator(ctx)
	if err != nil {
		t.Fatal(err)
	}
	var locals []string
	for _, k := range kept {
		if k.Kind == "pair" {
			locals = append(locals, k.Local)
		}
	}
	if strings.Join(locals, ",") != "root-key,root-pass" {
		t.Fatalf("the export does not name the local names: %v", kept)
	}
}

// **A delivered secret is accepted only under a name the module declares** (novox/hq
// 04-ISSUES/078). A value stored under a name nothing reads is a delivery that changed nothing
// and reported success; refused, naming what the module does declare.
func TestADeliveredSecretIsRefusedUnderANameTheModuleDoesNotDeclare(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "step-ca", Version: "1",
		OwnSecrets: map[string]string{"password": "/run/password"}}, Source{}); err != nil {
		t.Fatal(err)
	}
	err := inv.AcceptSecretForModule(ctx, "consumer", "step-ca", "root-key", "not-a-key")
	if err == nil {
		t.Fatal("a secret delivered under a name the module does not declare was accepted")
	}
	for _, want := range []string{"root-key", "password"} {
		if !strings.Contains(err.Error(), want) {
			t.Errorf("the refusal does not say %q: %v", want, err)
		}
	}
	if err := inv.AcceptSecretForModule(ctx, "consumer", "step-ca", "password", "hunter2"); err != nil {
		t.Fatalf("a secret delivered under a declared name was refused: %v", err)
	}
	// A module the mesh does not know is said, not stored.
	err = inv.AcceptSecretForModule(ctx, "consumer", "nobody", "password", "hunter2")
	if err == nil || !strings.Contains(err.Error(), "module add") {
		t.Errorf("a delivery to an unknown module was not refused with the remedy: %v", err)
	}
}

func TestADeliveredPairCredentialIsRefusedForARequirementTheModuleDoesNotHave(t *testing.T) {
	inv, ctx := twoNodesWithKeys(t)
	// gitea requires secret, postgres-database and object-store (the fixture); it keeps a secret
	// for the first two only, and requires no cache at all.
	err := inv.AcceptSecretForPair(ctx, "redis-cache", "consumer", "gitea", "provider", "", "hunter2")
	if err == nil || !strings.Contains(err.Error(), "postgres-database") {
		t.Fatalf("a pair credential for a requirement the module does not have was not refused naming what it requires: %v", err)
	}
	err = inv.AcceptSecretForPair(ctx, "object-store", "consumer", "gitea", "provider", "", "hunter2")
	if err == nil || !strings.Contains(err.Error(), "keeps no secret") {
		t.Fatalf("a pair credential for a requirement the module keeps no secret for was not refused: %v", err)
	}
	err = inv.AcceptSecretForPair(ctx, "secret", "consumer", "gitea", "provider", "extra", "hunter2")
	if err == nil || !strings.Contains(err.Error(), "drop --local") {
		t.Fatalf("a local named where the module keeps one secret was not refused: %v", err)
	}
	if err := inv.AcceptSecretForPair(ctx, "secret", "consumer", "gitea", "provider", "", "hunter2"); err != nil {
		t.Fatalf("a delivery for a requirement the module keeps one secret for was refused: %v", err)
	}
	// A module keeping several secrets for one requirement (ADR 0094) takes a delivery only
	// under one of its locals.
	m := catalogue.Manifest{Module: "mailu", Version: "1", Requires: []string{"secret"},
		SecretsMany: map[string]map[string]string{"secret": {"admin": "/run/admin", "api-token": "/run/api-token"}}}
	if err := inv.RegisterModule(ctx, m, Source{}); err != nil {
		t.Fatal(err)
	}
	err = inv.AcceptSecretForPair(ctx, "secret", "consumer", "mailu", "provider", "", "hunter2")
	if err == nil || !strings.Contains(err.Error(), "--local") {
		t.Errorf("a delivery to a module keeping several secrets, with no local named, was not refused: %v", err)
	}
	err = inv.AcceptSecretForPair(ctx, "secret", "consumer", "mailu", "provider", "secret-key", "hunter2")
	if err == nil || !strings.Contains(err.Error(), "api-token") {
		t.Errorf("a delivery under a local the module does not keep was not refused naming the ones it keeps: %v", err)
	}
	if err := inv.AcceptSecretForPair(ctx, "secret", "consumer", "mailu", "provider", "admin", "hunter2"); err != nil {
		t.Errorf("a delivery under a kept local was refused: %v", err)
	}
}