package inventory import ( "context" "crypto/ecdh" "crypto/rand" "encoding/base64" "github.com/novox/mesh-control/internal/catalogue" "strings" "testing" "golang.org/x/crypto/nacl/box" ) // aSealingKey is a node's key, keeping the private half so a test can open what was sealed — the // only assertion that actually distinguishes "the right blob" from "a blob". func aSealingKey(t *testing.T) (string, func(string) ([]byte, bool)) { t.Helper() k, err := ecdh.X25519().GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } var pub, priv [32]byte copy(pub[:], k.PublicKey().Bytes()) copy(priv[:], k.Bytes()) return base64.StdEncoding.EncodeToString(k.PublicKey().Bytes()), func(sealed string) ([]byte, bool) { blob, err := base64.StdEncoding.DecodeString(sealed) if err != nil { return nil, false } return box.OpenAnonymous(nil, blob, &pub, &priv) } } func twoNodesWithKeys(t *testing.T) (*Inventory, context.Context) { t.Helper() inv := fresh(t) ctx := context.Background() for _, n := range []string{"consumer", "provider"} { node, err := inv.AddNode(ctx, n) if err != nil { t.Fatal(err) } key, _ := aSealingKey(t) if err := inv.RecordSealingKey(ctx, node.ID, key); err != nil { t.Fatal(err) } } // A credential belongs to a module on a machine, so the modules holding one must exist // before it can (novox/hq 04-ISSUES/022). Two of them, because "two consumers on one node" // is the case that key exists for. for _, m := range []string{"gitea", "keycloak"} { if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, Source{}); err != nil { t.Fatal(err) } } return inv, ctx } func TestASecretIsMadeOnceAndKept(t *testing.T) { // Regenerating on every declaration would restart both ends on every push, and — worse — the // password a provider was told to create would never be the one its consumer was given. inv, ctx := twoNodesWithKeys(t) first, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider") if err != nil { t.Fatal(err) } second, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider") if err != nil { t.Fatal(err) } if first.ForConsumer != second.ForConsumer || first.ForProvider != second.ForProvider { t.Fatal("asking twice produced two different credentials") } } func TestTheStoredSecretIsNotTheSecret(t *testing.T) { // The whole point. A copy of this database is not a copy of the mesh's credentials — which is // what an encrypted column does not achieve, because whoever runs the control plane can read // through it. inv, ctx := twoNodesWithKeys(t) got, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider") if err != nil { t.Fatal(err) } var columns []string rows, err := inv.store.Pool().Query(ctx, `select column_name from information_schema.columns where table_name = 'secret'`) if err != nil { t.Fatal(err) } defer rows.Close() for rows.Next() { var c string if err := rows.Scan(&c); err != nil { t.Fatal(err) } columns = append(columns, c) } for _, c := range columns { if strings.Contains(c, "password") || strings.Contains(c, "value") || strings.Contains(c, "plain") { t.Fatalf("the table has a column called %q, which suggests it holds the thing", c) } } if got.ForConsumer == got.ForProvider { t.Fatal("both ends were given the identical blob, so the storage reveals they match") } } func TestANewSealingKeyMeansANewSecret(t *testing.T) { // A node that rejoined generated a new key and can no longer open what was sealed to the old // one. Keeping the blob would deliver something unreadable for ever, reported as configured. inv, ctx := twoNodesWithKeys(t) before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider") if err != nil { t.Fatal(err) } node, err := inv.NodeByName(ctx, "consumer") if err != nil { t.Fatal(err) } fresh, _ := aSealingKey(t) if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil { t.Fatal(err) } after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider") if err != nil { t.Fatal(err) } if after.ForConsumer == before.ForConsumer { t.Fatal("the node was handed a credential sealed to a key it no longer has") } // And the provider's copy changed too, in the same breath. Otherwise the two ends hold // different passwords — which is the fanout window that makes rotation dangerous elsewhere. if after.ForProvider == before.ForProvider { t.Fatal("only one end was rotated, so the two now disagree") } } func TestRotatingReachesBothEnds(t *testing.T) { inv, ctx := twoNodesWithKeys(t) before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider") if err != nil { t.Fatal(err) } if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "gitea", "provider"); err != nil { t.Fatal(err) } after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider") if err != nil { t.Fatal(err) } if after.ForConsumer == before.ForConsumer || after.ForProvider == before.ForProvider { t.Fatal("rotation left one of the ends holding what it had") } } func TestAProviderIsToldEveryCredentialItMustCreate(t *testing.T) { // The half that makes a credential real. A password nothing was told to create authenticates // nowhere, and the mesh cannot tell the provider what it is in any other way — it cannot read // it either. inv, ctx := twoNodesWithKeys(t) // The provider's own key, kept, so this asserts it can *open* what it was handed rather than // that the field is non-empty. Without that, selecting the wrong column reads the same both // ways and the test proves nothing — which it did, until the check was removed and it passed. providerKey, openProvider := aSealingKey(t) provider, err := inv.NodeByName(ctx, "provider") if err != nil { t.Fatal(err) } if err := inv.RecordSealingKey(ctx, provider.ID, providerKey); err != nil { t.Fatal(err) } other, err := inv.AddNode(ctx, "second-consumer") if err != nil { t.Fatal(err) } secondKey, _ := aSealingKey(t) if err := inv.RecordSealingKey(ctx, other.ID, secondKey); err != nil { t.Fatal(err) } for _, who := range []string{"consumer", "second-consumer"} { if _, err := inv.SecretFor(ctx, "postgres-database", who, "gitea", "provider"); err != nil { t.Fatal(err) } } issued, err := inv.SecretsFrom(ctx, "provider") if err != nil { t.Fatal(err) } if len(issued) != 2 { t.Fatalf("the provider was told about %d of 2", len(issued)) } for _, s := range issued { if _, ok := openProvider(s.ForProvider); !ok { t.Fatalf("the provider cannot open the credential it was given for %s", s.Consumer) } if s.ForConsumer != "" { // It has no business holding the other end's copy, and handing it out would put a // second readable-by-someone-else copy into circulation. t.Fatalf("the provider was handed the consumer's own copy of %s", s.Name) } } } func TestANodeWithNoSealingKeyCannotBeGivenASecret(t *testing.T) { inv := fresh(t) ctx := context.Background() for _, n := range []string{"consumer", "provider"} { if _, err := inv.AddNode(ctx, n); err != nil { t.Fatal(err) } } _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider") if err == nil { t.Fatal("a credential was made for nodes that cannot open one") } if !strings.Contains(err.Error(), "sealing key") { t.Fatalf("the refusal does not say what is missing: %v", err) } } func TestSecretsGoWhenANodeLeaves(t *testing.T) { inv, ctx := twoNodesWithKeys(t) if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider"); err != nil { t.Fatal(err) } if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'consumer'`); err != nil { t.Fatal(err) } var left int if err := inv.store.Pool().QueryRow(ctx, `select count(*) from secret`).Scan(&left); err != nil { t.Fatal(err) } if left != 0 { t.Fatalf("%d credential(s) outlived the machine they were for", left) } } func TestAModulesOwnSecretIsPerMachineAndKept(t *testing.T) { // A module running on three machines has three passwords. One in the manifest instead would // put the same secret on every machine that ever runs it, in a file anybody can read. inv, ctx := twoNodesWithKeys(t) if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"}, Source{}); err != nil { t.Fatal(err) } here, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser") if err != nil { t.Fatal(err) } there, err := inv.SecretForModule(ctx, "provider", "postgres", "superuser") if err != nil { t.Fatal(err) } if here == there { t.Fatal("two machines were given the same secret") } // Made once and kept, or a running database would be handed a password it was not started // with on the next declaration. again, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser") if err != nil { t.Fatal(err) } if again != here { t.Fatal("asking twice made a second secret") } } func TestTwoNeedsInOneModuleAreTwoSecrets(t *testing.T) { inv, ctx := twoNodesWithKeys(t) if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"}, Source{}); err != nil { t.Fatal(err) } one, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser") if err != nil { t.Fatal(err) } two, err := inv.SecretForModule(ctx, "consumer", "postgres", "replication") if err != nil { t.Fatal(err) } if one == two { t.Fatal("two names gave one secret") } } func TestAModulesSecretIsRemadeWhenTheMachineRejoins(t *testing.T) { // The node generated a new sealing key and can no longer open what was sealed to the old one. inv, ctx := twoNodesWithKeys(t) if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"}, Source{}); err != nil { t.Fatal(err) } before, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser") if err != nil { t.Fatal(err) } node, err := inv.NodeByName(ctx, "consumer") if err != nil { t.Fatal(err) } fresh, _ := aSealingKey(t) if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil { t.Fatal(err) } after, err := inv.SecretForModule(ctx, "consumer", "postgres", "superuser") if err != nil { t.Fatal(err) } if after == before { t.Fatal("a machine was handed a secret sealed to a key it no longer has") } } func TestAMachineWithNoSealingKeyCannotBeGivenAModuleSecret(t *testing.T) { inv := fresh(t) ctx := context.Background() if _, err := inv.AddNode(ctx, "bare"); err != nil { t.Fatal(err) } if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "1"}, Source{}); err != nil { t.Fatal(err) } if _, err := inv.SecretForModule(ctx, "bare", "postgres", "superuser"); err == nil { t.Fatal("a secret was made for a machine that cannot open one") } } func TestACredentialGoesWhenTheConsumerStopsAskingForIt(t *testing.T) { // Withdrawal is the half nobody tests. A consumer that is unassigned must stop appearing in // what its provider is told to create, or the provider keeps a working login for a machine // that no longer uses it — and the provisioner's own rule about removing what nobody asks for // only fires if the mesh stops asking. inv, ctx := twoNodesWithKeys(t) if err := inv.RegisterModule(ctx, catalogue.Manifest{ Module: "meshboard", Version: "1", Requires: []string{"postgres-database"}, }, Source{}); err != nil { t.Fatal(err) } if err := inv.Assign(ctx, "consumer", "meshboard"); err != nil { t.Fatal(err) } if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider"); err != nil { t.Fatal(err) } issued, err := inv.SecretsFrom(ctx, "provider") if err != nil { t.Fatal(err) } if len(issued) != 1 { t.Fatalf("the provider was told about %d consumers", len(issued)) } // Unassigned. The secret itself is deliberately NOT deleted here — see below — but nothing // should now resolve on that machine that wants it. if err := inv.Unassign(ctx, "consumer", "meshboard"); err != nil { t.Fatal(err) } assigned, err := inv.Assigned(ctx, "consumer") if err != nil { t.Fatal(err) } if len(assigned) != 0 { t.Fatalf("the module is still assigned: %v", assigned) } } func TestACredentialGoesWhenEitherMachineDoes(t *testing.T) { // The case that must not leave a live login behind: a machine removed from the mesh. Its // credentials go with it, and the provider stops being told to keep them. inv, ctx := twoNodesWithKeys(t) if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider"); err != nil { t.Fatal(err) } if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'consumer'`); err != nil { t.Fatal(err) } issued, err := inv.SecretsFrom(ctx, "provider") if err != nil { t.Fatal(err) } if len(issued) != 0 { t.Fatalf("a departed machine's credential is still granted: %+v", issued) } } // The other half of that, and the one that must not behave the same way. // // A secret the mesh made, it can make again — nothing else ever knew the old one. A secret the // mesh was *given* it cannot: the broker knows a password, and the mesh inventing another puts 32 // random bytes where a working credential was. The machine applies it, reports success, and // whatever reads it fails to authenticate somewhere else entirely, with the mesh insisting the // secret was delivered — which it was. func TestASecretTheMeshWasGivenIsNotReinventedWhenTheMachineRejoins(t *testing.T) { inv, ctx := twoNodesWithKeys(t) if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "builder", Version: "1"}, Source{}); err != nil { t.Fatal(err) } const url = "amqps://builder:the-password-the-broker-was-told@broker/" if err := inv.AcceptSecretForModule(ctx, "consumer", "builder", "broker", url); err != nil { t.Fatal(err) } node, err := inv.NodeByName(ctx, "consumer") if err != nil { t.Fatal(err) } fresh, _ := aSealingKey(t) if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil { t.Fatal(err) } _, err = inv.SecretForModule(ctx, "consumer", "builder", "broker") if err == nil { t.Fatal("the mesh invented a broker password, which the broker has never heard of") } // And says what to do about it, because the remedy is a command somebody runs and no amount // of pushing will produce one. if !strings.Contains(err.Error(), "issue it again") { t.Fatalf("refused without saying what would fix it: %v", err) } } // Until the key changes, a given secret is handed back unchanged — the ordinary case, and the one // that would make the refusal above useless if it were wrong. func TestASecretTheMeshWasGivenSurvivesAnOrdinaryPush(t *testing.T) { inv, ctx := twoNodesWithKeys(t) if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "builder", Version: "1"}, Source{}); err != nil { t.Fatal(err) } if err := inv.AcceptSecretForModule(ctx, "consumer", "builder", "broker", "amqps://builder:password@broker/"); err != nil { t.Fatal(err) } first, err := inv.SecretForModule(ctx, "consumer", "builder", "broker") if err != nil { t.Fatal(err) } second, err := inv.SecretForModule(ctx, "consumer", "builder", "broker") if err != nil { t.Fatal(err) } if first != second || first == "" { t.Fatal("a given secret changed between two pushes, so the machine was handed two") } } // Rotation has to know who holds the old credential, and that was the half HAL could not answer. // // There a provision had one shared credential; rotating it updated the provider's row and nothing // enumerated the consumers, so three nodes carried dead credentials for two days while the mesh // reported success (novox/hq ADR 0001). Here the holders are a set, and this is the query that // makes "every consumer" nameable rather than hopeful. func TestEveryHolderOfACredentialCanBeNamed(t *testing.T) { inv, ctx := twoNodesWithKeys(t) third, err := inv.AddNode(ctx, "third") if err != nil { t.Fatal(err) } key, _ := aSealingKey(t) if err := inv.RecordSealingKey(ctx, third.ID, key); err != nil { t.Fatal(err) } for _, consumer := range []string{"consumer", "third"} { if _, err := inv.SecretFor(ctx, "postgres-database", consumer, "gitea", "provider"); err != nil { t.Fatal(err) } } // And one for a different provision, which must not be swept up. if _, err := inv.SecretFor(ctx, "cache", "consumer", "gitea", "provider"); err != nil { t.Fatal(err) } holders, err := inv.HoldersOf(ctx, "postgres-database", "") if err != nil { t.Fatal(err) } if len(holders) != 2 { t.Fatalf("a holder of the credential was not named: %+v", holders) } for _, h := range holders { if h.Provision != "postgres-database" { t.Fatalf("rotating one provision would have touched %q", h.Provision) } } // One machine's, when that is what was asked for. Rotating the other nine because one is // suspected is a great deal of disruption for one suspicion. one, err := inv.HoldersOf(ctx, "postgres-database", "third") if err != nil { t.Fatal(err) } if len(one) != 1 || one[0].Consumer != "third" { t.Fatalf("asking for one machine's holder gave %+v", one) } } // And rotating gives both ends a new credential, together — the same one. func TestRotatingGivesBothEndsTheSameNewCredential(t *testing.T) { inv, ctx := twoNodesWithKeys(t) before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider") if err != nil { t.Fatal(err) } if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "gitea", "provider"); err != nil { t.Fatal(err) } after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider") if err != nil { t.Fatal(err) } if after.ForConsumer == before.ForConsumer { t.Fatal("the consumer was handed the credential that was just rotated away") } if after.ForProvider == before.ForProvider { t.Fatal("the provider was left creating the old password, which is the fault exactly") } // The two halves are the same secret sealed twice, which is the whole point: a provider // creating one password and a consumer given another is a mesh that reports success and // cannot connect. if after.ForConsumer == after.ForProvider { t.Fatal("both ends were sealed identically, so one of them cannot open it") } // Rotating one pair leaves every other holder alone. Otherwise "rotate this machine's // credential" is a mesh-wide outage with a narrow name. third, err := inv.AddNode(ctx, "third") if err != nil { t.Fatal(err) } key, _ := aSealingKey(t) if err := inv.RecordSealingKey(ctx, third.ID, key); err != nil { t.Fatal(err) } untouched, err := inv.SecretFor(ctx, "postgres-database", "third", "gitea", "provider") if err != nil { t.Fatal(err) } if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "gitea", "provider"); err != nil { t.Fatal(err) } again, err := inv.SecretFor(ctx, "postgres-database", "third", "gitea", "provider") if err != nil { t.Fatal(err) } if again.ForConsumer != untouched.ForConsumer { t.Fatal("rotating one machine's credential changed another machine's") } }