A consumer is a module on a machine, not a machine

novox/hq 04-ISSUES/022. A credential was keyed by provision, consumer
node and provider node, so "who is asking" was answered by naming a
host. The node this mesh exists to take over runs eight modules against
one database server.

The symptom had two halves and only one was loud. The provider refused,
naming the modules and explaining they would share one credential, which
reads as a decision rather than a limit. The consumer did not refuse: it
resolved cleanly, wrote one module's credential file and left the others
absent — a service that starts and cannot authenticate, with nothing
saying why. That is 021 again on a different axis.

Three modules wanting one database produced one need, carrying whichever
module mentioned it first, because the resolution walk is a work-list
over names. The fan-out now happens in one place, after the walk. The
record path already did this correctly and said why: a consumer here is
a module on a machine. It is the same rule.

Downstream: the secret's key gains the consuming module, the grant file
is named after both halves, needs are matched by provision and module
rather than provision alone, and the provisioners name the role and the
access key after the module. The refusal in ContributionsTo is gone
because there is nothing left to refuse.

Worth stating plainly: without that refusal, gitea's login would have
opened keycloak's database. From the provisioner's side it created
exactly what it was asked to create.

Existing secrets are discarded rather than backfilled. They cannot say
which module they were for, and a secret is remade and delivered to both
ends on the next push — so this costs one rotation and invents nothing.

Also guards the role name against PostgreSQL's 63-byte truncation, which
is a notice rather than an error and would reintroduce exactly this
collision at a length nobody tests.

Three faults injected — the fan-out removed, needs matched by name
alone, the grant file named after the machine — each caught.

internal/inventory/migrations/0015-a-consumer-is-a-module-on-a-machine.sql

@@ -0,0 +1,31 @@
+-- A credential belongs to a consumer, and a consumer is a module on a machine.
+--
+-- novox/hq 04-ISSUES/022. The key was (provision, consumer node, provider node), so "who is
+-- asking" was answered by naming a host. A node running three modules against one database server
+-- had one credential between them: the provisioner created one role, `mesh_<node>`, owning every
+-- database it was asked for, and gitea's login opened keycloak's data. Nothing anywhere would
+-- have said so -- from the provisioner's side it created exactly what it was asked to create.
+--
+-- **Two modules on one node are as separate as two on different nodes.** They are different
+-- containers, on different networks, with different data. This is the same correction as 021,
+-- which found the machine wrongly treated as a trust boundary; here it was wrongly treated as an
+-- identity.
+--
+-- It also restores withdrawal. One role per node cannot express "this module no longer has a
+-- login and the others still do", so a consumer that went away kept a working credential for as
+-- long as any other consumer on that machine remained.
+
+alter table secret add column consumer_module text references module(name) on delete cascade;
+
+-- Existing rows cannot say which module they were for, because at the time nothing recorded it.
+--
+-- **Discarded rather than guessed.** A secret is remade on the next declaration and reaches both
+-- ends in the same push, which is exactly what rotation does -- so this costs one rotation and
+-- nothing else. Backfilling with "whichever module resolves first" would be inventing an answer
+-- to the question this migration exists because nobody could answer.
+delete from secret;
+
+alter table secret alter column consumer_module set not null;
+
+alter table secret drop constraint secret_pkey;
+alter table secret add primary key (name, consumer, consumer_module, provider);

internal/inventory/secrets.go

@@ -14,16 +14,21 @@ import (
 
 // Secret is one provision's credential, sealed to each end.
 type Secret struct {
-	Name        string
-	Consumer    string
-	Provider    string
-	ForConsumer string
-	ForProvider string
-	ConsumerKey string
-	ProviderKey string
+	Name     string
+	Consumer string
+	// ConsumerModule is which module on that machine it is for.
+	//
+	// **Part of the key, not a label** (novox/hq 04-ISSUES/022). Two modules on one node wanting
+	// the same provision are two consumers, and were one credential until this.
+	ConsumerModule string
+	Provider       string
+	ForConsumer    string
+	ForProvider    string
+	ConsumerKey    string
+	ProviderKey    string
 }
 
-// SecretFor is the credential for one provision between two nodes, making one the first time.
+// SecretFor is the credential one module uses for one provision, making it the first time.
 //
 // **Made once and kept**, rather than regenerated whenever it is asked for. A secret that changed
 // on every declaration would restart both ends on every push and would mean the password a
@@ -34,7 +39,8 @@ type Secret struct {
 // can no longer open what was sealed to the old one, so keeping the blob would deliver something
 // unreadable for ever. The new secret reaches both ends in the same push, which is the only
 // moment they can be changed together.
-func (i *Inventory) SecretFor(ctx context.Context, name, consumer, provider string) (Secret, error) {
+func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModule, provider string) (
+	Secret, error) {
 	consumerKey, err := i.SealingKeyOf(ctx, consumer)
 	if err != nil {
 		return Secret{}, err
@@ -56,11 +62,12 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, provider stri
 	var held Secret
 	err = i.store.Pool().QueryRow(ctx,
 		`select for_consumer, for_provider, consumer_key, provider_key from secret
-		 where name = $1 and consumer = $2 and provider = $3`,
-		name, consumerNode.ID, providerNode.ID).
+		 where name = $1 and consumer = $2 and consumer_module = $3 and provider = $4`,
+		name, consumerNode.ID, consumerModule, providerNode.ID).
 		Scan(&held.ForConsumer, &held.ForProvider, &held.ConsumerKey, &held.ProviderKey)
 	if err == nil && held.ConsumerKey == consumerKey && held.ProviderKey == providerKey {
 		held.Name, held.Consumer, held.Provider = name, consumer, provider
+		held.ConsumerModule = consumerModule
 		return held, nil
 	}
 
@@ -69,19 +76,20 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, provider stri
 		return Secret{}, err
 	}
 	_, err = i.store.Pool().Exec(ctx,
-		`insert into secret (name, consumer, provider, for_consumer, for_provider,
+		`insert into secret (name, consumer, consumer_module, provider, for_consumer, for_provider,
 		                     consumer_key, provider_key)
-		 values ($1, $2, $3, $4, $5, $6, $7)
-		 on conflict (name, consumer, provider) do update set
+		 values ($1, $2, $3, $4, $5, $6, $7, $8)
+		 on conflict (name, consumer, consumer_module, provider) do update set
 		   for_consumer = excluded.for_consumer, for_provider = excluded.for_provider,
 		   consumer_key = excluded.consumer_key, provider_key = excluded.provider_key,
 		   created_at = now()`,
-		name, consumerNode.ID, providerNode.ID,
+		name, consumerNode.ID, consumerModule, providerNode.ID,
 		made.ForConsumer, made.ForProvider, made.ConsumerKey, made.ProviderKey)
 	if err != nil {
 		return Secret{}, err
 	}
-	return Secret{Name: name, Consumer: consumer, Provider: provider,
+	return Secret{Name: name, Consumer: consumer, ConsumerModule: consumerModule,
+		Provider:    provider,
 		ForConsumer: made.ForConsumer, ForProvider: made.ForProvider,
 		ConsumerKey: made.ConsumerKey, ProviderKey: made.ProviderKey}, nil
 }
@@ -94,7 +102,7 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, provider stri
 //
 // The new secret then reaches both ends on the same push, together, which is what makes rotation
 // a single event rather than a fanout with a window where half the mesh holds a dead credential.
-func (i *Inventory) RotateSecret(ctx context.Context, name, consumer, provider string) error {
+func (i *Inventory) RotateSecret(ctx context.Context, name, consumer, consumerModule, provider string) error {
 	consumerNode, err := i.NodeByName(ctx, consumer)
 	if err != nil {
 		return err
@@ -104,8 +112,9 @@ func (i *Inventory) RotateSecret(ctx context.Context, name, consumer, provider s
 		return err
 	}
 	_, err = i.store.Pool().Exec(ctx,
-		`delete from secret where name = $1 and consumer = $2 and provider = $3`,
-		name, consumerNode.ID, providerNode.ID)
+		`delete from secret where name = $1 and consumer = $2 and consumer_module = $3
+		   and provider = $4`,
+		name, consumerNode.ID, consumerModule, providerNode.ID)
 	return err
 }
 
@@ -116,9 +125,9 @@ func (i *Inventory) SecretsFrom(ctx context.Context, provider string) ([]Secret,
 		return nil, err
 	}
 	rows, err := i.store.Pool().Query(ctx,
-		`select s.name, c.name, s.for_provider from secret s
+		`select s.name, c.name, s.consumer_module, s.for_provider from secret s
 		 join node c on c.id = s.consumer
-		 where s.provider = $1 order by s.name, c.name`, providerNode.ID)
+		 where s.provider = $1 order by s.name, c.name, s.consumer_module`, providerNode.ID)
 	if err != nil {
 		return nil, err
 	}
@@ -127,7 +136,7 @@ func (i *Inventory) SecretsFrom(ctx context.Context, provider string) ([]Secret,
 	var out []Secret
 	for rows.Next() {
 		s := Secret{Provider: provider}
-		if err := rows.Scan(&s.Name, &s.Consumer, &s.ForProvider); err != nil {
+		if err := rows.Scan(&s.Name, &s.Consumer, &s.ConsumerModule, &s.ForProvider); err != nil {
 			return nil, err
 		}
 		out = append(out, s)
@@ -235,7 +244,10 @@ func (i *Inventory) AcceptSecretForModule(ctx context.Context, node, module, nam
 type Holder struct {
 	Provision string
 	Consumer  string
-	Provider  string
+	// ConsumerModule is which module on that machine holds it. Part of what identifies a
+	// credential (novox/hq 04-ISSUES/022), so rotating one consumer's does not touch another's.
+	ConsumerModule string
+	Provider       string
 }
 
 // HoldersOf is every pair sharing a credential for one provision.
@@ -249,11 +261,11 @@ type Holder struct {
 // Empty consumer means all of them.
 func (i *Inventory) HoldersOf(ctx context.Context, provision, consumer string) ([]Holder, error) {
 	rows, err := i.store.Pool().Query(ctx,
-		`select s.name, c.name, p.name from secret s
+		`select s.name, c.name, s.consumer_module, p.name from secret s
 		   join node c on c.id = s.consumer
 		   join node p on p.id = s.provider
 		  where s.name = $1 and ($2 = '' or c.name = $2)
-		  order by c.name, p.name`, provision, consumer)
+		  order by c.name, s.consumer_module, p.name`, provision, consumer)
 	if err != nil {
 		return nil, err
 	}
@@ -262,7 +274,7 @@ func (i *Inventory) HoldersOf(ctx context.Context, provision, consumer string) (
 	var out []Holder
 	for rows.Next() {
 		var h Holder
-		if err := rows.Scan(&h.Provision, &h.Consumer, &h.Provider); err != nil {
+		if err := rows.Scan(&h.Provision, &h.Consumer, &h.ConsumerModule, &h.Provider); err != nil {
 			return nil, err
 		}
 		out = append(out, h)

internal/inventory/secrets_test.go

@@ -47,6 +47,15 @@ func twoNodesWithKeys(t *testing.T) (*Inventory, context.Context) {
 			t.Fatal(err)
 		}
 	}
+	// A credential belongs to a module on a machine, so the modules holding one must exist
+	// before it can (novox/hq 04-ISSUES/022). Two of them, because "two consumers on one node"
+	// is the case that key exists for.
+	for _, m := range []string{"gitea", "keycloak"} {
+		if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"},
+			Source{}); err != nil {
+			t.Fatal(err)
+		}
+	}
 	return inv, ctx
 }
 
@@ -54,11 +63,11 @@ func TestASecretIsMadeOnceAndKept(t *testing.T) {
 	// Regenerating on every declaration would restart both ends on every push, and — worse — the
 	// password a provider was told to create would never be the one its consumer was given.
 	inv, ctx := twoNodesWithKeys(t)
-	first, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
+	first, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider")
 	if err != nil {
 		t.Fatal(err)
 	}
-	second, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
+	second, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -72,7 +81,7 @@ func TestTheStoredSecretIsNotTheSecret(t *testing.T) {
 	// what an encrypted column does not achieve, because whoever runs the control plane can read
 	// through it.
 	inv, ctx := twoNodesWithKeys(t)
-	got, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
+	got, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -105,7 +114,7 @@ func TestANewSealingKeyMeansANewSecret(t *testing.T) {
 	// A node that rejoined generated a new key and can no longer open what was sealed to the old
 	// one. Keeping the blob would deliver something unreadable for ever, reported as configured.
 	inv, ctx := twoNodesWithKeys(t)
-	before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
+	before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -117,7 +126,7 @@ func TestANewSealingKeyMeansANewSecret(t *testing.T) {
 	if err := inv.RecordSealingKey(ctx, node.ID, fresh); err != nil {
 		t.Fatal(err)
 	}
-	after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
+	after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -133,14 +142,14 @@ func TestANewSealingKeyMeansANewSecret(t *testing.T) {
 
 func TestRotatingReachesBothEnds(t *testing.T) {
 	inv, ctx := twoNodesWithKeys(t)
-	before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
+	before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider")
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "provider"); err != nil {
+	if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "gitea", "provider"); err != nil {
 		t.Fatal(err)
 	}
-	after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
+	after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -175,7 +184,7 @@ func TestAProviderIsToldEveryCredentialItMustCreate(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, who := range []string{"consumer", "second-consumer"} {
-		if _, err := inv.SecretFor(ctx, "postgres-database", who, "provider"); err != nil {
+		if _, err := inv.SecretFor(ctx, "postgres-database", who, "gitea", "provider"); err != nil {
 			t.Fatal(err)
 		}
 	}
@@ -206,7 +215,7 @@ func TestANodeWithNoSealingKeyCannotBeGivenASecret(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
-	_, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
+	_, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider")
 	if err == nil {
 		t.Fatal("a credential was made for nodes that cannot open one")
 	}
@@ -217,7 +226,7 @@ func TestANodeWithNoSealingKeyCannotBeGivenASecret(t *testing.T) {
 
 func TestSecretsGoWhenANodeLeaves(t *testing.T) {
 	inv, ctx := twoNodesWithKeys(t)
-	if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider"); err != nil {
+	if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider"); err != nil {
 		t.Fatal(err)
 	}
 	if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'consumer'`); err != nil {
@@ -340,7 +349,7 @@ func TestACredentialGoesWhenTheConsumerStopsAskingForIt(t *testing.T) {
 	if err := inv.Assign(ctx, "consumer", "meshboard"); err != nil {
 		t.Fatal(err)
 	}
-	if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider"); err != nil {
+	if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider"); err != nil {
 		t.Fatal(err)
 	}
 
@@ -370,7 +379,7 @@ func TestACredentialGoesWhenEitherMachineDoes(t *testing.T) {
 	// The case that must not leave a live login behind: a machine removed from the mesh. Its
 	// credentials go with it, and the provider stops being told to keep them.
 	inv, ctx := twoNodesWithKeys(t)
-	if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider"); err != nil {
+	if _, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider"); err != nil {
 		t.Fatal(err)
 	}
 	if _, err := inv.store.Pool().Exec(ctx, `delete from node where name = 'consumer'`); err != nil {
@@ -464,12 +473,12 @@ func TestEveryHolderOfACredentialCanBeNamed(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, consumer := range []string{"consumer", "third"} {
-		if _, err := inv.SecretFor(ctx, "postgres-database", consumer, "provider"); err != nil {
+		if _, err := inv.SecretFor(ctx, "postgres-database", consumer, "gitea", "provider"); err != nil {
 			t.Fatal(err)
 		}
 	}
 	// And one for a different provision, which must not be swept up.
-	if _, err := inv.SecretFor(ctx, "cache", "consumer", "provider"); err != nil {
+	if _, err := inv.SecretFor(ctx, "cache", "consumer", "gitea", "provider"); err != nil {
 		t.Fatal(err)
 	}
 
@@ -500,14 +509,14 @@ func TestEveryHolderOfACredentialCanBeNamed(t *testing.T) {
 // And rotating gives both ends a new credential, together — the same one.
 func TestRotatingGivesBothEndsTheSameNewCredential(t *testing.T) {
 	inv, ctx := twoNodesWithKeys(t)
-	before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
+	before, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider")
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "provider"); err != nil {
+	if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "gitea", "provider"); err != nil {
 		t.Fatal(err)
 	}
-	after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "provider")
+	after, err := inv.SecretFor(ctx, "postgres-database", "consumer", "gitea", "provider")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -534,14 +543,14 @@ func TestRotatingGivesBothEndsTheSameNewCredential(t *testing.T) {
 	if err := inv.RecordSealingKey(ctx, third.ID, key); err != nil {
 		t.Fatal(err)
 	}
-	untouched, err := inv.SecretFor(ctx, "postgres-database", "third", "provider")
+	untouched, err := inv.SecretFor(ctx, "postgres-database", "third", "gitea", "provider")
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "provider"); err != nil {
+	if err := inv.RotateSecret(ctx, "postgres-database", "consumer", "gitea", "provider"); err != nil {
 		t.Fatal(err)
 	}
-	again, err := inv.SecretFor(ctx, "postgres-database", "third", "provider")
+	again, err := inv.SecretFor(ctx, "postgres-database", "third", "gitea", "provider")
 	if err != nil {
 		t.Fatal(err)
 	}