novox/mesh-controller
3
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

A consumer is a module on a machine, not a machine

Browse Source 
novox/hq 04-ISSUES/022. A credential was keyed by provision, consumer
node and provider node, so "who is asking" was answered by naming a
host. The node this mesh exists to take over runs eight modules against
one database server.

The symptom had two halves and only one was loud. The provider refused,
naming the modules and explaining they would share one credential, which
reads as a decision rather than a limit. The consumer did not refuse: it
resolved cleanly, wrote one module's credential file and left the others
absent — a service that starts and cannot authenticate, with nothing
saying why. That is 021 again on a different axis.

Three modules wanting one database produced one need, carrying whichever
module mentioned it first, because the resolution walk is a work-list
over names. The fan-out now happens in one place, after the walk. The
record path already did this correctly and said why: a consumer here is
a module on a machine. It is the same rule.

Downstream: the secret's key gains the consuming module, the grant file
is named after both halves, needs are matched by provision and module
rather than provision alone, and the provisioners name the role and the
access key after the module. The refusal in ContributionsTo is gone
because there is nothing left to refuse.

Worth stating plainly: without that refusal, gitea's login would have
opened keycloak's database. From the provisioner's side it created
exactly what it was asked to create.

Existing secrets are discarded rather than backfilled. They cannot say
which module they were for, and a secret is remade and delivered to both
ends on the next push — so this costs one rotation and invents nothing.

Also guards the role name against PostgreSQL's 63-byte truncation, which
is a notice rather than an error and would reintroduce exactly this
collision at a length nobody tests.

Three faults injected — the fan-out removed, needs matched by name
alone, the grant file named after the machine — each caught.
This commit is contained in:
jschoubben
2026-09-01 02:40:09 +02:00
parent 1314be5282
commit 0af3ea1acf
14 changed files with 440 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/inventory/secrets.go
+38 -26
View File
@@ -14,16 +14,21 @@ import (
// Secret is one provision's credential, sealed to each end.
type Secret struct {
Name string
Consumer string
Provider string
ForConsumer string
ForProvider string
ConsumerKey string
ProviderKey string
Name string
Consumer string
// ConsumerModule is which module on that machine it is for.
//
// **Part of the key, not a label** (novox/hq 04-ISSUES/022). Two modules on one node wanting
// the same provision are two consumers, and were one credential until this.
ConsumerModule string
Provider string
ForConsumer string
ForProvider string
ConsumerKey string
ProviderKey string
}
// SecretFor is the credential for one provision between two nodes, making one the first time.
// SecretFor is the credential one module uses for one provision, making it the first time.
//
// **Made once and kept**, rather than regenerated whenever it is asked for. A secret that changed
// on every declaration would restart both ends on every push and would mean the password a
@@ -34,7 +39,8 @@ type Secret struct {
// can no longer open what was sealed to the old one, so keeping the blob would deliver something
// unreadable for ever. The new secret reaches both ends in the same push, which is the only
// moment they can be changed together.
func (i *Inventory) SecretFor(ctx context.Context, name, consumer, provider string) (Secret, error) {
func (i *Inventory) SecretFor(ctx context.Context, name, consumer, consumerModule, provider string) (
Secret, error) {
consumerKey, err := i.SealingKeyOf(ctx, consumer)
if err != nil {
return Secret{}, err
@@ -56,11 +62,12 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, provider stri
var held Secret
err = i.store.Pool().QueryRow(ctx,
`select for_consumer, for_provider, consumer_key, provider_key from secret
where name = $1 and consumer = $2 and provider = $3`,
name, consumerNode.ID, providerNode.ID).
where name = $1 and consumer = $2 and consumer_module = $3 and provider = $4`,
name, consumerNode.ID, consumerModule, providerNode.ID).
Scan(&held.ForConsumer, &held.ForProvider, &held.ConsumerKey, &held.ProviderKey)
if err == nil && held.ConsumerKey == consumerKey && held.ProviderKey == providerKey {
held.Name, held.Consumer, held.Provider = name, consumer, provider
held.ConsumerModule = consumerModule
return held, nil
}
@@ -69,19 +76,20 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, provider stri
return Secret{}, err
}
_, err = i.store.Pool().Exec(ctx,
`insert into secret (name, consumer, provider, for_consumer, for_provider,
`insert into secret (name, consumer, consumer_module, provider, for_consumer, for_provider,
consumer_key, provider_key)
values ($1, $2, $3, $4, $5, $6, $7)
on conflict (name, consumer, provider) do update set
values ($1, $2, $3, $4, $5, $6, $7, $8)
on conflict (name, consumer, consumer_module, provider) do update set
for_consumer = excluded.for_consumer, for_provider = excluded.for_provider,
consumer_key = excluded.consumer_key, provider_key = excluded.provider_key,
created_at = now()`,
name, consumerNode.ID, providerNode.ID,
name, consumerNode.ID, consumerModule, providerNode.ID,
made.ForConsumer, made.ForProvider, made.ConsumerKey, made.ProviderKey)
if err != nil {
return Secret{}, err
}
return Secret{Name: name, Consumer: consumer, Provider: provider,
return Secret{Name: name, Consumer: consumer, ConsumerModule: consumerModule,
Provider: provider,
ForConsumer: made.ForConsumer, ForProvider: made.ForProvider,
ConsumerKey: made.ConsumerKey, ProviderKey: made.ProviderKey}, nil
}
@@ -94,7 +102,7 @@ func (i *Inventory) SecretFor(ctx context.Context, name, consumer, provider stri
//
// The new secret then reaches both ends on the same push, together, which is what makes rotation
// a single event rather than a fanout with a window where half the mesh holds a dead credential.
func (i *Inventory) RotateSecret(ctx context.Context, name, consumer, provider string) error {
func (i *Inventory) RotateSecret(ctx context.Context, name, consumer, consumerModule, provider string) error {
consumerNode, err := i.NodeByName(ctx, consumer)
if err != nil {
return err
@@ -104,8 +112,9 @@ func (i *Inventory) RotateSecret(ctx context.Context, name, consumer, provider s
return err
}
_, err = i.store.Pool().Exec(ctx,
`delete from secret where name = $1 and consumer = $2 and provider = $3`,
name, consumerNode.ID, providerNode.ID)
`delete from secret where name = $1 and consumer = $2 and consumer_module = $3
and provider = $4`,
name, consumerNode.ID, consumerModule, providerNode.ID)
return err
}
@@ -116,9 +125,9 @@ func (i *Inventory) SecretsFrom(ctx context.Context, provider string) ([]Secret,
return nil, err
}
rows, err := i.store.Pool().Query(ctx,
`select s.name, c.name, s.for_provider from secret s
`select s.name, c.name, s.consumer_module, s.for_provider from secret s
join node c on c.id = s.consumer
where s.provider = $1 order by s.name, c.name`, providerNode.ID)
where s.provider = $1 order by s.name, c.name, s.consumer_module`, providerNode.ID)
if err != nil {
return nil, err
}
@@ -127,7 +136,7 @@ func (i *Inventory) SecretsFrom(ctx context.Context, provider string) ([]Secret,
var out []Secret
for rows.Next() {
s := Secret{Provider: provider}
if err := rows.Scan(&s.Name, &s.Consumer, &s.ForProvider); err != nil {
if err := rows.Scan(&s.Name, &s.Consumer, &s.ConsumerModule, &s.ForProvider); err != nil {
return nil, err
}
out = append(out, s)
@@ -235,7 +244,10 @@ func (i *Inventory) AcceptSecretForModule(ctx context.Context, node, module, nam
type Holder struct {
Provision string
Consumer string
Provider string
// ConsumerModule is which module on that machine holds it. Part of what identifies a
// credential (novox/hq 04-ISSUES/022), so rotating one consumer's does not touch another's.
ConsumerModule string
Provider string
}
// HoldersOf is every pair sharing a credential for one provision.
@@ -249,11 +261,11 @@ type Holder struct {
// Empty consumer means all of them.
func (i *Inventory) HoldersOf(ctx context.Context, provision, consumer string) ([]Holder, error) {
rows, err := i.store.Pool().Query(ctx,
`select s.name, c.name, p.name from secret s
`select s.name, c.name, s.consumer_module, p.name from secret s
join node c on c.id = s.consumer
join node p on p.id = s.provider
where s.name = $1 and ($2 = '' or c.name = $2)
order by c.name, p.name`, provision, consumer)
order by c.name, s.consumer_module, p.name`, provision, consumer)
if err != nil {
return nil, err
}
@@ -262,7 +274,7 @@ func (i *Inventory) HoldersOf(ctx context.Context, provision, consumer string) (
var out []Holder
for rows.Next() {
var h Holder
if err := rows.Scan(&h.Provision, &h.Consumer, &h.Provider); err != nil {
if err := rows.Scan(&h.Provision, &h.Consumer, &h.ConsumerModule, &h.Provider); err != nil {
return nil, err
}
out = append(out, h)