Issue an account whose credential is not sealed, and forget an unassigned module's accounts (issue 1945, ADR 0543)

Met a second time live on 2026-10-25 at 20:04 (note 35): mesh-build-go was
assigned to ace again, and its account rows, minted at 17:19, had outlived the
unassignment. The issue minted only when no row stood, so it minted and sealed
nothing; the node had no credential and the bus no user, and the module
crash-looped 20 times. The watch still said "the grants written".

- A bus account now says when its credential was sealed to its node (migration
  0108; every account standing before is taken as sealed). An issue under the
  assign act, the assignments' watch or a registration mints only an account
  that is not issued, a row with no seal counting as none, under the store's
  lock on the account across its look, mint and seal, and marks it sealed once
  sealed. A seal that fails leaves it unsealed for the next issue.
- Unassigning a module forgets its bus accounts and their credentials on that
  node, so an assignment again issues them afresh.
- Only sealed accounts count as minted for a composition and are named in a
  grant, so the assigner, which now wants a node only once the grant names its
  account (mesh-delivery, same branch), holds the node until it is.
- The assignment's log line says which accounts were issued and sealed, or that
  they stood issued and sealed.

The process-only issue drops the atomic insert of the review of 9ff47f6a for
the lock, which also covers the seal. Tests: cases in the existing tables.
Research 072: the grants piece (memberships and credentials heard from placement).
This commit is contained in:
2026-10-25 20:17:46 +01:00
parent 40034be479
commit 79ac62ffbc
11 changed files with 347 additions and 111 deletions
+81 -32
View File
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"sync/atomic"
"time"
"github.com/jackc/pgx/v5"
"golang.org/x/crypto/bcrypt"
@@ -52,6 +53,8 @@ type BusUser struct {
// PasswordHash is what the composed file carries. The plaintext is returned once, by Mint, and
// then exists only where it was sealed.
PasswordHash string
// Sealed says its credential reached its node: an issue's mint is not sealed until then (ADR 0543 §2).
Sealed bool
}
// The kinds of bus user the mesh records. The same words the composer uses, so a row and a
@@ -100,52 +103,98 @@ func (i *Inventory) RecordBusPassword(ctx context.Context, u BusUser, password s
return i.writeBusUser(ctx, u, string(hash))
}
// writeBusUser is the row, whoever chose the password.
// writeBusUser is the row, whoever chose the password, taken as sealed.
func (i *Inventory) writeBusUser(ctx context.Context, u BusUser, hash string) error {
return i.writeBusUserSealed(ctx, u, hash, true)
}
// writeBusUserSealed is the row, sealed now or not yet.
func (i *Inventory) writeBusUserSealed(ctx context.Context, u BusUser, hash string, sealed bool) error {
if _, err := i.store.Pool().Exec(ctx,
`insert into bus_user (username, kind, node, module, password_hash)
values ($1, $2, $3, $4, $5)
`insert into bus_user (username, kind, node, module, password_hash, sealed_at)
values ($1, $2, $3, $4, $5, case when $6 then now() end)
on conflict (username) do update
set kind = excluded.kind, node = excluded.node, module = excluded.module,
password_hash = excluded.password_hash, minted_at = now()`,
u.Username, u.Kind, u.Node, u.Module, hash); err != nil {
password_hash = excluded.password_hash, minted_at = now(), sealed_at = excluded.sealed_at`,
u.Username, u.Kind, u.Node, u.Module, hash, sealed); err != nil {
return fmt.Errorf("cannot record the bus user %s: %w", u.Username, err)
}
return nil
}
// MintBusPasswordIfAbsent mints u's password only when no row names u yet, in one statement: of two callers that both
// found the account missing, exactly one wins, and the other is told it lost (won false, no password) and must seal
// nothing, so the hash the bus holds is always the password the node is sealed (novox/hq ADR 0543, the review of
// issue 1945: the assign act and the assignments' watch once each minted, the later hash beside the earlier seal).
func (i *Inventory) MintBusPasswordIfAbsent(ctx context.Context, u BusUser) (string, bool, error) {
if u.Username == "" || u.Kind == "" {
return "", false, errors.New("a bus user needs a username and a kind")
}
raw := make([]byte, 32)
if _, err := rand.Read(raw); err != nil {
return "", false, fmt.Errorf("cannot generate a bus password: %w", err)
}
password := base64.RawURLEncoding.EncodeToString(raw)
hash, err := hashBusPassword(password)
// LockBusAccount holds the store's lock on one account until the answered release is called, so two issuers of it
// (the assign act, the assignments' watch, a registration's process accounts) never interleave their look, mint and
// seal: the hash the bus holds is always the password sealed to the node (novox/hq ADR 0543 §2, issue 1945).
func (i *Inventory) LockBusAccount(ctx context.Context, username string) (func(), error) {
conn, err := i.store.Pool().Acquire(ctx)
if err != nil {
return "", false, fmt.Errorf("cannot hash a bus password: %w", err)
return nil, fmt.Errorf("the lock on %s cannot be taken: %w", username, err)
}
tag, err := i.store.Pool().Exec(ctx,
`insert into bus_user (username, kind, node, module, password_hash)
values ($1, $2, $3, $4, $5)
on conflict (username) do nothing`,
u.Username, u.Kind, u.Node, u.Module, string(hash))
if _, err := conn.Exec(ctx, `select pg_advisory_lock(hashtext($1))`, "bus_user/"+username); err != nil {
conn.Release()
return nil, fmt.Errorf("the lock on %s cannot be taken: %w", username, err)
}
return func() {
_, _ = conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock(hashtext($1))`, "bus_user/"+username)
conn.Release()
}, nil
}
// ForgetBusAccountsOf forgets the bus accounts of module on node, and the credentials sealed to the node for them (its
// `broker`, each process's `broker-<id>`, its provisioner client's): an unassigned module keeps no account, so an
// assignment again mints and seals afresh, never trusting a row whose credential the node no longer holds (issue 1945,
// live on 2026-10-25 at 20:04: ace's mesh-build-go rows from 17:19 survived its unassignment, nothing was minted or
// sealed again, and the module found no credential).
func (i *Inventory) ForgetBusAccountsOf(ctx context.Context, node, module string) error {
if node == "" || module == "" {
return errors.New("forgetting the bus accounts of no node or no module would forget others' accounts")
}
tx, err := i.store.Pool().Begin(ctx)
if err != nil {
return "", false, fmt.Errorf("cannot record the bus user %s: %w", u.Username, err)
return err
}
if tag.RowsAffected() == 0 {
return "", false, nil
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
if _, err := tx.Exec(ctx, `delete from bus_user where node = $1 and module = $2 and kind <> $3`,
node, module, BusController); err != nil {
return fmt.Errorf("the bus accounts of %s on %s are not forgotten: %w", module, node, err)
}
return password, true, nil
if _, err := tx.Exec(ctx,
`delete from module_secret s using node n
where s.node = n.id and n.name = $1 and s.module = $2
and (s.name = 'broker' or s.name like 'broker-%' or s.name = 'provisioner-broker')`, node, module); err != nil {
return fmt.Errorf("the bus credentials of %s on %s are not forgotten: %w", module, node, err)
}
return tx.Commit(ctx)
}
// MintBusPasswordUnsealed mints u's password, replacing whatever stands, with no seal yet: the account is not issued
// until MarkBusAccountSealed says its credential reached the node (ADR 0543 §2).
func (i *Inventory) MintBusPasswordUnsealed(ctx context.Context, u BusUser) (string, error) {
return i.mintBusPassword(ctx, u, false)
}
// MarkBusAccountSealed says the credential last minted for username is sealed to its node.
func (i *Inventory) MarkBusAccountSealed(ctx context.Context, username string) error {
_, err := i.store.Pool().Exec(ctx, `update bus_user set sealed_at = now() where username = $1`, username)
return err
}
// BusAccountIssued says whether an account is minted, and whether its credential is sealed to its node.
func (i *Inventory) BusAccountIssued(ctx context.Context, username string) (minted, sealed bool, err error) {
var at *time.Time
err = i.store.Pool().QueryRow(ctx, `select sealed_at from bus_user where username = $1`, username).Scan(&at)
if errors.Is(err, pgx.ErrNoRows) {
return false, false, nil
}
return err == nil, err == nil && at != nil, err
}
func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, error) {
return i.mintBusPassword(ctx, u, true)
}
// mintBusPassword mints, marked sealed when the caller seals it itself in the same act (every mint but an issue's).
func (i *Inventory) mintBusPassword(ctx context.Context, u BusUser, sealed bool) (string, error) {
if u.Username == "" || u.Kind == "" {
return "", errors.New("a bus user needs a username and a kind")
}
@@ -163,7 +212,7 @@ func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, err
return "", fmt.Errorf("cannot hash a bus password: %w", err)
}
if err := i.writeBusUser(ctx, u, string(hash)); err != nil {
if err := i.writeBusUserSealed(ctx, u, string(hash), sealed); err != nil {
return "", err
}
return password, nil
@@ -176,7 +225,7 @@ func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, err
// password, and the composer refuses it rather than writing a user anybody is.
func (i *Inventory) BusUsers(ctx context.Context) (map[string]BusUser, error) {
rows, err := i.store.Pool().Query(ctx,
`select username, kind, node, module, password_hash from bus_user order by username`)
`select username, kind, node, module, password_hash, sealed_at is not null from bus_user order by username`)
if err != nil {
return nil, err
}
@@ -184,7 +233,7 @@ func (i *Inventory) BusUsers(ctx context.Context) (map[string]BusUser, error) {
out := map[string]BusUser{}
for rows.Next() {
var u BusUser
if err := rows.Scan(&u.Username, &u.Kind, &u.Node, &u.Module, &u.PasswordHash); err != nil {
if err := rows.Scan(&u.Username, &u.Kind, &u.Node, &u.Module, &u.PasswordHash, &u.Sealed); err != nil {
return nil, err
}
out[u.Username] = u