Issue an account whose credential is not sealed, and forget an unassigned module's accounts (issue 1945, ADR 0543)

Met a second time live on 2026-10-25 at 20:04 (note 35): mesh-build-go was
assigned to ace again, and its account rows, minted at 17:19, had outlived the
unassignment. The issue minted only when no row stood, so it minted and sealed
nothing; the node had no credential and the bus no user, and the module
crash-looped 20 times. The watch still said "the grants written".

- A bus account now says when its credential was sealed to its node (migration
  0108; every account standing before is taken as sealed). An issue under the
  assign act, the assignments' watch or a registration mints only an account
  that is not issued, a row with no seal counting as none, under the store's
  lock on the account across its look, mint and seal, and marks it sealed once
  sealed. A seal that fails leaves it unsealed for the next issue.
- Unassigning a module forgets its bus accounts and their credentials on that
  node, so an assignment again issues them afresh.
- Only sealed accounts count as minted for a composition and are named in a
  grant, so the assigner, which now wants a node only once the grant names its
  account (mesh-delivery, same branch), holds the node until it is.
- The assignment's log line says which accounts were issued and sealed, or that
  they stood issued and sealed.

The process-only issue drops the atomic insert of the review of 9ff47f6a for
the lock, which also covers the seal. Tests: cases in the existing tables.
Research 072: the grants piece (memberships and credentials heard from placement).
This commit is contained in:
2026-10-25 20:17:46 +01:00
parent 40034be479
commit 79ac62ffbc
11 changed files with 347 additions and 111 deletions
+2 -2
View File
@@ -421,9 +421,9 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
// A module running the provisioner client alone is minted the client's account (issue 917). // A module running the provisioner client alone is minted the client's account (issue 917).
user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username() user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username()
} }
if _, minted, err := inv.BusUserHash(ctx, user); err != nil { if _, sealed, err := inv.BusAccountIssued(ctx, user); err != nil {
return "" return ""
} else if minted { } else if sealed { // a row whose credential is not sealed to the node is not issued (issue 1945)
// Its own account stands and is kept; a process it gained is not issued yet (issue 1226). // Its own account stands and is kept; a process it gained is not issued yet (issue 1226).
return processAccountsOnAssign(ctx, inv, m, node) return processAccountsOnAssign(ctx, inv, m, node)
} }
+24 -6
View File
@@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"slices"
"strings" "strings"
"sync" "sync"
"time" "time"
@@ -195,22 +196,27 @@ func issueAssignmentAccounts(ctx context.Context, inv *inventory.Inventory, shel
if !known || mayIssue(m) != nil { if !known || mayIssue(m) != nil {
return false, "", nil return false, "", nil
} }
if user, own := ownAccountOf(m, node); own { if _, own := ownAccountOf(m, node); own {
_, minted, err := inv.BusUserHash(ctx, user) minted, err := inv.BusMinted(ctx) // sealed accounts alone: a row with no sealed credential is not issued
if err != nil { if err != nil {
return false, "", fmt.Errorf("whether %s is minted cannot be read: %w", user, err) return false, "", fmt.Errorf("the issued accounts cannot be read: %w", err)
} }
if !minted { if missingAccounts(m, node, minted) {
busAddress, err := broker.BusAddress() busAddress, err := broker.BusAddress()
if err != nil { if err != nil {
return false, "", err return false, "", err
} }
// Its processes' accounts too: issueOnTheNewBus issues every one of them with the module's own. // Its processes' and provisioner client's accounts too: issueOnTheNewBus issues each that is not issued.
ctx, got := withIssued(ctx)
if err := issueOnTheNewBus(ctx, inv, m, node, busAddress); err != nil { if err := issueOnTheNewBus(ctx, inv, m, node, busAddress); err != nil {
return false, "", err return false, "", err
} }
return true, "its bus credential issued as " + user, nil if issued := got.list(); len(issued) > 0 {
return true, "issued and sealed to " + node + ": " + strings.Join(issued, ", "), nil
}
return false, "", nil
} }
return false, "its bus accounts on " + node + " stood issued and sealed", nil
} }
line, err := issueMissingProcessAccounts(ctx, inv, m, node) line, err := issueMissingProcessAccounts(ctx, inv, m, node)
if err != nil { if err != nil {
@@ -219,6 +225,18 @@ func issueAssignmentAccounts(ctx context.Context, inv *inventory.Inventory, shel
return line != "", line, nil return line != "", line, nil
} }
// missingAccounts says some account of m on node is not issued: its own, a process's, or its provisioner client's.
func missingAccounts(m catalogue.Manifest, node string, minted []string) bool {
if user, own := ownAccountOf(m, node); own && !slices.Contains(minted, user) {
return true
}
if runsTheProvisionerClient(m) &&
!slices.Contains(minted, broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username()) {
return true
}
return len(processAccountsMissing(m, node, minted)) > 0
}
// liveAssignmentAccountSteps are the steps on the controller's own stores and the bus: the composer's passes, as the // liveAssignmentAccountSteps are the steps on the controller's own stores and the bus: the composer's passes, as the
// done keys' take-ins use them, so a node asked for by both is composed once a pass. // done keys' take-ins use them, so a node asked for by both is composed once a pass.
func liveAssignmentAccountSteps(open *stores) assignmentAccountSteps { func liveAssignmentAccountSteps(open *stores) assignmentAccountSteps {
@@ -12,6 +12,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/chain" "github.com/novox/mesh-controller/internal/chain"
"github.com/novox/mesh-controller/internal/inventory"
) )
// An assignment that comes to stand issues its node's bus accounts and the module's grant (ADR 0543, issue 1945): on // An assignment that comes to stand issues its node's bus accounts and the module's grant (ADR 0543, issue 1945): on
@@ -324,6 +325,48 @@ func TestTheAccountsAModuleSpeaksWithOnANode(t *testing.T) {
} }
}) })
// An issue mints and seals an account only when it is not issued: no row, or a row with no sealed credential (issue
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go had a row from 17:19 and no credential), under the lock on
// the account; a seal that fails leaves it unsealed for the next issue. A rotation always mints.
for _, c := range []struct {
name string
only bool
minted, sealed bool
sealErr error
want string
}{
{"no row: minted unsealed, sealed, marked", true, false, false, nil, "lock mint-unsealed seal mark release"},
{"a row sealed: nothing", true, true, true, nil, "lock release"},
{"a row with no sealed credential: minted and sealed again", true, true, false, nil, "lock mint-unsealed seal mark release"},
{"a seal that fails: not marked", true, false, false, errors.New("the vault did not take it"), "lock mint-unsealed seal release"},
{"a rotation mints whatever stands", false, true, true, nil, "mint seal"},
} {
t.Run("issuing an account: "+c.name, func(t *testing.T) {
acts := &anAccountStore{minted: c.minted, sealed: c.sealed}
ctx, got := withIssued(t.Context())
if c.only {
ctx = onlyIfAbsent(ctx)
}
err := issueAccount(ctx, acts, inventory.BusUser{Username: "ace.mesh-build-go", Kind: inventory.BusModule},
func(password string) error {
acts.did("seal")
if password == "" {
t.Error("sealed no password")
}
return c.sealErr
})
if (err != nil) != (c.sealErr != nil) {
t.Fatalf("error %v, want %v", err, c.sealErr)
}
if strings.Join(acts.acts, " ") != c.want {
t.Fatalf("acts %q, want %q", strings.Join(acts.acts, " "), c.want)
}
if issued := len(got.list()) == 1; issued != (c.sealErr == nil && strings.Contains(c.want, "seal")) {
t.Fatalf("said issued %v", got.list())
}
})
}
t.Run("a process may be issued only with its own broker secret", func(t *testing.T) { t.Run("a process may be issued only with its own broker secret", func(t *testing.T) {
// mesh-mail's real shape: no `broker` of its own, its process declaring bus (review round 1 of issue 1181). // mesh-mail's real shape: no `broker` of its own, its process declaring bus (review round 1 of issue 1181).
if err := mayIssue(mailSenderWithProcess()); err != nil { if err := mayIssue(mailSenderWithProcess()); err != nil {
@@ -342,3 +385,30 @@ func TestTheAccountsAModuleSpeaksWithOnANode(t *testing.T) {
} }
}) })
} }
// anAccountStore is an accountIssuer that writes down what was asked of it.
type anAccountStore struct {
minted, sealed bool
acts []string
}
func (a *anAccountStore) did(act string) { a.acts = append(a.acts, act) }
func (a *anAccountStore) LockBusAccount(context.Context, string) (func(), error) {
a.did("lock")
return func() { a.did("release") }, nil
}
func (a *anAccountStore) BusAccountIssued(context.Context, string) (bool, bool, error) {
return a.minted, a.sealed, nil
}
func (a *anAccountStore) MintBusPassword(context.Context, inventory.BusUser) (string, error) {
a.did("mint")
return "a-password", nil
}
func (a *anAccountStore) MintBusPasswordUnsealed(context.Context, inventory.BusUser) (string, error) {
a.did("mint-unsealed")
return "a-password", nil
}
func (a *anAccountStore) MarkBusAccountSealed(context.Context, string) error {
a.did("mark")
return nil
}
+5 -1
View File
@@ -248,7 +248,11 @@ func grantsIssued(ctx context.Context, inv *inventory.Inventory, js jetstream.Je
} }
hashes := make(map[string]string, len(kept)) hashes := make(map[string]string, len(kept))
for name, u := range kept { for name, u := range kept {
hashes[name] = u.PasswordHash // A grant names an account whose credential is sealed to its node, never one only minted: the assigner wants a
// node only once the grant names its account (issue 1945, rule 6 per node).
if u.Sealed {
hashes[name] = u.PasswordHash
}
} }
current, err := currentFingerprints(ctx, inv) current, err := currentFingerprints(ctx, inv)
if err != nil { if err != nil {
+101 -31
View File
@@ -9,8 +9,10 @@ import (
"net" "net"
"os" "os"
"path/filepath" "path/filepath"
"slices"
"sort" "sort"
"strings" "strings"
"sync"
"time" "time"
"git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -965,24 +967,91 @@ func processPrincipalFor(m catalogue.Manifest, processID string) (broker.Process
return broker.ProcessPrincipal{}, false return broker.ProcessPrincipal{}, false
} }
// mintOnlyIfAbsent marks a context whose issue mints only accounts no row names yet (onlyIfAbsent). // mintOnlyIfAbsent marks a context whose issue mints only accounts that are not issued (onlyIfAbsent).
type mintOnlyIfAbsent struct{} type mintOnlyIfAbsent struct{}
// onlyIfAbsent is ctx for an issue that must never replace a minted account: the assign act's and the assignments' // onlyIfAbsent is ctx for an issue that never replaces an issued account: the assign act's, the assignments' watch's
// watch's, which may run at once for one account (novox/hq ADR 0543, the review of issue 1945). // and a registration's, which may run at once for one account (novox/hq ADR 0543, issue 1945).
func onlyIfAbsent(ctx context.Context) context.Context { func onlyIfAbsent(ctx context.Context) context.Context {
return context.WithValue(ctx, mintOnlyIfAbsent{}, true) return context.WithValue(ctx, mintOnlyIfAbsent{}, true)
} }
// mintBusUser mints u's password. By default it replaces whatever stands (a rotation: `module issue`); under // issuedNow collects the accounts an issue minted and sealed, for the line that says it (withIssued).
// onlyIfAbsent it mints only when no row names u, and won false says another caller minted it first and seals it, type issuedNow struct {
// so this one seals nothing. mu sync.Mutex
func mintBusUser(ctx context.Context, inv *inventory.Inventory, u inventory.BusUser) (string, bool, error) { users []string
if only, _ := ctx.Value(mintOnlyIfAbsent{}).(bool); only { }
return inv.MintBusPasswordIfAbsent(ctx, u)
type issuedNowKey struct{}
// withIssued is ctx collecting the accounts its issues mint and seal.
func withIssued(ctx context.Context) (context.Context, *issuedNow) {
got := &issuedNow{}
return context.WithValue(ctx, issuedNowKey{}, got), got
}
func (g *issuedNow) list() []string {
if g == nil {
return nil
} }
password, err := inv.MintBusPassword(ctx, u) g.mu.Lock()
return password, err == nil, err defer g.mu.Unlock()
return slices.Clone(g.users)
}
// accountIssuer is what issuing one account needs of the store.
type accountIssuer interface {
LockBusAccount(ctx context.Context, username string) (func(), error)
BusAccountIssued(ctx context.Context, username string) (minted, sealed bool, err error)
MintBusPassword(ctx context.Context, u inventory.BusUser) (string, error)
MintBusPasswordUnsealed(ctx context.Context, u inventory.BusUser) (string, error)
MarkBusAccountSealed(ctx context.Context, username string) error
}
// issueAccount mints u's password and seals it to its node with seal. By default it replaces whatever stands (a
// rotation: `module issue`). Under onlyIfAbsent it issues only an account that is not issued, a row with no sealed
// credential counting as none (issue 1945, live on 2026-10-25 at 20:04: a row whose credential the node no longer held
// left ace's mesh-build-go with no credential), under the store's lock on the account, so two issuers never interleave
// their look, mint and seal: the hash the bus holds is always the password sealed. A seal that fails leaves the
// account unsealed, issued again by the next issue.
func issueAccount(ctx context.Context, accounts accountIssuer, u inventory.BusUser, seal func(password string) error) error {
only, _ := ctx.Value(mintOnlyIfAbsent{}).(bool)
if !only {
password, err := accounts.MintBusPassword(ctx, u)
if err != nil {
return err
}
if err := seal(password); err != nil {
return err
}
} else {
release, err := accounts.LockBusAccount(ctx, u.Username)
if err != nil {
return err
}
defer release()
if minted, sealed, err := accounts.BusAccountIssued(ctx, u.Username); err != nil {
return err
} else if minted && sealed {
return nil
}
password, err := accounts.MintBusPasswordUnsealed(ctx, u)
if err != nil {
return err
}
if err := seal(password); err != nil {
return err
}
if err := accounts.MarkBusAccountSealed(ctx, u.Username); err != nil {
return fmt.Errorf("%s is sealed and not marked so, and is issued again by the next issue: %w", u.Username, err)
}
}
if got, _ := ctx.Value(issuedNowKey{}).(*issuedNow); got != nil {
got.mu.Lock()
got.users = append(got.users, u.Username)
got.mu.Unlock()
}
return nil
} }
// issueOnTheNewBus gives an assigned module its credential on the bus being built. // issueOnTheNewBus gives an assigned module its credential on the bus being built.
@@ -1010,17 +1079,12 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
} }
if _, reads := m.OwnSecrets["broker"]; reads { if _, reads := m.OwnSecrets["broker"]; reads {
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username() user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
password, won, err := mintBusUser(ctx, inv, inventory.BusUser{ if err := issueAccount(ctx, inv, inventory.BusUser{Username: user, Kind: busKindOf(m.Module), Node: node,
Username: user, Kind: busKindOf(m.Module), Node: node, Module: m.Module, Module: m.Module}, func(password string) error {
}) return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
if err != nil { }); err != nil {
return err return err
} }
if won {
if err := issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password); err != nil {
return err
}
}
} }
// Every process with a bus declaration gets an account of its own (novox/hq ADR 0436 §1, §2), minted and // Every process with a bus declaration gets an account of its own (novox/hq ADR 0436 §1, §2), minted and
// delivered whether or not the module itself speaks on the bus — its own secret is broker-<process id>, // delivered whether or not the module itself speaks on the bus — its own secret is broker-<process id>,
@@ -1043,12 +1107,15 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
func issueProcess(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, pr broker.ProcessPrincipal, func issueProcess(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, pr broker.ProcessPrincipal,
node, busAddress string, known broker.Broker, reachable string) error { node, busAddress string, known broker.Broker, reachable string) error {
user := broker.Principal{Kind: broker.KindProcess, Node: node, Module: m.Module, Process: pr.Process}.Username() user := broker.Principal{Kind: broker.KindProcess, Node: node, Module: m.Module, Process: pr.Process}.Username()
password, won, err := mintBusUser(ctx, inv, inventory.BusUser{ return issueAccount(ctx, inv, inventory.BusUser{Username: user, Kind: inventory.BusProcess, Node: node, Module: m.Module},
Username: user, Kind: inventory.BusProcess, Node: node, Module: m.Module, func(password string) error {
}) return sealProcess(ctx, inv, m, pr, node, busAddress, known, reachable, user, password)
if err != nil || !won { })
return err // lost: the caller that minted it seals it }
}
// sealProcess seals a process's minted account to its node, hands it to the vault and makes its work-queue consumers.
func sealProcess(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, pr broker.ProcessPrincipal,
node, busAddress string, known broker.Broker, reachable, user, password string) error {
name := processSecretName(pr.Process) name := processSecretName(pr.Process)
held, err := heldProcessAccount(known, reachable, node, m.Module, user, password) held, err := heldProcessAccount(known, reachable, node, m.Module, user, password)
if err != nil { if err != nil {
@@ -1113,12 +1180,15 @@ func runsTheProvisionerClient(m catalogue.Manifest) bool {
func issueProvisionerClient(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string, func issueProvisionerClient(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string,
known broker.Broker, reachable string) error { known broker.Broker, reachable string) error {
user := broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username() user := broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username()
password, won, err := mintBusUser(ctx, inv, inventory.BusUser{ return issueAccount(ctx, inv, inventory.BusUser{Username: user, Kind: inventory.BusProvisioner, Node: node,
Username: user, Kind: inventory.BusProvisioner, Node: node, Module: m.Module, Module: m.Module}, func(password string) error {
return sealProvisionerClient(ctx, inv, m, node, known, reachable, user, password)
}) })
if err != nil || !won { }
return err // lost: the caller that minted it seals it
} // sealProvisionerClient seals the provisioner client's minted account to its machine and hands it to the vault.
func sealProvisionerClient(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string,
known broker.Broker, reachable, user, password string) error {
held, err := json.Marshal(struct { held, err := json.Marshal(struct {
URL string `json:"url"` URL string `json:"url"`
Fingerprint string `json:"fingerprint,omitempty"` Fingerprint string `json:"fingerprint,omitempty"`
+6 -3
View File
@@ -94,15 +94,18 @@ func issueMissingProcessAccounts(ctx context.Context, inv *inventory.Inventory,
if err != nil { if err != nil {
busAddress = "" busAddress = ""
} }
var ids []string ctx, got := withIssued(ctx)
for _, pr := range missing { for _, pr := range missing {
if err := issueProcess(ctx, inv, m, pr, node, busAddress, known, reachable); err != nil { if err := issueProcess(ctx, inv, m, pr, node, busAddress, known, reachable); err != nil {
return "", err return "", err
} }
ids = append(ids, pr.Process) }
issued := got.list()
if len(issued) == 0 {
return "", nil // another issuer sealed them meanwhile
} }
return fmt.Sprintf("its process account(s) %s issued and sealed to %s, and arrive with the push", return fmt.Sprintf("its process account(s) %s issued and sealed to %s, and arrive with the push",
strings.Join(ids, ", "), node), nil strings.Join(issued, ", "), node), nil
} }
// issueGainedProcessAccounts runs after a module's new definition is registered: on each machine assigned it, the // issueGainedProcessAccounts runs after a module's new definition is registered: on each machine assigned it, the
+81 -32
View File
@@ -7,6 +7,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"sync/atomic" "sync/atomic"
"time"
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
"golang.org/x/crypto/bcrypt" "golang.org/x/crypto/bcrypt"
@@ -52,6 +53,8 @@ type BusUser struct {
// PasswordHash is what the composed file carries. The plaintext is returned once, by Mint, and // PasswordHash is what the composed file carries. The plaintext is returned once, by Mint, and
// then exists only where it was sealed. // then exists only where it was sealed.
PasswordHash string PasswordHash string
// Sealed says its credential reached its node: an issue's mint is not sealed until then (ADR 0543 §2).
Sealed bool
} }
// The kinds of bus user the mesh records. The same words the composer uses, so a row and a // The kinds of bus user the mesh records. The same words the composer uses, so a row and a
@@ -100,52 +103,98 @@ func (i *Inventory) RecordBusPassword(ctx context.Context, u BusUser, password s
return i.writeBusUser(ctx, u, string(hash)) return i.writeBusUser(ctx, u, string(hash))
} }
// writeBusUser is the row, whoever chose the password. // writeBusUser is the row, whoever chose the password, taken as sealed.
func (i *Inventory) writeBusUser(ctx context.Context, u BusUser, hash string) error { func (i *Inventory) writeBusUser(ctx context.Context, u BusUser, hash string) error {
return i.writeBusUserSealed(ctx, u, hash, true)
}
// writeBusUserSealed is the row, sealed now or not yet.
func (i *Inventory) writeBusUserSealed(ctx context.Context, u BusUser, hash string, sealed bool) error {
if _, err := i.store.Pool().Exec(ctx, if _, err := i.store.Pool().Exec(ctx,
`insert into bus_user (username, kind, node, module, password_hash) `insert into bus_user (username, kind, node, module, password_hash, sealed_at)
values ($1, $2, $3, $4, $5) values ($1, $2, $3, $4, $5, case when $6 then now() end)
on conflict (username) do update on conflict (username) do update
set kind = excluded.kind, node = excluded.node, module = excluded.module, set kind = excluded.kind, node = excluded.node, module = excluded.module,
password_hash = excluded.password_hash, minted_at = now()`, password_hash = excluded.password_hash, minted_at = now(), sealed_at = excluded.sealed_at`,
u.Username, u.Kind, u.Node, u.Module, hash); err != nil { u.Username, u.Kind, u.Node, u.Module, hash, sealed); err != nil {
return fmt.Errorf("cannot record the bus user %s: %w", u.Username, err) return fmt.Errorf("cannot record the bus user %s: %w", u.Username, err)
} }
return nil return nil
} }
// MintBusPasswordIfAbsent mints u's password only when no row names u yet, in one statement: of two callers that both // LockBusAccount holds the store's lock on one account until the answered release is called, so two issuers of it
// found the account missing, exactly one wins, and the other is told it lost (won false, no password) and must seal // (the assign act, the assignments' watch, a registration's process accounts) never interleave their look, mint and
// nothing, so the hash the bus holds is always the password the node is sealed (novox/hq ADR 0543, the review of // seal: the hash the bus holds is always the password sealed to the node (novox/hq ADR 0543 §2, issue 1945).
// issue 1945: the assign act and the assignments' watch once each minted, the later hash beside the earlier seal). func (i *Inventory) LockBusAccount(ctx context.Context, username string) (func(), error) {
func (i *Inventory) MintBusPasswordIfAbsent(ctx context.Context, u BusUser) (string, bool, error) { conn, err := i.store.Pool().Acquire(ctx)
if u.Username == "" || u.Kind == "" {
return "", false, errors.New("a bus user needs a username and a kind")
}
raw := make([]byte, 32)
if _, err := rand.Read(raw); err != nil {
return "", false, fmt.Errorf("cannot generate a bus password: %w", err)
}
password := base64.RawURLEncoding.EncodeToString(raw)
hash, err := hashBusPassword(password)
if err != nil { if err != nil {
return "", false, fmt.Errorf("cannot hash a bus password: %w", err) return nil, fmt.Errorf("the lock on %s cannot be taken: %w", username, err)
} }
tag, err := i.store.Pool().Exec(ctx, if _, err := conn.Exec(ctx, `select pg_advisory_lock(hashtext($1))`, "bus_user/"+username); err != nil {
`insert into bus_user (username, kind, node, module, password_hash) conn.Release()
values ($1, $2, $3, $4, $5) return nil, fmt.Errorf("the lock on %s cannot be taken: %w", username, err)
on conflict (username) do nothing`, }
u.Username, u.Kind, u.Node, u.Module, string(hash)) return func() {
_, _ = conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock(hashtext($1))`, "bus_user/"+username)
conn.Release()
}, nil
}
// ForgetBusAccountsOf forgets the bus accounts of module on node, and the credentials sealed to the node for them (its
// `broker`, each process's `broker-<id>`, its provisioner client's): an unassigned module keeps no account, so an
// assignment again mints and seals afresh, never trusting a row whose credential the node no longer holds (issue 1945,
// live on 2026-10-25 at 20:04: ace's mesh-build-go rows from 17:19 survived its unassignment, nothing was minted or
// sealed again, and the module found no credential).
func (i *Inventory) ForgetBusAccountsOf(ctx context.Context, node, module string) error {
if node == "" || module == "" {
return errors.New("forgetting the bus accounts of no node or no module would forget others' accounts")
}
tx, err := i.store.Pool().Begin(ctx)
if err != nil { if err != nil {
return "", false, fmt.Errorf("cannot record the bus user %s: %w", u.Username, err) return err
} }
if tag.RowsAffected() == 0 { defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
return "", false, nil if _, err := tx.Exec(ctx, `delete from bus_user where node = $1 and module = $2 and kind <> $3`,
node, module, BusController); err != nil {
return fmt.Errorf("the bus accounts of %s on %s are not forgotten: %w", module, node, err)
} }
return password, true, nil if _, err := tx.Exec(ctx,
`delete from module_secret s using node n
where s.node = n.id and n.name = $1 and s.module = $2
and (s.name = 'broker' or s.name like 'broker-%' or s.name = 'provisioner-broker')`, node, module); err != nil {
return fmt.Errorf("the bus credentials of %s on %s are not forgotten: %w", module, node, err)
}
return tx.Commit(ctx)
}
// MintBusPasswordUnsealed mints u's password, replacing whatever stands, with no seal yet: the account is not issued
// until MarkBusAccountSealed says its credential reached the node (ADR 0543 §2).
func (i *Inventory) MintBusPasswordUnsealed(ctx context.Context, u BusUser) (string, error) {
return i.mintBusPassword(ctx, u, false)
}
// MarkBusAccountSealed says the credential last minted for username is sealed to its node.
func (i *Inventory) MarkBusAccountSealed(ctx context.Context, username string) error {
_, err := i.store.Pool().Exec(ctx, `update bus_user set sealed_at = now() where username = $1`, username)
return err
}
// BusAccountIssued says whether an account is minted, and whether its credential is sealed to its node.
func (i *Inventory) BusAccountIssued(ctx context.Context, username string) (minted, sealed bool, err error) {
var at *time.Time
err = i.store.Pool().QueryRow(ctx, `select sealed_at from bus_user where username = $1`, username).Scan(&at)
if errors.Is(err, pgx.ErrNoRows) {
return false, false, nil
}
return err == nil, err == nil && at != nil, err
} }
func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, error) { func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, error) {
return i.mintBusPassword(ctx, u, true)
}
// mintBusPassword mints, marked sealed when the caller seals it itself in the same act (every mint but an issue's).
func (i *Inventory) mintBusPassword(ctx context.Context, u BusUser, sealed bool) (string, error) {
if u.Username == "" || u.Kind == "" { if u.Username == "" || u.Kind == "" {
return "", errors.New("a bus user needs a username and a kind") return "", errors.New("a bus user needs a username and a kind")
} }
@@ -163,7 +212,7 @@ func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, err
return "", fmt.Errorf("cannot hash a bus password: %w", err) return "", fmt.Errorf("cannot hash a bus password: %w", err)
} }
if err := i.writeBusUser(ctx, u, string(hash)); err != nil { if err := i.writeBusUserSealed(ctx, u, string(hash), sealed); err != nil {
return "", err return "", err
} }
return password, nil return password, nil
@@ -176,7 +225,7 @@ func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, err
// password, and the composer refuses it rather than writing a user anybody is. // password, and the composer refuses it rather than writing a user anybody is.
func (i *Inventory) BusUsers(ctx context.Context) (map[string]BusUser, error) { func (i *Inventory) BusUsers(ctx context.Context) (map[string]BusUser, error) {
rows, err := i.store.Pool().Query(ctx, rows, err := i.store.Pool().Query(ctx,
`select username, kind, node, module, password_hash from bus_user order by username`) `select username, kind, node, module, password_hash, sealed_at is not null from bus_user order by username`)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -184,7 +233,7 @@ func (i *Inventory) BusUsers(ctx context.Context) (map[string]BusUser, error) {
out := map[string]BusUser{} out := map[string]BusUser{}
for rows.Next() { for rows.Next() {
var u BusUser var u BusUser
if err := rows.Scan(&u.Username, &u.Kind, &u.Node, &u.Module, &u.PasswordHash); err != nil { if err := rows.Scan(&u.Username, &u.Kind, &u.Node, &u.Module, &u.PasswordHash, &u.Sealed); err != nil {
return nil, err return nil, err
} }
out[u.Username] = u out[u.Username] = u
+42 -34
View File
@@ -4,7 +4,9 @@ package inventory
import ( import (
"context" "context"
"slices"
"testing" "testing"
"time"
"golang.org/x/crypto/bcrypt" "golang.org/x/crypto/bcrypt"
) )
@@ -63,48 +65,54 @@ func TestMintingAgainRotatesAndMintingIfAbsentHasOneWinner(t *testing.T) {
t.Fatal("the previous password still verifies, so a rotation revoked nothing") t.Fatal("the previous password still verifies, so a rotation revoked nothing")
} }
// **Minting only if absent never rotates, and of several at once exactly one wins** (ADR 0543, the review of issue // **An issue's mint is not issued until its seal is marked, and forgotten with its assignment** (ADR 0543 §2, issue
// 1945): the assign act and the assignments' watch may both find an account missing; the winner's password is the // 1945, live on 2026-10-25 at 20:04): a row whose credential the node does not hold is not issued.
// one the store's hash verifies, so the one it seals is the one the bus accepts, and a loser seals nothing. if minted, sealed, err := inv.BusAccountIssued(ctx, "one.shop"); err != nil || !minted || !sealed {
if _, won, err := inv.MintBusPasswordIfAbsent(ctx, aBusUser("shop")); err != nil || won { t.Fatalf("a mint that seals in its own act: minted %v sealed %v (%v)", minted, sealed, err)
t.Fatalf("a mint-if-absent over a minted account won %v (%v): it would replace a sealed password", won, err)
} }
const racers = 8 if _, err := inv.MintBusPasswordUnsealed(ctx, aBusUser("racer")); err != nil {
type minted struct { t.Fatal(err)
password string
won bool
err error
} }
results := make(chan minted, racers) if minted, sealed, err := inv.BusAccountIssued(ctx, "one.racer"); err != nil || !minted || sealed {
for range racers { t.Fatalf("an issue's mint before its seal: minted %v sealed %v (%v)", minted, sealed, err)
go func() {
p, won, err := inv.MintBusPasswordIfAbsent(ctx, aBusUser("racer"))
results <- minted{p, won, err}
}()
} }
var winner string if names, err := inv.BusMinted(ctx); err != nil || slices.Contains(names, "one.racer") {
wins := 0 t.Fatalf("an account not sealed counts as minted: %v (%v)", names, err)
for range racers {
r := <-results
if r.err != nil {
t.Fatal(r.err)
}
if r.won {
wins++
winner = r.password
} else if r.password != "" {
t.Fatal("a caller that lost was handed a password to seal")
}
} }
if wins != 1 { if err := inv.MarkBusAccountSealed(ctx, "one.racer"); err != nil {
t.Fatalf("%d of %d concurrent mints won, want exactly one", wins, racers) t.Fatal(err)
} }
stored, _, err := inv.BusUserHash(ctx, "one.racer") if _, sealed, _ := inv.BusAccountIssued(ctx, "one.racer"); !sealed {
t.Fatal("an account marked sealed is not")
}
// The lock on an account holds a second issuer until the first lets go.
release, err := inv.LockBusAccount(ctx, "one.racer")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := bcrypt.CompareHashAndPassword([]byte(stored), []byte(winner)); err != nil { waiting := make(chan struct{})
t.Fatal("the stored hash is not the winner's password: the node would be sealed a password the bus refuses") go func() {
again, err := inv.LockBusAccount(ctx, "one.racer")
if err == nil {
again()
}
close(waiting)
}()
select {
case <-waiting:
t.Fatal("a second issuer took the lock the first held")
case <-time.After(200 * time.Millisecond):
}
release()
<-waiting
if err := inv.ForgetBusAccountsOf(ctx, "one", "racer"); err != nil {
t.Fatal(err)
}
if minted, _, _ := inv.BusAccountIssued(ctx, "one.racer"); minted {
t.Fatal("an unassigned module's account was kept")
}
if minted, _, _ := inv.BusAccountIssued(ctx, "one.shop"); !minted {
t.Fatal("forgetting one module's accounts forgot another's")
} }
} }
+4
View File
@@ -670,6 +670,10 @@ func (i *Inventory) Unassign(ctx context.Context, nodeName, module string) error
if err := i.chainUnassign(ctx, c, node, module); err != nil { if err := i.chainUnassign(ctx, c, node, module); err != nil {
return err return err
} }
// Its bus accounts on the node go with it: an assignment again issues them afresh (issue 1945).
if err := i.ForgetBusAccountsOf(ctx, nodeName, module); err != nil {
return err
}
// The seat it held falls back to derivation rather than pointing at nothing: what the store's foreign key did. // The seat it held falls back to derivation rather than pointing at nothing: what the store's foreign key did.
if _, err := i.store.Pool().Exec(ctx, `delete from seat_holding where node = $1 and module = $2`, node.ID, module); err != nil { if _, err := i.store.Pool().Exec(ctx, `delete from seat_holding where node = $1 and module = $2`, node.ID, module); err != nil {
return err return err
@@ -0,0 +1,8 @@
-- A bus account says when the credential minted for it was sealed to its node (novox/hq ADR 0543 §2, issue 1945). The
-- assign act and the assignments' watch issue an account only when it is not issued: a row with no sealed credential
-- is not issued. On 2026-10-25 at 20:04 ace's mesh-build-go had a row minted at 17:19 whose credential the node no
-- longer held; minting nothing, the mesh sealed nothing, and the module crash-looped for its missing credential. Null
-- while an issue mints and seals; set once the credential is handed over. Every account standing before this migration
-- is taken as sealed: each works or worked, and its own issue sealed it.
alter table bus_user add column sealed_at timestamptz;
update bus_user set sealed_at = minted_at;
+4 -2
View File
@@ -226,8 +226,10 @@ func (i *Inventory) BusMinted(ctx context.Context) ([]string, error) {
return nil, err return nil, err
} }
out := make([]string, 0, len(kept)) out := make([]string, 0, len(kept))
for name := range kept { for name, u := range kept {
out = append(out, name) if u.Sealed { // an account whose credential is not sealed to its node is not issued (ADR 0543 §2)
out = append(out, name)
}
} }
sort.Strings(out) sort.Strings(out)
return out, nil return out, nil