mesh-grants: take each user's hash from the vault's bus-accounts and say per user whether one is held, so take-over waits until every user has a credential

ADR 0547 clause 9: the vault writes the hash of each bus account it holds into bus-accounts.<user>. The grants part
watches that state (no timer), carries the hash with each shadow user in memory only, and its user-list tool says per
user whether a hash is held and how its line would differ from the loaded one (same, adds, left out for want of a
credential, loses its login); never the hash, since bcrypt salts make two hashes incomparable. Take-over names the
users with no hash.

Issue: 1945

modules/mesh-grants/cmd/mesh-grants/tools.go

@@ -76,7 +76,9 @@ func toolsOf(p *grants.Part) []stdio.Tool {
 		{Name: grants.ToolUserList, Description: "The bus's user list in shadow (ADR 0547 clauses 6 and 8): the list the grants part " +
 			"composed (digest, revision, users), what the bus node says it loaded (source, revision, digest, users, error), and how " +
 			"they differ: users added, users dropped (and of those, gone from placement), each user's grant differences, the " +
-			"assignments left out, and every input the bus does not carry, named. Read-only.",
+			"assignments left out, every input the bus does not carry, named; and per user whether the vault holds its hash " +
+			"(mesh-vault.bus-accounts, ADR 0547 clause 9) and how its line would differ from the loaded one (same, adds, left-out " +
+			"for want of a credential, loses-login), never a hash. Read-only.",
 			Run: func(map[string]any) (any, error) { return p.UserList() }},
 		{Name: grants.ToolComposeNow, Description: "Compose again now (ADR 0547 clause 6), as the part does on every change it hears: asks " +
 			"the part's process (which alone holds the grant to publish a membership) to compose the memberships, the user list and " +

modules/mesh-grants/grants_test.go

@@ -385,6 +385,57 @@ func TestGrants(t *testing.T) {
 		}
 	})
 
+	t.Run("credentials", func(t *testing.T) {
+		shadow := []grants.User{{User: "a.held"}, {User: "a.new"}, {User: "a.bare"}}
+		loaded := []grants.User{{User: "a.held"}, {User: "a.bare"}, {User: "enrol.x"}, {User: "person.op"}}
+		hashes := map[string]grants.HeldHash{"a.held": {Hash: "$2a$10$h1", Generation: 2}, "a.new": {Hash: "$2a$10$h2", Generation: 1},
+			"person.op": {Hash: "$2a$10$h3", Adopted: true}}
+		c := grants.CredentialsOf(shadow, loaded, hashes)
+		for _, row := range []struct {
+			user, would   string
+			held, adopted bool
+		}{
+			{"a.bare", grants.WouldLeaveOut, false, false},
+			{"a.held", grants.WouldSame, true, false},
+			{"a.new", grants.WouldAdd, true, false},
+			{"enrol.x", grants.WouldLoseLogin, false, false},
+			{"person.op", grants.WouldNotComposed, true, true},
+		} {
+			i := slices.IndexFunc(c.Users, func(u grants.Credential) bool { return u.User == row.user })
+			if i < 0 || c.Users[i].Would != row.would || c.Users[i].HashHeld != row.held || c.Users[i].Adopted != row.adopted {
+				t.Errorf("%s: %+v, want %s held %v", row.user, c.Users, row.would, row.held)
+			}
+		}
+		if c.HashesHeld != 3 || !slices.Equal(c.LeftOut, []string{"a.bare"}) || !slices.Equal(c.LosesLogin, []string{"enrol.x"}) ||
+			!slices.Equal(c.Adds, []string{"a.new"}) {
+			t.Errorf("credentials = %+v", c)
+		}
+		// Composed from the vault's state: a hash changes the list's state (written on the event), and no hash is ever
+		// in what the part writes or says.
+		b := fixture()
+		b.loaded = shadowOf(t, b)
+		p := partOn(b)
+		before := p.Compose()
+		b.put(grants.ReadAccounts, "a.mod-x", map[string]any{"user": "a.mod-x", "hash": "$2a$10$secrethashbytes", "generation": 1})
+		after := p.Compose()
+		cr := after.UserList.Credentials
+		if !after.Written || after.Revision != before.Revision+1 || cr.HashesHeld != 1 || slices.Contains(cr.LeftOut, "a.mod-x") ||
+			!slices.Contains(cr.LeftOut, "b.mod-y") {
+			t.Errorf("a hash heard = %+v", after.UserList.Credentials)
+		}
+		err := p.TakeOver("person.operator", "hashes")
+		if err == nil || !strings.Contains(err.Error(), "no hash for") || strings.Contains(err.Error(), "a.mod-x,") {
+			t.Errorf("take-over = %v, want the users with no hash named", err)
+		}
+		for state, keys := range b.own {
+			for k, v := range keys {
+				if strings.Contains(string(v), "secrethashbytes") {
+					t.Errorf("the hash is written in %s %s", state, k)
+				}
+			}
+		}
+	})
+
 	t.Run("compare", func(t *testing.T) {
 		b := fixture()
 		shadow := shadowOf(t, b)

modules/mesh-grants/internal/grants/follow.go

@@ -16,8 +16,8 @@ type Watcher interface {
 
 // Followed are the states whose change composes again: placement, the indexer's manifests by module, the resolved
 // config values, the node agents' haves (a machine's first have, and the bus node's loaded list) and the assigner's
-// wants (the manifest each machine is wanted to run).
-var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants}
+// wants (the manifest each machine is wanted to run), and the vault's hashes of the bus accounts.
+var Followed = []string{ReadAssignments, ReadModules, ReadResolved, ReadHave, ReadWants, ReadAccounts}
 
 // Follower composes again on every change it hears, never on a timer. It never stops for a refusal: a watch it cannot
 // open is said, composed without, and opened again at the next wake (any change it does hear, or a reconnect).

modules/mesh-grants/internal/grants/inputs.go

@@ -28,10 +28,13 @@ const (
 	ReadResolved        = "mesh-config.resolved"
 	ReadWants           = "mesh-delivery-assigner.wants"
 	ReadBuilds          = "mesh-delivery-builder.builds"
+	// ReadAccounts is the vault's bcrypt hashes of the bus accounts it holds, by bus user (ADR 0547 clause 9).
+	ReadAccounts = "mesh-vault.bus-accounts"
 )
 
 // Reads is every state the part reads, in the manifest's order.
-var Reads = []string{ReadAssignments, ReadControllerGrant, ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds}
+var Reads = []string{ReadAssignments, ReadControllerGrant, ReadModules, ReadManifests, ReadResolved, ReadHave, ReadWants, ReadBuilds,
+	ReadAccounts}
 
 // The bus module's tools the part calls, as its manifest's invokes name them: read-only, each.
 const (
@@ -67,7 +70,6 @@ var standingGaps = []Gap{
 	{Input: "people", Why: "the people and the tools they may invoke are in the controller's store alone: no person user is composed"},
 	{Input: "seat holdings", Why: "who holds each seat held once for the mesh is in the controller's store alone: a mesh seat claimed on two machines is composed as held on both"},
 	{Input: "announced states", Why: "the controller does not announce its states-announced readable, so every read is composed from `reads`, as when announcements cannot be read"},
-	{Input: "credentials", Why: "the shadow holds no hash, so a user it composes that the loaded list lacks may be one left out for want of a credential"},
 }
 
 // assignment is a key of the controller's assignments as composing needs it (mesh-controller internal/chain Assignment):
@@ -108,7 +110,32 @@ type Inputs struct {
 	// BusNode is the machine running the bus's module, and Loaded what its node agent says it loaded.
 	BusNode string
 	Loaded  *delivery.UserListWritten
-	Gaps    []Gap
+	// Hashes are the vault's hashes of the bus accounts, by bus user (ReadAccounts); kept in memory, never written.
+	Hashes map[string]HeldHash
+	Gaps   []Gap
+}
+
+// HeldHash is one key of the vault's bus-accounts: the hash, and what it is of. The hash is never said or written by
+// the part in shadow.
+type HeldHash struct {
+	Hash       string    `json:"hash"`
+	Generation int       `json:"generation,omitempty"`
+	At         time.Time `json:"at"`
+	Adopted    bool      `json:"adopted,omitempty"`
+}
+
+// readHashes reads the vault's bus-accounts; a refusal is a gap, and every user then has no hash held.
+func readHashes(r Reader, gaps *[]Gap) map[string]HeldHash {
+	out := map[string]HeldHash{}
+	for user, raw := range each(r, ReadAccounts, gaps) {
+		var h HeldHash
+		if json.Unmarshal(raw, &h) != nil || h.Hash == "" {
+			*gaps = append(*gaps, Gap{Input: ReadAccounts + " " + user, Why: "holds no hash"})
+			continue
+		}
+		out[user] = h
+	}
+	return out
 }
 
 // BusModule is the bus's module: it holds mesh-broker.
@@ -301,6 +328,7 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
 		sort.Strings(fromTrunk)
 		in.Gaps = append(in.Gaps, Gap{Input: "wanted manifests", Why: "no build record names the manifest at the wanted fingerprint, so the trunk's is used: " + strings.Join(fromTrunk, ", ")})
 	}
+	in.Hashes = readHashes(r, &in.Gaps)
 	for _, raw := range each(r, ReadResolved, &in.Gaps) {
 		var v resolved
 		if json.Unmarshal(raw, &v) != nil || v.Node == "" || v.Module == "" || v.Key == "" || len(v.Value) == 0 {

modules/mesh-grants/internal/grants/part.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"slices"
 	"sort"
 	"strings"
 	"sync"
@@ -66,9 +67,11 @@ type UserListState struct {
 	LeftOut []Gap                     `json:"left_out"`
 	Compare Comparison                `json:"compare"`
 	Loaded  *delivery.UserListWritten `json:"loaded,omitempty"`
-	BusNode string                    `json:"bus_node,omitempty"`
-	Gaps    []Gap                     `json:"gaps"`
-	At      time.Time                 `json:"at"`
+	// Credentials says per user whether the vault holds its hash and how its line would differ (ADR 0547 clause 9).
+	Credentials Credentials `json:"credentials"`
+	BusNode     string      `json:"bus_node,omitempty"`
+	Gaps        []Gap       `json:"gaps"`
+	At          time.Time   `json:"at"`
 }
 
 func (s UserListState) inputs() string {
@@ -141,12 +144,14 @@ func (p *Part) Compose() Result {
 	if err != nil {
 		res.Errors = append(res.Errors, "the shadow list could not be composed: "+err.Error())
 	}
+	users = WithHashes(users, in.Hashes)
 	st.Digest, st.Users = Digest(users), len(users)
 	loaded, err := LoadedUsers(p.Call)
 	st.Compare = Compare(users, loaded, Placed(in))
 	if err != nil {
 		st.Compare.Unread = err.Error()
 	}
+	st.Credentials = CredentialsOf(users, loaded, in.Hashes)
 	st.Inputs = st.inputs()
 	res.UserList = st
 	if raw, rev, found, err := p.Store.Get(StateUserList, KeyList); err != nil {
@@ -324,6 +329,18 @@ func (p *Part) TakeOver(caller, why string) error {
 			reasons = append(reasons, fmt.Sprintf("%d users differ between the shadow list and the loaded list: %s", len(d), strings.Join(d, ", ")))
 		}
 	}
+	cr, gone := res.UserList.Credentials, res.UserList.Compare.Gone
+	var unhashed []string
+	for _, u := range append(append([]string(nil), cr.LeftOut...), cr.LosesLogin...) {
+		if !slices.Contains(gone, u) {
+			unhashed = append(unhashed, u)
+		}
+	}
+	if len(unhashed) > 0 {
+		sort.Strings(unhashed)
+		reasons = append(reasons, fmt.Sprintf("the vault holds no hash for %d users (ADR 0547 clause 9: handed or adopted before take-over): %s",
+			len(unhashed), strings.Join(unhashed, ", ")))
+	}
 	reasons = append(reasons, NotComposable)
 	a.Refused = "take-over refused: " + strings.Join(reasons, "; ")
 	return fmt.Errorf("%s%s", a.Refused, p.record(a))

modules/mesh-grants/internal/grants/userlist.go

@@ -18,6 +18,96 @@ type User struct {
 	Publish        []string `json:"publish"`
 	Subscribe      []string `json:"subscribe"`
 	AllowResponses bool     `json:"allow_responses"`
+	// Hash is the bcrypt hash the vault holds for the user (WithHashes): in memory alone, never in a digest, a compare
+	// or the part's state, and empty for a loaded user.
+	Hash string `json:"-"`
+}
+
+// WithHashes gives each user of the shadow list the hash the vault holds for it, when it holds one.
+func WithHashes(users []User, hashes map[string]HeldHash) []User {
+	for i := range users {
+		users[i].Hash = hashes[users[i].User].Hash
+	}
+	return users
+}
+
+// Credential is one user's hash as the list would carry it: whether the vault holds one, and how the user stands in the
+// shadow list and the loaded list. Never the hash: a bcrypt hash of the same password differs by its salt, so two hashes
+// say nothing compared.
+type Credential struct {
+	User     string `json:"user"`
+	InShadow bool   `json:"in_shadow"`
+	Loaded   bool   `json:"loaded"`
+	HashHeld bool   `json:"hash_held"`
+	Adopted  bool   `json:"adopted,omitempty"`
+	// Generation is the vault's generation of the account the hash is of; zero for an adopted one.
+	Generation int `json:"generation,omitempty"`
+	// Would is how the list with the vault's hashes would differ from the loaded one for this user: same (a hash held,
+	// loaded), adds (a hash held, not loaded), left-out (in the shadow list, no hash: left out for want of a
+	// credential), loses-login (loaded, no hash held: its login would be lost) or not-composed (a hash held, loaded,
+	// and not in the shadow list).
+	Would string `json:"would"`
+}
+
+// The ways a user's line would differ.
+const (
+	WouldSame        = "same"
+	WouldAdd         = "adds"
+	WouldLeaveOut    = "left-out"
+	WouldLoseLogin   = "loses-login"
+	WouldNotComposed = "not-composed"
+)
+
+// Credentials is every user's Credential, and the users of each way that differs.
+type Credentials struct {
+	Users      []Credential `json:"users"`
+	HashesHeld int          `json:"hashes_held"`
+	LeftOut    []string     `json:"left_out"`
+	LosesLogin []string     `json:"loses_login"`
+	Adds       []string     `json:"adds"`
+}
+
+// CredentialsOf says, of every user of the shadow and the loaded list, whether the vault holds its hash and how its line
+// would differ. Pure.
+func CredentialsOf(shadow, loaded []User, hashes map[string]HeldHash) Credentials {
+	c := Credentials{Users: []Credential{}, LeftOut: []string{}, LosesLogin: []string{}, Adds: []string{}}
+	rows := map[string]*Credential{}
+	row := func(user string) *Credential {
+		if rows[user] == nil {
+			h, held := hashes[user]
+			rows[user] = &Credential{User: user, HashHeld: held, Adopted: h.Adopted, Generation: h.Generation}
+		}
+		return rows[user]
+	}
+	for _, u := range shadow {
+		row(u.User).InShadow = true
+	}
+	for _, u := range loaded {
+		row(u.User).Loaded = true
+	}
+	for _, user := range sortedKeys(rows) {
+		r := rows[user]
+		switch {
+		case r.HashHeld && r.InShadow && r.Loaded:
+			r.Would = WouldSame
+		case r.HashHeld && r.InShadow:
+			r.Would = WouldAdd
+			c.Adds = append(c.Adds, user)
+		case r.InShadow:
+			r.Would = WouldLeaveOut
+			c.LeftOut = append(c.LeftOut, user)
+		case !r.HashHeld:
+			r.Would = WouldLoseLogin
+			c.LosesLogin = append(c.LosesLogin, user)
+		default:
+			r.Would = WouldNotComposed
+		}
+		if r.HashHeld {
+			c.HashesHeld++
+		}
+		c.Users = append(c.Users, *r)
+	}
+	return c
 }
 
 // ComposeUsers is the shadow list: every user the composer derives from the records, with its grants. Pure.

modules/mesh-grants/module.json

@@ -10,7 +10,8 @@
     "mesh-delivery-builder.builds",
     "mesh-delivery-indexer.manifests",
     "mesh-delivery-indexer.modules",
-    "node-engine.have"
+    "node-engine.have",
+    "mesh-vault.bus-accounts"
   ],
   "invokes": [
     "nats.nats_streams",