mesh-grants: write grants.<module>.<fp> in shadow into the state grants, say grant-written, and show every difference from the controller's in grants_grants (issue 774, ADR 0547 §5)

Why: the assigner switches its grants source when the controller's lease is handed away; until then the written keys are compared with the controller's, so the switch rests on a measured equality. Needs the group's mesh-sdk pin (PartGrantKey) before it builds.

modules/mesh-grants/grants_test.go

@@ -1208,7 +1208,7 @@ func TestGrants(t *testing.T) {
 			t.Errorf("membership = %v", ms)
 		}
 		gr, _ := p.Grants("mod-x")
-		if g, _ := gr.(map[string]any); g["writes"] != grants.WritesNoGrant || g["count"] != 1 {
+		if g, _ := gr.(map[string]any); g["writes"] != grants.WritesGrants || g["controller_count"] != 1 {
 			t.Errorf("grants = %v", gr)
 		}
 
@@ -1568,7 +1568,7 @@ func TestGrants(t *testing.T) {
 				}
 			}
 			for _, w := range b.writes {
-				if !slices.Contains(grants.States, w) {
+				if !slices.Contains(grants.States, strings.TrimSuffix(w, " delete")) {
 					t.Errorf("wrote %s, which is not the part's own state", w)
 				}
 			}
@@ -1888,6 +1888,69 @@ func TestGrants(t *testing.T) {
 		}
 	})
 
+	// 774/L3a (novox/hq ADR 0547 §5, ADR 0561 §2): the part writes grants.<module>.<fp> in shadow, once each and only for a
+	// module whose accounts the vault holds a hash for, says grant-written for each, deletes one only when a module has no
+	// account with a hash and every read was made, and the tool says every difference from the controller's.
+	t.Run("grants: written in shadow once each, only for hashed accounts, deleted only when read whole, compared with the controller's", func(t *testing.T) {
+		b := fixture()
+		b.loaded = shadowOf(t, b)
+		p := partOn(b)
+		fp := "src2:" + strings.Repeat("ab", 32)
+		key := func(module string) string { k, _ := delivery.PartGrantKey(module, fp); return k }
+		hash := func(user string) {
+			b.put(grants.ReadAccounts, user, map[string]any{"user": user, "hash": "$2a$10$x", "generation": 1})
+		}
+		held := func() []string { return slices.Sorted(maps.Keys(b.own[grants.StateGrants])) }
+		// The controller's grant for this fingerprint, beside the stale one the fixture holds for another.
+		delete(b.states[grants.ReadControllerGrant], "mod-x.0123456789ab0123456789ab0123456789ab0123456789ab0123456789ab0123")
+		b.put(grants.ReadControllerGrant, key("mod-x"), delivery.ControllerGrant{V: delivery.Version, Accounts: []string{"a.mod-x"}})
+		hash("a.mod-x")
+		hash("a.nats")
+		res := p.Compose()
+		if res.Grants == nil || !slices.Equal(held(), []string{key("mod-x"), key("nats")}) || len(res.Grants.Written) != 2 {
+			t.Fatalf("grants of the hashed modules: held %v, %+v", held(), res.Grants)
+		}
+		if g, err := delivery.DecodeControllerGrant(b.own[grants.StateGrants][key("mod-x")]); err != nil || !slices.Equal(g.Accounts, []string{"a.mod-x"}) {
+			t.Errorf("the grant of mod-x is %+v, %v", g, err)
+		}
+		said := func() int {
+			n := 0
+			for _, e := range b.events {
+				if e == grants.EventGrantWritten {
+					n++
+				}
+			}
+			return n
+		}
+		if said() != 2 {
+			t.Errorf("grant-written said %d times for two grants: %v", said(), b.events)
+		}
+		if res := p.Compose(); len(res.Grants.Written) != 0 || said() != 2 {
+			t.Errorf("a second pass wrote %v and said %d", res.Grants.Written, said())
+		}
+		// The tool: equal on mod-x, and nats is the part's alone (the controller's fixture holds none).
+		got, err := p.Grants("")
+		gm, _ := got.(map[string]any)
+		diff, _ := gm["differences"].([]grants.GrantDifference)
+		if err != nil || gm["equal"] != false || len(diff) != 1 || diff[0].Key != key("nats") || diff[0].Kind != "only the part's" {
+			t.Errorf("grants = %+v, %v", got, err)
+		}
+		// A user with a hash later: written once. Without it again: deleted. Unread hashes: nothing deleted.
+		hash("b.mod-y")
+		if res := p.Compose(); !slices.Equal(res.Grants.Written, []string{key("mod-y")}) {
+			t.Errorf("mod-y was granted %v when its account got a hash", res.Grants.Written)
+		}
+		b.refused[grants.ReadAccounts] = true
+		if res := p.Compose(); res.Grants == nil || len(res.Grants.Deleted)+len(res.Grants.Written) != 0 || len(res.Grants.Kept) != 1 || len(held()) != 3 {
+			t.Errorf("a pass that could not read the hashes changed grants: %+v, held %v", res.Grants, held())
+		}
+		delete(b.refused, grants.ReadAccounts)
+		delete(b.states[grants.ReadAccounts], "b.mod-y")
+		if res := p.Compose(); !slices.Equal(res.Grants.Deleted, []string{key("mod-y")}) || len(held()) != 2 {
+			t.Errorf("a module with no hashed account kept its grant: %+v, held %v", res.Grants, held())
+		}
+	})
+
 	// 774/AC4 (novox/hq ADR 0561 §2): every account the vault mints that the records compose is written to the state
 	// accounts once, with the bus's address and a module's claims, read by the vault alone; a key is deleted when placement
 	// no longer names its user, and never in a pass whose reads failed.

modules/mesh-grants/internal/grants/grants.go

@@ -0,0 +1,180 @@
+package grants
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"git.novox.be/novox/mesh-sdk/go/buscompose"
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// **The grants the part writes** (novox/hq ADR 0547 §4, §5, ADR 0561 §2, §6; issue 774). A module's grant is the record
+// that its bus credential exists for the fingerprint the catalogue's trunk names for it: `<module>.<fingerprint hex>` in
+// the part's state grants, naming the accounts (the module's own and each process's) the vault holds a hash for. It is the
+// controller's grant as the controller composes it (cmd/mesh-controller grants.go grantsFrom), so the two compare by key.
+// Until the controller's lease is handed away nobody reads the part's keys: the assigner reads the controller's
+// (delivery.GrantSource), and the tool grants says every difference, so the switch is made on a measured equality.
+//
+// A module of which an account has no hash yet is granted the accounts that have one; a module none of whose accounts has
+// one has no grant, and one it was written for is deleted (an older grant naming a node whose account is gone would let the
+// assigner want it, and a grant naming no account names every node). Nothing else is ever deleted, and nothing is written
+// or deleted in a pass that could not read the vault's hashes, the indexer's modules or a manifest of the module.
+
+// GrantsWritten is what one pass did to the state grants.
+type GrantsWritten struct {
+	Wanted  int      `json:"wanted"`
+	Written []string `json:"written,omitempty"`
+	Deleted []string `json:"deleted,omitempty"`
+	// Kept says why a pass wrote and deleted nothing, or left a module out.
+	Kept []string `json:"kept,omitempty"`
+}
+
+// GrantsOf is the grant of every module at the fingerprint the indexer's trunk names for it, by key, and the keys that
+// have none (drop): see the comment above. A module in skip (its manifest was refused) is neither. Pure.
+func GrantsOf(in Inputs, recs buscompose.Records, skip map[string]bool) (map[string]delivery.ControllerGrant, []string, error) {
+	principals, err := buscompose.Users(recs)
+	if err != nil {
+		return nil, nil, err
+	}
+	accounts, missing := map[string][]string{}, map[string]bool{}
+	for _, p := range principals {
+		if p.Module == "" || (p.Kind != buscompose.KindModule && p.Kind != buscompose.KindProcess) {
+			continue
+		}
+		if in.Hashes[p.Username()].Hash == "" {
+			missing[p.Module] = true
+			continue
+		}
+		accounts[p.Module] = append(accounts[p.Module], p.Username())
+	}
+	out := map[string]delivery.ControllerGrant{}
+	var drop []string
+	for module, fp := range in.Fingerprints {
+		if skip[module] || !strings.HasPrefix(fp, delivery.SourcePrefix2) {
+			continue
+		}
+		key, err := delivery.PartGrantKey(module, fp)
+		if err != nil {
+			continue
+		}
+		names := accounts[module]
+		if missing[module] && len(names) == 0 {
+			drop = append(drop, key)
+			continue
+		}
+		sort.Strings(names)
+		out[key] = delivery.ControllerGrant{Accounts: names}
+	}
+	sort.Strings(drop)
+	return out, drop, nil
+}
+
+// grantsReadFailed names a read of this pass without which no grant is written: the hashes and the indexer's modules and
+// manifests. A manifest refused for one module only leaves that module out (writeGrants).
+func grantsReadFailed(in Inputs) string {
+	for _, g := range in.Gaps {
+		if !g.Failed {
+			continue
+		}
+		for _, input := range []string{ReadAccounts, ReadModules, ReadManifests} {
+			if strings.HasPrefix(g.Input, input) {
+				return g.Input + ": " + g.Why
+			}
+		}
+	}
+	return ""
+}
+
+// writeGrants writes each grant whose key is absent or differs and says grant-written for each, and deletes the key of a
+// module that has none now.
+func (p *Part) writeGrants(res *Result, in Inputs, recs buscompose.Records) {
+	if why := grantsReadFailed(in); why != "" {
+		res.Grants = &GrantsWritten{Kept: []string{"no grant is written or deleted in a pass that could not read " + why}}
+		return
+	}
+	skip := map[string]bool{}
+	for _, g := range in.Refused {
+		skip[strings.TrimPrefix(g.Input, "manifest of ")] = true
+	}
+	want, drop, err := GrantsOf(in, recs, skip)
+	if err != nil {
+		res.Errors = append(res.Errors, "the grants could not be composed: "+err.Error())
+		return
+	}
+	done := &GrantsWritten{Wanted: len(want)}
+	res.Grants = done
+	for _, key := range sortedKeys(want) {
+		g := want[key]
+		data, err := delivery.EncodeControllerGrant(g)
+		if err != nil {
+			res.Errors = append(res.Errors, fmt.Sprintf("the grant %s cannot be encoded: %v", key, err))
+			continue
+		}
+		if raw, _, found, err := p.Store.Get(StateGrants, key); err != nil {
+			res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be read: %v", StateGrants, key, err))
+			continue
+		} else if found {
+			if was, err := delivery.DecodeControllerGrant(raw); err == nil && delivery.SameControllerGrant(was, withVersion(g)) {
+				continue
+			}
+		}
+		if _, err := p.Store.Put(StateGrants, key, data); err != nil {
+			res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be written: %v", StateGrants, key, err))
+			continue
+		}
+		done.Written = append(done.Written, key)
+		module, fp, _ := delivery.ParsePartGrantKey(key)
+		p.say(res, EventGrantWritten, map[string]any{"module": module, "fingerprint": fp, "accounts": g.Accounts})
+	}
+	for _, key := range drop {
+		if _, _, found, err := p.Store.Get(StateGrants, key); err != nil || !found {
+			continue
+		}
+		if err := p.Store.Delete(StateGrants, key); err != nil {
+			res.Errors = append(res.Errors, fmt.Sprintf("its own %s %s cannot be deleted: %v", StateGrants, key, err))
+			continue
+		}
+		done.Deleted = append(done.Deleted, key)
+	}
+}
+
+func withVersion(g delivery.ControllerGrant) delivery.ControllerGrant {
+	g.V = delivery.Version
+	return g
+}
+
+// GrantDifference is one key on which the part's grants and the controller's differ.
+type GrantDifference struct {
+	Key string `json:"key"`
+	// Kind is `only the controller's`, `only the part's` or `accounts differ`.
+	Kind       string   `json:"kind"`
+	Controller []string `json:"controller_accounts,omitempty"`
+	Part       []string `json:"part_accounts,omitempty"`
+}
+
+// CompareGrants lists every key the two sets differ on, in key order: a key held by one alone, or held by both with other
+// accounts. Pure.
+func CompareGrants(controller, part map[string]delivery.ControllerGrant) []GrantDifference {
+	keys := map[string]bool{}
+	for k := range controller {
+		keys[k] = true
+	}
+	for k := range part {
+		keys[k] = true
+	}
+	out := []GrantDifference{}
+	for _, k := range sortedKeys(keys) {
+		c, inC := controller[k]
+		g, inP := part[k]
+		switch {
+		case inC && !inP:
+			out = append(out, GrantDifference{Key: k, Kind: "only the controller's", Controller: c.Accounts})
+		case inP && !inC:
+			out = append(out, GrantDifference{Key: k, Kind: "only the part's", Part: g.Accounts})
+		case !delivery.SameControllerGrant(withVersion(c), withVersion(g)):
+			out = append(out, GrantDifference{Key: k, Kind: "accounts differ", Controller: c.Accounts, Part: g.Accounts})
+		}
+	}
+	return out
+}

modules/mesh-grants/internal/grants/inputs.go

@@ -115,6 +115,9 @@ type Inputs struct {
 	Manifests map[string]catalogue.Manifest
 	// Digests is each module's manifest digest, by module.
 	Digests map[string]string
+	// Fingerprints is the fingerprint of the new form the indexer's trunk value names for each module, by module: what a
+	// grant is written for (grants.go).
+	Fingerprints map[string]string
 	// Running is the manifest a machine is wanted to run, by `<node>.<module>`: the one the builder built at the
 	// fingerprint the assigner's want names. An assignment with none is composed from the trunk's manifest.
 	Running map[string]catalogue.Manifest
@@ -237,7 +240,7 @@ func each(r Reader, state string, gaps *[]Gap) map[string][]byte {
 
 // ReadInputs reads what the shadow composes from. It never fails: what it cannot read is a gap.
 func ReadInputs(r Reader, cache *Manifests) Inputs {
-	in := Inputs{Assigned: map[string][]string{}, Manifests: map[string]catalogue.Manifest{}, Digests: map[string]string{},
+	in := Inputs{Assigned: map[string][]string{}, Manifests: map[string]catalogue.Manifest{}, Digests: map[string]string{}, Fingerprints: map[string]string{},
 		Running: map[string]catalogue.Manifest{},
 		Values:  map[string]any{}, Gaps: append([]Gap(nil), standingGaps...)}
 	nodes := map[string]bool{}
@@ -338,6 +341,9 @@ func ReadInputs(r Reader, cache *Manifests) Inputs {
 		if err == nil && v.Manifest == "" {
 			err = fmt.Errorf("the indexer's value names no manifest digest")
 		}
+		if err == nil && v.FP2 != "" {
+			in.Fingerprints[module] = v.FP2
+		}
 		var m catalogue.Manifest
 		if err == nil {
 			m, err = cache.get(r, v.Manifest)

modules/mesh-grants/internal/grants/part.go

@@ -23,32 +23,36 @@ const (
 	StateMemberships = "memberships"
 	StateAssertions  = "assertions"
 	StateTakeover    = "takeover"
-	KeyList          = "list"
-	KeySwitch        = "switch"
+	// StateGrants is the part's grants by module and fingerprint, the key layout of the controller's (grants.go).
+	StateGrants = delivery.PartGrantsState
+	KeyList     = "list"
+	KeySwitch   = "switch"
 	// KeyPass is the last pass's result, and KeyAsked a tool's ask to compose again now, both in StateUserList.
 	KeyPass  = "pass"
 	KeyAsked = "asked"
 )
 
 // States is every state the part keeps.
-var States = []string{StateAccounts, StateAssertions, StateMemberships, StateTakeover, StateUserList}
+var States = []string{StateAccounts, StateAssertions, StateGrants, StateMemberships, StateTakeover, StateUserList}
 
 // The events the part says.
 const (
 	EventUserList   = "user-list-changed"
 	EventMembership = "membership-changed"
+	// EventGrantWritten is said when the part wrote a grant of its state grants (ADR 0547 §5, ADR 0561 §2).
+	EventGrantWritten = "grant-written"
 	// EventAsserted is said when the part created a bucket, log or stream a manifest declares (issue 1790).
 	EventAsserted = "asserted"
 )
 
 // Emits is every event the part says.
-var Emits = []string{EventAsserted, EventHashesAccepted, EventMembership, EventUserList}
+var Emits = []string{EventAsserted, EventGrantWritten, EventHashesAccepted, EventMembership, EventUserList}
 
 // Store is the part's own state, by local name: the only thing the part writes.
 type Store interface {
 	Get(state, key string) ([]byte, uint64, bool, error)
 	Put(state, key string, value []byte) (uint64, error)
-	// Keys and Delete are for the wanted accounts alone (accounts.go): every other state is only put.
+	// Keys and Delete are for the wanted accounts (accounts.go) and the grants (grants.go): every other state is only put.
 	Keys(state string) ([]string, error)
 	Delete(state, key string) error
 }
@@ -118,9 +122,11 @@ type Result struct {
 	// Consumers is what the pass found and created of the durable consumers an account needs (ADR 0561 §6).
 	Consumers *ConsumersState `json:"consumers,omitempty"`
 	// Accounts is what the pass wrote of the wanted accounts the vault mints from (ADR 0561 §2).
-	Accounts            *AccountsWritten `json:"accounts,omitempty"`
-	Errors              []string         `json:"errors,omitempty"`
-	PublishedOrAsserted string           `json:"published_or_asserted"`
+	Accounts *AccountsWritten `json:"accounts,omitempty"`
+	// Grants is what the pass wrote of its grants (grants.go).
+	Grants              *GrantsWritten `json:"grants,omitempty"`
+	Errors              []string       `json:"errors,omitempty"`
+	PublishedOrAsserted string         `json:"published_or_asserted"`
 }
 
 // NothingTakesEffect is what every composition says it did on the bus beyond its own state and the memberships.
@@ -265,6 +271,10 @@ func (p *Part) Compose() Result {
 	if p.Store != nil {
 		p.writeAccounts(&res, in, recs)
 	}
+	// The grants the assigner reads once the lease is handed away, in shadow until then (ADR 0547 §4, §5).
+	if p.Store != nil {
+		p.writeGrants(&res, in, recs)
+	}
 	// The list itself, sealed to the bus node and named in `user-list.sealed` (ADR 0547 clause 7).
 	if list, n, leftOut, err := ComposeList(recs, in.Hashes); err != nil {
 		res.Errors = append(res.Errors, "the user list could not be composed whole: "+err.Error())

modules/mesh-grants/internal/grants/tools.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"sort"
 	"strings"
 
 	"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -30,38 +29,83 @@ const (
 var Tools = []string{ToolGrants, ToolMembership, ToolUserList, ToolComposeNow, ToolAssert, ToolTakeOver, ToolHandBack, ToolTakeOverState,
 	ToolAccounts, ToolConsumerMatch, ToolAcceptHashes}
 
-// WritesNoGrant is what grants says of the part's own grants in this milestone.
-const WritesNoGrant = "none: the grants part writes no grants.<module>.<fp> before ADR 0547's third milestone (clause 5); the controller's are beside"
+// WritesGrants is what grants says of the part's own grants in this milestone: written, and read by nobody until the
+// controller's lease is handed away (delivery.GrantSource).
+const WritesGrants = "in shadow: the grants part writes grants.<module>.<fp> into its state grants, which the assigner reads only once the controller's lease is handed away (ADR 0547 clause 5); the controller's are beside, with every difference"
 
-// Grants answers the tool grants: what the part writes of grants (none), and the controller's, one module's or all.
+// Grants answers the tool grants: the grants the part wrote, the controller's, and every difference between them, one
+// module's or all.
 func (p *Part) Grants(module string) (any, error) {
-	keys, err := p.Read.Keys(ReadControllerGrant)
-	if err != nil {
-		return map[string]any{"writes": WritesNoGrant, "controller_unread": err.Error()}, nil
-	}
-	type grant struct {
-		Module      string   `json:"module"`
-		Fingerprint string   `json:"fingerprint"`
-		Accounts    []string `json:"accounts,omitempty"`
-	}
-	out := []grant{}
-	for _, k := range keys {
-		m, fp, err := delivery.ParseControllerGrantKey(k)
-		if err != nil || (module != "" && m != module) {
-			continue
+	out := map[string]any{"writes": WritesGrants}
+	read := func(get func() ([]string, func(string) ([]byte, bool, error), error)) (map[string]delivery.ControllerGrant, error) {
+		keys, one, err := get()
+		if err != nil {
+			return nil, err
 		}
-		g := grant{Module: m, Fingerprint: fp}
-		if module != "" {
-			if raw, found, err := p.Read.Get(ReadControllerGrant, k); err == nil && found {
-				if v, err := delivery.DecodeControllerGrant(raw); err == nil {
-					g.Accounts = v.Accounts
-				}
+		m := map[string]delivery.ControllerGrant{}
+		for _, k := range keys {
+			mod, _, err := delivery.ParseControllerGrantKey(k)
+			if err != nil || (module != "" && mod != module) {
+				continue
 			}
+			raw, found, err := one(k)
+			if err != nil {
+				return nil, err
+			}
+			if !found {
+				continue
+			}
+			g, err := delivery.DecodeControllerGrant(raw)
+			if err != nil {
+				return nil, fmt.Errorf("%s: %w", k, err)
+			}
+			m[k] = g
 		}
-		out = append(out, g)
+		return m, nil
 	}
-	sort.Slice(out, func(i, j int) bool { return out[i].Module+out[i].Fingerprint < out[j].Module+out[j].Fingerprint })
-	return map[string]any{"writes": WritesNoGrant, "controller": out, "count": len(out)}, nil
+	controller, cerr := read(func() ([]string, func(string) ([]byte, bool, error), error) {
+		keys, err := p.Read.Keys(ReadControllerGrant)
+		return keys, func(k string) ([]byte, bool, error) { return p.Read.Get(ReadControllerGrant, k) }, err
+	})
+	mine, perr := read(func() ([]string, func(string) ([]byte, bool, error), error) {
+		keys, err := p.Store.Keys(StateGrants)
+		return keys, func(k string) ([]byte, bool, error) {
+			raw, _, found, err := p.Store.Get(StateGrants, k)
+			return raw, found, err
+		}, err
+	})
+	type grant struct {
+		Key      string   `json:"key"`
+		Module   string   `json:"module"`
+		Accounts []string `json:"accounts,omitempty"`
+	}
+	list := func(m map[string]delivery.ControllerGrant) []grant {
+		out := []grant{}
+		for _, k := range sortedKeys(m) {
+			mod, _, _ := delivery.ParseControllerGrantKey(k)
+			out = append(out, grant{Key: k, Module: mod, Accounts: m[k].Accounts})
+		}
+		return out
+	}
+	if cerr != nil {
+		out["controller_unread"] = cerr.Error()
+	} else {
+		out["controller"], out["controller_count"] = list(controller), len(controller)
+	}
+	if perr != nil {
+		out["written_unread"] = perr.Error()
+	} else {
+		out["written"], out["count"] = list(mine), len(mine)
+	}
+	if cerr == nil && perr == nil {
+		diff := CompareGrants(controller, mine)
+		out["differences"], out["differing"] = diff, len(diff)
+		out["equal"] = len(diff) == 0
+	}
+	if last := p.Last(); last.Grants != nil {
+		out["last_pass"] = last.Grants
+	}
+	return out, nil
 }
 
 // MembershipOf answers the tool membership: one module on one node as composed now, and as last published by the

modules/mesh-grants/module.json

@@ -31,6 +31,7 @@
   ],
   "emits": [
     "asserted",
+    "grant-written",
     "hashes-accepted",
     "membership-changed",
     "user-list-changed"
@@ -57,6 +58,10 @@
       "name": "assertions",
       "history": 5
     },
+    {
+      "name": "grants",
+      "history": 5
+    },
     {
       "name": "memberships",
       "history": 5