Forget the accounts of modules no longer on their nodes at the start's look, and grant sealed accounts alone (issue 1945, ADR 0543, review)

Migration 0108 marks every standing account sealed, ace's stale mesh-build-go
rows from 17:19 among them, and the forgetting ran only at a later unassign:
the next assignment to ace would have issued nothing again.

- The start's look first forgets, from each node's own resolution (assigned
  and required), the accounts of every module that no longer resolves on its
  node, and says each; only from a reading that holds assignments, and a node
  whose resolution is unknown is left alone.
- Unassign forgets a module's accounts only once it no longer resolves on the
  node (a module another requires keeps running), and never the node's own
  provisioner account; the forgetting moved from the inventory to the act.
- Forgetting holds each account's lock across its delete, so an issue in flight
  never writes a row back after it.
- A module whose accounts are not all sealed is granted its sealed accounts
  alone, and its grant is dropped when none is: an older grant naming ace no
  longer lets the assigner want it.
- An issue's locked section has a deadline of two minutes, since it holds a
  connection of the store's pool across the vault and the bus.
Tests: cases in the existing tables.
Research 072: the grants piece (memberships and credentials heard from placement).

cmd/mesh-controller/assignment_accounts.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"maps"
 	"slices"
 	"strings"
 	"sync"
@@ -40,6 +41,9 @@ import (
 type assignmentAccountSteps struct {
 	// holds says this controller holds the lease: only its holder issues.
 	holds func(ctx context.Context) bool
+	// forgetStale forgets every module's accounts on a node where the module no longer resolves, and answers a line for
+	// each (issue 1945: rows of an unassignment before this release, marked sealed by migration 0108).
+	forgetStale func(ctx context.Context) ([]string, error)
 	// catalogue reads the catalogue once, for every assignment a look issues for.
 	catalogue func(ctx context.Context) (map[string]catalogue.Manifest, error)
 	// issue issues the module's accounts on the node that are not minted, from the catalogue read; issued says it
@@ -86,6 +90,17 @@ func (s assignmentAccountSteps) settle(ctx context.Context, node, module string)
 // it issued (an assignment that came to stand while no controller heard it may have its accounts and still lack its
 // place in the bus's user list or its grant). It answers what it did.
 func (s assignmentAccountSteps) look(ctx context.Context, standing []chain.Assignment) (string, error) {
+	var forgot []string
+	if len(standing) > 0 && s.forgetStale != nil {
+		// Only from a reading of the assignments that holds some: an empty or failed one would forget every account.
+		if !s.holds(ctx) {
+			return "", errNotHolding
+		}
+		var err error
+		if forgot, err = s.forgetStale(ctx); err != nil {
+			return "", fmt.Errorf("the accounts of modules no longer on their nodes are not forgotten: %w", err)
+		}
+	}
 	var looked []chain.Assignment
 	for _, a := range standing {
 		if a.Module != controllerModule {
@@ -93,7 +108,7 @@ func (s assignmentAccountSteps) look(ctx context.Context, standing []chain.Assig
 		}
 	}
 	if len(looked) == 0 {
-		return "", nil
+		return strings.Join(forgot, "\n"), nil
 	}
 	if !s.holds(ctx) {
 		return "", errNotHolding
@@ -114,7 +129,7 @@ func (s assignmentAccountSteps) look(ctx context.Context, standing []chain.Assig
 		}
 	}
 	said, err := s.cover(ctx)
-	lines = append(lines, fmt.Sprintf("%d standing assignment(s) looked at; %s", len(looked), said))
+	lines = append(append(forgot, lines...), fmt.Sprintf("%d standing assignment(s) looked at; %s", len(looked), said))
 	return strings.Join(lines, "\n"), err
 }
 
@@ -247,7 +262,8 @@ func liveAssignmentAccountSteps(open *stores) assignmentAccountSteps {
 			_, err := theLease.epoch(ctx)
 			return err == nil
 		},
-		catalogue: func(ctx context.Context) (map[string]catalogue.Manifest, error) { return inv.Catalogue(ctx) },
+		forgetStale: func(ctx context.Context) ([]string, error) { return forgetStaleAccounts(ctx, open) },
+		catalogue:   func(ctx context.Context) (map[string]catalogue.Manifest, error) { return inv.Catalogue(ctx) },
 		issue: func(ctx context.Context, shelf map[string]catalogue.Manifest, node, module string) (bool, string, error) {
 			return issueAssignmentAccounts(ctx, inv, shelf, node, module)
 		},
@@ -403,3 +419,71 @@ func watchAssignmentsOnce(ctx context.Context, watch func(context.Context) (<-ch
 		}
 	}
 }
+
+// staleAccounts is, by node, each module some account of which stands on a node where the module no longer resolves:
+// the accounts of a module's own, its processes' and its provisioner client's. A node whose resolution is unknown is
+// left alone.
+func staleAccounts(users map[string]inventory.BusUser, resolved map[string]map[string]bool) map[string][]string {
+	seen := map[string]map[string]bool{}
+	for _, u := range users {
+		switch u.Kind {
+		case inventory.BusModule, inventory.BusNodeTools, inventory.BusProcess, inventory.BusProvisioner:
+		default:
+			continue
+		}
+		on, known := resolved[u.Node]
+		if u.Node == "" || u.Module == "" || !known || on[u.Module] {
+			continue
+		}
+		if seen[u.Node] == nil {
+			seen[u.Node] = map[string]bool{}
+		}
+		seen[u.Node][u.Module] = true
+	}
+	out := map[string][]string{}
+	for node, modules := range seen {
+		out[node] = slices.Sorted(maps.Keys(modules))
+	}
+	return out
+}
+
+// forgetStaleAccounts forgets the accounts of every module on a node where it no longer resolves (assigned or
+// required), from each node's own resolution; a node that does not resolve is left alone.
+func forgetStaleAccounts(ctx context.Context, open *stores) ([]string, error) {
+	inv := open.inventory
+	users, err := inv.BusUsers(ctx)
+	if err != nil {
+		return nil, err
+	}
+	ctx = withInputCache(ctx) // one reading of the mesh for every node resolved
+	resolved := map[string]map[string]bool{}
+	for _, u := range users {
+		if _, done := resolved[u.Node]; done || u.Node == "" {
+			continue
+		}
+		plan, _, err := resolveNode(ctx, open, u.Node)
+		if err != nil {
+			continue // unknown: nothing of this node is forgotten
+		}
+		on := map[string]bool{}
+		for _, m := range plan.Modules {
+			on[m.Module] = true
+		}
+		resolved[u.Node] = on
+	}
+	var lines []string
+	stale := staleAccounts(users, resolved)
+	for _, node := range slices.Sorted(maps.Keys(stale)) {
+		for _, module := range stale[node] {
+			forgot, err := inv.ForgetBusAccountsOf(ctx, node, module, true)
+			if err != nil {
+				return lines, err
+			}
+			if len(forgot) > 0 {
+				lines = append(lines, fmt.Sprintf("%s no longer runs on %s: its bus accounts forgotten: %s", module, node,
+					strings.Join(forgot, ", ")))
+			}
+		}
+	}
+	return lines, nil
+}