Forget the accounts of modules no longer on their nodes at the start's look, and grant sealed accounts alone (issue 1945, ADR 0543, review)

Migration 0108 marks every standing account sealed, ace's stale mesh-build-go
rows from 17:19 among them, and the forgetting ran only at a later unassign:
the next assignment to ace would have issued nothing again.

- The start's look first forgets, from each node's own resolution (assigned
  and required), the accounts of every module that no longer resolves on its
  node, and says each; only from a reading that holds assignments, and a node
  whose resolution is unknown is left alone.
- Unassign forgets a module's accounts only once it no longer resolves on the
  node (a module another requires keeps running), and never the node's own
  provisioner account; the forgetting moved from the inventory to the act.
- Forgetting holds each account's lock across its delete, so an issue in flight
  never writes a row back after it.
- A module whose accounts are not all sealed is granted its sealed accounts
  alone, and its grant is dropped when none is: an older grant naming ace no
  longer lets the assigner want it.
- An issue's locked section has a deadline of two minutes, since it holds a
  connection of the store's pool across the vault and the bus.
Tests: cases in the existing tables.
Research 072: the grants piece (memberships and credentials heard from placement).

cmd/mesh-controller/assignment_accounts_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -194,6 +195,7 @@ func TestSettlingAndTheStartsLook(t *testing.T) {
 		wantComposed       []string
 		wantGrants         bool
 		wantErr            string
+		stale              []string // what forgetting the accounts of modules no longer on their nodes says
 	}{
 		{name: "an account not minted", module: "mesh-build-node", issued: true, holder: "novox", wantIssues: 1,
 			wantComposed: []string{"novox"}, wantGrants: true},
@@ -214,13 +216,21 @@ func TestSettlingAndTheStartsLook(t *testing.T) {
 			look: []chain.Assignment{ace("mesh-build-node"), ace("mesh-build-go")}, holder: "novox", wantIssues: 2,
 			wantComposed: []string{"novox"}, wantGrants: true},
 		{name: "the start's look at nothing does nothing", look: []chain.Assignment{ace(controllerModule)}, holder: "novox"},
+		{name: "the start's look forgets the accounts of modules no longer on their nodes first",
+			look: []chain.Assignment{ace("mesh-build-node")}, holder: "novox", wantIssues: 1, wantComposed: []string{"novox"},
+			wantGrants: true, stale: []string{"mesh-build-go no longer runs on ace: its bus accounts forgotten: ace.mesh-build-go"}},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
 			var composed []string
 			issues, catalogues, grants := 0, 0, 0
+			forgets := 0
 			steps := assignmentAccountSteps{
 				holds: func(context.Context) bool { return !c.notHolding },
+				forgetStale: func(context.Context) ([]string, error) {
+					forgets++
+					return c.stale, nil
+				},
 				catalogue: func(context.Context) (map[string]catalogue.Manifest, error) {
 					catalogues++
 					return map[string]catalogue.Manifest{}, nil
@@ -262,6 +272,14 @@ func TestSettlingAndTheStartsLook(t *testing.T) {
 			if want := map[bool]int{true: 1}[c.wantGrants]; grants != want {
 				t.Fatalf("grants written %d time(s), want %d", grants, want)
 			}
+			if want := map[bool]int{true: 1}[len(c.look) > 0 && !c.notHolding]; forgets != want {
+				t.Fatalf("forgot stale accounts %d time(s), want %d (only from a reading that holds assignments)", forgets, want)
+			}
+			for _, line := range c.stale {
+				if !strings.Contains(said, line) {
+					t.Fatalf("said %q, want it to say %q", said, line)
+				}
+			}
 			if c.wantGrants && c.wantErr == "" && !strings.Contains(said, "the grants written") {
 				t.Fatalf("said %q, want it to name the grants", said)
 			}
@@ -367,6 +385,26 @@ func TestTheAccountsAModuleSpeaksWithOnANode(t *testing.T) {
 		})
 	}
 
+	// The accounts of a module that no longer resolves on its node are stale, its provisioner client's among them; a
+	// module still resolved (assigned or required), an account of another kind, and a node whose resolution is unknown
+	// are left alone (issue 1945: ace's mesh-build-go rows from 17:19, marked sealed by migration 0108).
+	t.Run("stale accounts", func(t *testing.T) {
+		u := func(kind, node, module string) inventory.BusUser {
+			return inventory.BusUser{Username: node + "." + module + "." + kind, Kind: kind, Node: node, Module: module}
+		}
+		users := map[string]inventory.BusUser{}
+		for _, b := range []inventory.BusUser{u(inventory.BusModule, "ace", "mesh-build-go"),
+			u(inventory.BusProcess, "ace", "mesh-build-go"), u(inventory.BusProvisioner, "ace", "mesh-provisioner"),
+			u(inventory.BusModule, "ace", "mesh-build-node"), u(inventory.BusNode, "ace", ""),
+			u(inventory.BusModule, "g14", "mesh-build-go")} {
+			users[b.Username] = b
+		}
+		got := staleAccounts(users, map[string]map[string]bool{"ace": {"mesh-build-node": true}})
+		if want := map[string][]string{"ace": {"mesh-build-go", "mesh-provisioner"}}; !reflect.DeepEqual(got, want) {
+			t.Fatalf("stale %v, want %v", got, want)
+		}
+	})
+
 	t.Run("a process may be issued only with its own broker secret", func(t *testing.T) {
 		// mesh-mail's real shape: no `broker` of its own, its process declaring bus (review round 1 of issue 1181).
 		if err := mayIssue(mailSenderWithProcess()); err != nil {