Hold a joining build for a person without superseding its delivery, join only on an engine that keeps own groups, and keep a held module's settings (hq ADR 0328 §8, ADR 0334; re-review of 96e15f86)

A delivery does not send a joining build held for its library or its engine: the module waits for a person,
the plan stays open saying so and judges its other modules, and sends it once nothing holds it. A held move is
no move for the gate. A module joins only on an engine at contract 7, and a held module is composed with the
settings it last ran with. The runner's verdicts are said for their modules and judged by whether a move
changes them. mesh-host pinned at #97 (7a41e120), its bundle now examples/foundation-first-node.lock.

cmd/mesh-controller/held_library.go

@@ -2,25 +2,43 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"sort"
 
 	"github.com/novox/mesh-controller/internal/catalogue"
 	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
 )
 
-// A module that joins keeps the user it ran as until every library it uses is taken (novox/hq ADR 0328 §8).
+// A module that joins keeps the user it ran as until it may join (novox/hq ADR 0328 §8, ADR 0334).
 //
-// **Held at the build it ran before, composed exactly as then.** A module whose registered build uses a library not yet
-// taken, on a machine that was last sent another build of it, is composed at that build — the manifest the build
-// records hold, in place of the registered one, as a recorded module is kept (recorded_kept.go): its resources, its
-// ports in the filter, its contributions and bindings are what they were, and nothing of the new contract is sent for
-// it. The send records that the machine still carries that build. Once the library is taken, the next composition
-// takes the new build. A module the machine never ran has nothing to keep, and is composed as registered; a build the
-// records no longer hold cannot be kept, and the module is composed as registered (it then waits as joining modules do).
+// **Held at the build it ran before, composed exactly as then.** A module whose registered build joins — declares
+// libraries or uses its identity — on a machine that was last sent another build of it, is held there while
 //
-// shelf, when given, has each held module's manifest replaced. It answers what it holds: module → the commit kept.
+//   - the machine's engine is older than link.OwnGroupContract: it cannot put the tool runner in the module's own group
+//     nor keep its own directories' group, so the module's tools would lose their read (the re-review of 96e15f86, 2);
+//   - or a library it uses is not yet taken (§8).
+//
+// Held, it is composed at that build — the manifest the build records hold, in place of the registered one, as a
+// recorded module is kept (recorded_kept.go) — and with the settings it last ran with (SettingsAsOf the machine's last
+// send): its resources, its ports in the filter, its contributions and bindings are what they were, and nothing of the
+// new contract is sent for it. The send records that the machine still carries that build. A delivery that brings the
+// joining build does not send it while it is held: the module waits for a person, the plan says so and judges its
+// other modules, and sends it once nothing holds it (release_plan.go, heldIn). A module the machine never ran has
+// nothing to keep, and is composed as registered; a build the records no longer hold cannot be kept.
+
+// heldModule is one module held on a machine: the build it keeps and why, in words with no path.
+type heldModule struct {
+	commit string
+	// library is the first library it uses that is not taken, empty when none.
+	library string
+	why     string
+}
+
+// holdJoining is every module held on a machine, by module; shelf, when given, has each one's manifest replaced by
+// the build it keeps.
 func holdJoining(ctx context.Context, inv *inventory.Inventory, node string,
-	shelf map[string]catalogue.Manifest) (map[string]string, error) {
+	shelf map[string]catalogue.Manifest) (map[string]heldModule, error) {
 	registered := shelf
 	if registered == nil {
 		var err error
@@ -34,7 +52,7 @@ func holdJoining(ctx context.Context, inv *inventory.Inventory, node string,
 	}
 	var joining []string
 	for m := range sent {
-		if man, has := registered[m]; has && len(man.Libraries) > 0 {
+		if man, has := registered[m]; has && catalogue.Joins(man) {
 			joining = append(joining, m)
 		}
 	}
@@ -42,6 +60,11 @@ func holdJoining(ctx context.Context, inv *inventory.Inventory, node string,
 		return nil, nil
 	}
 	sort.Strings(joining)
+	stated, had, err := inv.HealthOf(ctx, node)
+	if err != nil {
+		return nil, err
+	}
+	engineJoins := had && stated.Contract >= link.OwnGroupContract
 	libs, err := inv.Libraries(ctx)
 	if err != nil {
 		return nil, err
@@ -54,40 +77,86 @@ func holdJoining(ctx context.Context, inv *inventory.Inventory, node string,
 	if err != nil {
 		return nil, err
 	}
-	out := map[string]string{}
+	out := map[string]heldModule{}
 	for _, m := range joining {
 		was := sent[m]
 		if was == "" || sameCommit(was, current[m].Commit) {
 			continue // the machine already runs the registered build: nothing before it to keep
 		}
-		untaken := ""
+		h := heldModule{commit: was}
 		for _, u := range registered[m].Libraries {
 			if !taken[u.Library] {
-				untaken = u.Library
+				h.library = u.Library
 				break
 			}
 		}
-		if untaken == "" {
+		switch {
+		case !engineJoins:
+			h.why = fmt.Sprintf("%s keeps the build it runs on %s until that machine's node-engine keeps a module's own "+
+				"group (novox/hq ADR 0334)", m, node)
+		case h.library != "":
+			h.why = fmt.Sprintf("%s waits for the operator to let the mesh keep the library %s (novox/hq ADR 0328 §8)",
+				m, h.library)
+		default:
 			continue
 		}
 		before, found, err := inv.ManifestAt(ctx, m, was)
 		if err != nil {
 			return nil, err
 		}
-		if !found || usesLibrary(before, untaken) {
+		if !found || (engineJoins && usesLibrary(before, h.library)) {
 			continue
 		}
 		if shelf != nil {
 			shelf[m] = before
 		}
-		out[m] = was
+		out[m] = h
 	}
 	return out, nil
 }
 
+// heldIn is why a module is held on any of the machines given, or "": what a delivery's first send waits on.
+func heldIn(ctx context.Context, inv *inventory.Inventory, module string, machines []string) (string, error) {
+	for _, node := range machines {
+		held, err := holdJoining(ctx, inv, node, nil)
+		if err != nil {
+			return "", err
+		}
+		if h, is := held[module]; is {
+			return h.why, nil
+		}
+	}
+	return "", nil
+}
+
+// heldSettings puts, for each module held on a machine, the settings it last ran with in place of today's: the layers
+// as they stood at the machine's last send. Today's stand where no send is recorded.
+func heldSettings(ctx context.Context, inv *inventory.Inventory, node string, held map[string]heldModule,
+	settings catalogue.SettingsBy) error {
+	if len(held) == 0 {
+		return nil
+	}
+	last, found, err := inv.LastSend(ctx, node)
+	if err != nil || !found {
+		return err
+	}
+	for m := range held {
+		layers, err := inv.SettingsAsOf(ctx, node, m, last.SentAt)
+		if err != nil {
+			return err
+		}
+		if len(layers) == 0 {
+			delete(settings, m)
+			continue
+		}
+		settings[m] = layers
+	}
+	return nil
+}
+
 func usesLibrary(m catalogue.Manifest, library string) bool {
 	for _, u := range m.Libraries {
-		if u.Library == library {
+		if library != "" && u.Library == library {
 			return true
 		}
 	}

cmd/mesh-controller/held_plan_test.go

@@ -0,0 +1,329 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/lease"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// The re-review of 96e15f86, item 1 (novox/hq ADR 0328 §8): a delivery that brings a joining build of a module whose
+// library is not taken does not send it — the module waits for a person, the plan stays open saying so, and the other
+// modules of the plan are sent and judged as usual. Once the library is taken, the waiting build is sent and judged,
+// with no new merge.
+func TestADeliveryWaitsForALibraryAndJudgesTheRest(t *testing.T) {
+	judgeLeavesAsTheCore(t)
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	keeper, _ := withConditionsInMemory(t)
+	was := doctorFrom
+	doctorFrom = &doctor{open: open, keeper: keeper, teller: &conditions.Told{}}
+	t.Cleanup(func() { doctorFrom = was })
+
+	register(t, open, catalogue.Manifest{Module: catalogue.LibrariesModule, Version: "1"})
+	if err := inv.SetSettings(ctx, "", catalogue.LibrariesModule, map[string]any{catalogue.LibrariesSetting: map[string]any{
+		"media": map[string]any{"home": "anchor", "root": "/storage/media"}}}); err != nil {
+		t.Fatal(err)
+	}
+	app1 := catalogue.Manifest{Module: "app", Version: "c1"}
+	app2 := catalogue.Manifest{Module: "app", Version: "c2"}
+	builds := []struct {
+		id, module, commit string
+		m                  catalogue.Manifest
+		at                 time.Time
+	}{
+		{"build-app-1", "app", "c1", app1, time.Now().Add(-2 * time.Hour)},
+		{"build-radarr-1", "radarr", "r1", radarrBefore(), time.Now().Add(-2 * time.Hour)},
+		{"build-app-2", "app", "c2", app2, time.Now().Add(-time.Minute)},
+		{"build-radarr-2", "radarr", "r2", radarrJoined(), time.Now().Add(-time.Minute)},
+	}
+	for _, b := range builds {
+		raw, _ := json.Marshal(b.m)
+		if err := inv.RecordBuild(ctx, inventory.Build{ID: b.id, Module: b.module, Commit: b.commit,
+			Repository: "novox/mesh-media-catalog", Path: "modules/" + b.module, Ref: "main", Manifest: raw,
+			Asked: b.at, At: b.at}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	registerAt := func(m catalogue.Manifest, commit string, asked time.Time) {
+		if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-media-catalog", Seat: "git",
+			Path: "modules/" + m.Module, Ref: "main", BuiltFrom: commit, Head: commit, Asked: asked}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	registerAt(app1, "c1", time.Now().Add(-2*time.Hour))
+	registerAt(radarrBefore(), "r1", time.Now().Add(-2*time.Hour))
+	if _, err := inv.Assign(ctx, "anchor", "app"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.Assign(ctx, "anchor", "radarr"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-0", map[string]string{"app": "c1", "radarr": "r1"}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.OwnGroupContract,
+		SaidAt: time.Now(), HeardAt: time.Now()}); err != nil {
+		t.Fatal(err)
+	}
+	registerAt(app2, "c2", time.Now().Add(-time.Minute))
+	registerAt(radarrJoined(), "r2", time.Now().Add(-time.Minute))
+
+	// Every send records what it carries: the registered builds, a held module at the build it runs.
+	var sent []string
+	wasSend := sendRollout
+	sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
+		current, err := open.inventory.CurrentBuilds(ctx)
+		if err != nil {
+			return nil, err
+		}
+		for _, node := range names {
+			held, err := holdJoining(ctx, open.inventory, node, nil)
+			if err != nil {
+				return nil, err
+			}
+			carried := map[string]string{"app": current["app"].Commit, "radarr": current["radarr"].Commit}
+			for m, h := range held {
+				carried[m] = h.commit
+			}
+			sent = append(sent, node+":app="+carried["app"]+",radarr="+carried["radarr"])
+			digest := fmt.Sprintf("d-%s-%d", node, len(sent))
+			if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
+				return nil, err
+			}
+			if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
+				Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
+				return nil, err
+			}
+		}
+		return names, nil
+	}
+	t.Cleanup(func() { sendRollout = wasSend })
+	wasGather := gatherGateFacts
+	gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
+		f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
+			rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
+		reports, err := open.inventory.LastReports(ctx)
+		for _, r := range reports {
+			f.reports[r.Node] = r
+		}
+		return f, err
+	}
+	t.Cleanup(func() { gatherGateFacts = wasGather })
+	wasSettle, wasEvery := gateSettle, gateEvery
+	gateSettle, gateEvery = 0, 0
+	t.Cleanup(func() { gateSettle, gateEvery = wasSettle, wasEvery })
+
+	built := time.Now().UTC()
+	plan := inventory.Plan{ID: "plan-held", Repository: "novox/mesh-media-catalog", Branch: "main", Commit: "c2",
+		Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app", "radarr"}},
+		Modules: map[string]*inventory.PlanModule{
+			"app":    {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-app-2"},
+			"radarr": {State: "built", BuiltAt: &built, Commit: "r2", Build: "build-radarr-2"}}}
+	if err := inv.SavePlan(ctx, &plan); err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 4; i++ {
+		advancePlans(ctx, open)
+	}
+	p, err := inv.PlanByID(ctx, "plan-held")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if p.State == inventory.PlanSuperseded || p.State == inventory.PlanDone || p.State == inventory.PlanFailed {
+		t.Fatalf("the plan is %s (%s); want it open, waiting for the library", p.State, p.Note)
+	}
+	if g := p.Modules["app"].Gate; g == nil || g.Verdict != inventory.GatePassed {
+		t.Fatalf("app was not judged and passed beside the held module: %+v (%s)", p.Modules["app"], p.Note)
+	}
+	if p.Modules["radarr"].FirstAt != nil || !strings.Contains(p.Note, "library media") {
+		t.Fatalf("radarr was sent, or the plan does not say it waits for the library: %+v, %q", p.Modules["radarr"], p.Note)
+	}
+	if slices.ContainsFunc(sent, func(s string) bool { return strings.Contains(s, "radarr=r2") }) {
+		t.Fatalf("radarr's joining build was sent while held: %v", sent)
+	}
+
+	// The library is taken: the waiting build is sent, with no new merge.
+	if err := inv.RecordTakeStarted(ctx, inventory.TakenLibrary{Library: "media", Home: "anchor", Root: "/storage/media",
+		By: "the operator", Ask: "c01", Warrant: "c01"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.TakeDone(ctx, "media", "c01"); err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 4; i++ {
+		advancePlans(ctx, open)
+	}
+	if !slices.ContainsFunc(sent, func(s string) bool { return strings.Contains(s, "radarr=r2") }) {
+		t.Fatalf("radarr's joining build was not sent once its library was taken: %v", sent)
+	}
+}
+
+// The re-review of 96e15f86, item 2: a module joins only on an engine stating link.OwnGroupContract (7). On an engine
+// at 6 a module that ran there before is held at the build it ran, whatever its library — so its tools never lose
+// their read of its directories.
+func TestAJoiningModuleIsHeldOnAnEngineOlderThanOwnGroups(t *testing.T) {
+	open, record := heldMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	if err := inv.RecordTakeStarted(ctx, inventory.TakenLibrary{Library: "media", Home: "anchor", Root: "/storage/media",
+		By: "the operator", Ask: "c01", Warrant: "c01"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.TakeDone(ctx, "media", "c01"); err != nil {
+		t.Fatal(err)
+	}
+	_ = record
+	if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.LibraryContract,
+		SaidAt: time.Now(), HeardAt: time.Now()}); err != nil {
+		t.Fatal(err)
+	}
+	held, err := holdJoining(ctx, inv, "anchor", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if held["radarr"].commit != "c1c1c1c1" {
+		t.Fatalf("on an engine at contract 6 radarr is not held at the build it ran: %+v", held)
+	}
+	if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.OwnGroupContract,
+		SaidAt: time.Now().Add(time.Second), HeardAt: time.Now()}); err != nil {
+		t.Fatal(err)
+	}
+	if held, err = holdJoining(ctx, inv, "anchor", nil); err != nil || len(held) > 0 {
+		t.Fatalf("on an engine at contract 7 with its library taken radarr is still held: %+v (%v)", held, err)
+	}
+}
+
+// heldMesh is anchor running radarr's build from before it joined, the joining build registered, media defined.
+func heldMesh(t *testing.T) (*stores, inventory.Node) {
+	t.Helper()
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	register(t, open, catalogue.Manifest{Module: catalogue.LibrariesModule, Version: "1"})
+	if err := inv.SetSettings(ctx, "", catalogue.LibrariesModule, map[string]any{catalogue.LibrariesSetting: map[string]any{
+		"media": map[string]any{"home": "anchor", "root": "/storage/media"}}}); err != nil {
+		t.Fatal(err)
+	}
+	for _, b := range []struct {
+		id, commit string
+		m          catalogue.Manifest
+	}{{"b1", "c1c1c1c1", radarrBefore()}, {"b2", "c2c2c2c2", radarrJoined()}} {
+		raw, _ := json.Marshal(b.m)
+		if err := inv.RecordBuild(ctx, inventory.Build{ID: b.id, Repository: "https://forge.invalid/media.git", Ref: "main",
+			Module: "radarr", Commit: b.commit, On: "builder", Manifest: raw}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.RegisterModule(ctx, radarrBefore(), inventory.Source{Repository: "https://forge.invalid/media.git",
+		Ref: "main", BuiltFrom: "c1c1c1c1"}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := assign(ctx, open, "anchor", "radarr"); err != nil {
+		t.Fatal(err)
+	}
+	record, err := inv.NodeByName(ctx, "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSentWith(ctx, record.ID, "d1", map[string]string{"radarr": "c1c1c1c1"}, 1, 1,
+		inventory.Send{Sequence: 1}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RegisterModule(ctx, radarrJoined(), inventory.Source{Repository: "https://forge.invalid/media.git",
+		Ref: "main", BuiltFrom: "c2c2c2c2"}); err != nil {
+		t.Fatal(err)
+	}
+	return open, record
+}
+
+// The re-review of 96e15f86, item 3: a held module is composed with the settings it last ran with — a layer changed
+// for the joining build (its puid removed) does not reach the build it still runs.
+func TestAHeldModuleKeepsTheSettingsItRanWith(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	register(t, open, catalogue.Manifest{Module: catalogue.LibrariesModule, Version: "1"})
+	if err := inv.SetSettings(ctx, "", catalogue.LibrariesModule, map[string]any{catalogue.LibrariesSetting: map[string]any{
+		"media": map[string]any{"home": "anchor", "root": "/storage/media"}}}); err != nil {
+		t.Fatal(err)
+	}
+	before := radarrBefore()
+	before.Resources = append(before.Resources, map[string]any{"id": "identity", "type": "file",
+		"path": "${dir:config}/identity.env", "mode": "0644", "content": "PUID=${setting:puid}\n"})
+	joined := radarrJoined()
+	for _, b := range []struct {
+		id, commit string
+		m          catalogue.Manifest
+	}{{"b1", "c1c1c1c1", before}, {"b2", "c2c2c2c2", joined}} {
+		raw, _ := json.Marshal(b.m)
+		if err := inv.RecordBuild(ctx, inventory.Build{ID: b.id, Repository: "https://forge.invalid/media.git", Ref: "main",
+			Module: "radarr", Commit: b.commit, On: "builder", Manifest: raw}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := inv.RegisterModule(ctx, before, inventory.Source{Repository: "https://forge.invalid/media.git",
+		Ref: "main", BuiltFrom: "c1c1c1c1"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.SetSettings(ctx, "anchor", "radarr", map[string]any{"puid": "1000"}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := assign(ctx, open, "anchor", "radarr"); err != nil {
+		t.Fatal(err)
+	}
+	record, err := inv.NodeByName(ctx, "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSentWith(ctx, record.ID, "d1", map[string]string{"radarr": "c1c1c1c1"}, 1, 1,
+		inventory.Send{Sequence: 1}); err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(20 * time.Millisecond)
+	if err := inv.RegisterModule(ctx, joined, inventory.Source{Repository: "https://forge.invalid/media.git",
+		Ref: "main", BuiltFrom: "c2c2c2c2"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.ClearSettings(ctx, "anchor", "radarr"); err != nil { // the joining build reads no puid
+		t.Fatal(err)
+	}
+	if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.OwnGroupContract,
+		SaidAt: time.Now(), HeardAt: time.Now()}); err != nil {
+		t.Fatal(err)
+	}
+	gens, err := generators(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	plan, settings, err := planFor(ctx, open, "anchor")
+	if err != nil {
+		t.Fatal(err)
+	}
+	held, err := declarationWith(ctx, open, "anchor", plan, settings, gens, Reading)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if why, left := held.leftOutWhy["radarr"]; left {
+		t.Fatalf("radarr held was left out: %s", why)
+	}
+	for _, r := range held.Resources {
+		if r["id"] == "radarr.identity" {
+			if r["content"] != "PUID=1000\n" {
+				t.Fatalf("radarr held reads %q; want the puid it ran with", r["content"])
+			}
+			return
+		}
+	}
+	t.Fatal("radarr held has no identity file")
+}

cmd/mesh-controller/kept_libraries_test.go

@@ -543,3 +543,43 @@ func TestALibraryIsTakenOnlyWhenTheEngineSaysTheActIsDone(t *testing.T) {
 		t.Fatalf("the act done did not make the library taken (%v)", err)
 	}
 }
+
+// ADR 0334: the engine states the tool runner's membership as resources of kind runner — `runner:<module>` of that
+// module, `runner:library:<name>` of no module. A library's is said for the modules that use it; neither fails the
+// first-node gate of a move that leaves the module's libraries as they were, and both hold one that changes them.
+func TestARunnersVerdictIsSaidAndJudgedByWhetherTheMoveChangesIt(t *testing.T) {
+	inv := inventory.ForTest(t)
+	ctx := t.Context()
+	if _, err := inv.AddNode(ctx, "ace"); err != nil {
+		t.Fatal(err)
+	}
+	for _, m := range []catalogue.Manifest{radarrManifest, plexManifest} {
+		if err := inv.RegisterModule(ctx, m, inventory.Source{}); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := inv.Assign(ctx, "ace", m.Module); err != nil {
+			t.Fatal(err)
+		}
+	}
+	lib := inventory.ResourceHealth{Resource: "runner:library:media", Kind: link.KindRunner, Target: "jochen",
+		State: link.StateUnhealthy, Reason: "the tool runner is not in mesh-media yet", Since: time.Now()}
+	copies := libraryVerdictsFor(ctx, inv, "ace", []inventory.ResourceHealth{lib})
+	if len(copies) != 2 || copies[0].Module != "plex" || copies[1].Module != "radarr" {
+		t.Fatalf("the runner's library verdict is said for %+v; want plex and radarr", copies)
+	}
+	own := inventory.ResourceHealth{Module: "radarr", Resource: "runner:radarr", Kind: link.KindRunner, Target: "jochen",
+		State: link.StateUnhealthy, Reason: "the tool runner is not in mesh-radarr yet", Since: time.Now()}
+	now := time.Now()
+	for _, r := range []inventory.ResourceHealth{copies[1], own} {
+		f := gateFacts{now: now, waits: libraryFactsWith(), groupsAdded: map[string]bool{"radarr": false},
+			accessesChanged: map[string]bool{"radarr": false},
+			health:          map[string]inventory.NodeHealth{"ace": {Node: "ace", HeardAt: now, Resources: []inventory.ResourceHealth{r}}}}
+		if h, why := moduleHealthWord("radarr", "ace", now.Add(-time.Minute), f); h != healthGood {
+			t.Errorf("%s, a move that changes no library: %v %q; want not counted", r.Resource, h, why)
+		}
+		f.accessesChanged = map[string]bool{"radarr": true}
+		if h, _ := moduleHealthWord("radarr", "ace", now.Add(-time.Minute), f); h != healthNotYet {
+			t.Errorf("%s, a move that changes a library: %v; want held to the bound", r.Resource, h)
+		}
+	}
+}

cmd/mesh-controller/module_health.go

@@ -42,6 +42,9 @@ const (
 	moduleUnhealthyAfter = 2
 )
 
+// runnerLibrary prefixes the engine's verdict on the tool runner's membership of a library's group (ADR 0334).
+const runnerLibrary = "runner:library:"
+
 // libraryVerdictsFor is, for each library verdict in a statement (kind library, no module), one copy for each module
 // on the machine — assigned, or pulled in by a requirement — whose manifest uses that library: what the module is judged and said by. None when the
 // assignments or the catalogue cannot be read: the library's own verdict stands alone, and no module is excused by it.
@@ -49,7 +52,8 @@ func libraryVerdictsFor(ctx context.Context, inv *inventory.Inventory, node stri
 	rs []inventory.ResourceHealth) []inventory.ResourceHealth {
 	var libraries []inventory.ResourceHealth
 	for _, r := range rs {
-		if r.Module == "" && r.Kind == link.KindLibrary && strings.HasPrefix(r.Resource, libraryAccess) {
+		if r.Module == "" && ((r.Kind == link.KindLibrary && strings.HasPrefix(r.Resource, libraryAccess)) ||
+			(r.Kind == link.KindRunner && strings.HasPrefix(r.Resource, runnerLibrary))) {
 			libraries = append(libraries, r)
 		}
 	}
@@ -80,7 +84,7 @@ func libraryVerdictsFor(ctx context.Context, inv *inventory.Inventory, node stri
 	}
 	var out []inventory.ResourceHealth
 	for _, lib := range libraries {
-		name := strings.TrimPrefix(lib.Resource, libraryAccess)
+		name := strings.TrimPrefix(strings.TrimPrefix(lib.Resource, runnerLibrary), libraryAccess)
 		for _, module := range assigned {
 			for _, u := range shelf[module].Libraries {
 				if u.Library == name {
@@ -790,6 +794,11 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
 			(r.State == link.StateUnknown || (r.Kind == link.KindLibrary && r.State == link.StateStarting)) {
 			continue
 		}
+		// The tool runner's membership (ADR 0334 §2), of a move that changes no library of the module, is not this
+		// move's: it is said as the module's condition, and holds a move that changes them to the gate's bound.
+		if r.Kind == link.KindRunner && !changes {
+			continue
+		}
 		switch r.State {
 		case link.StateHealthy:
 		case link.StateStarting:

cmd/mesh-controller/plan.go

@@ -134,8 +134,9 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
 	if _, err := keepRecorded(ctx, open, nodeName, shelf); err != nil {
 		return catalogue.Resolution{}, nil, err
 	}
-	// And a module that joins a library not yet taken, at the build it ran before (novox/hq ADR 0328 §8).
-	if _, err := holdJoining(ctx, inv, nodeName, shelf); err != nil {
+	// And a module that may not join yet, at the build it ran before (novox/hq ADR 0328 §8, ADR 0334).
+	held, err := holdJoining(ctx, inv, nodeName, shelf)
+	if err != nil {
 		return catalogue.Resolution{}, nil, err
 	}
 
@@ -223,6 +224,10 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso
 		}
 		settings[m.Module] = layers
 	}
+	// A held module with the settings it last ran with (ADR 0328 §8).
+	if err := heldSettings(ctx, inv, nodeName, held, settings); err != nil {
+		return catalogue.Resolution{}, nil, err
+	}
 	return resolved, settings, nil
 }
 
@@ -490,8 +495,8 @@ func declarationWith(ctx context.Context, open *stores, node string,
 		if kept == nil && len(held) > 0 {
 			kept = map[string]string{}
 		}
-		for m, was := range held {
-			kept[m] = was
+		for m, h := range held {
+			kept[m] = h.commit
 		}
 		for m, was := range kept {
 			if _, carried := out.Builds[m]; carried {
@@ -1035,15 +1040,16 @@ func engineJudgesRoot(ctx context.Context, inv *inventory.Inventory, node string
 	return had && stated.Contract >= link.RootContract, nil
 }
 
-// engineKeepsLibraries says whether a machine's node-engine makes the mesh's users and library groups and keeps
-// libraries (novox/hq ADR 0328), by its own newest statement, for the same reason as engineReadsHealth: an older engine
-// parses strictly, and is sent none of it.
+// engineKeepsLibraries says whether a machine's node-engine makes the mesh's users and library groups, keeps
+// libraries, and keeps a joining module's own groups (novox/hq ADR 0328, ADR 0334: its statement's contract is
+// link.OwnGroupContract or later), by its own newest statement. On an older one no module joins: the libraries
+// alone are not enough, since its tools would lose their read of a joining module's directories.
 func engineKeepsLibraries(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
 	stated, had, err := inv.HealthOf(ctx, node)
 	if err != nil {
 		return false, err
 	}
-	return had && stated.Contract >= link.LibraryContract, nil
+	return had && stated.Contract >= link.OwnGroupContract, nil
 }
 
 // libraryRendering is what a machine's composition needs of ADR 0328.
@@ -1083,11 +1089,7 @@ func libraryFacts(ctx context.Context, inv *inventory.Inventory, node string, pl
 	if lf.keeps, err = engineKeepsLibraries(ctx, inv, node); err != nil || !lf.keeps {
 		return lf, err
 	}
-	if stated, had, err := inv.HealthOf(ctx, node); err != nil {
-		return lf, err
-	} else {
-		lf.ownGroups = had && stated.Contract >= link.OwnGroupContract
-	}
+	lf.ownGroups = true // a module joins only on an engine that also keeps its own groups (ADR 0334)
 	lf.identities = map[string]catalogue.ModuleIdentity{}
 	for _, m := range plan.Modules {
 		if !catalogue.Joins(m) {

cmd/mesh-controller/release.go

@@ -253,9 +253,24 @@ func withoutKept(ctx context.Context, open *stores, node string, moves []invento
 		keeping = keepingRecorded(ctx)
 	}
 	keeps, err := recordedKept(keeping, open, node)
-	if err != nil || len(keeps) == 0 {
+	if err != nil {
 		return moves, err
 	}
+	// And a joining module held at the build it runs (novox/hq ADR 0328 §8): its new build is no move this send makes,
+	// and judging it would read the gate as superseded by its own send (the re-review of 96e15f86, item 1).
+	held, err := holdJoining(ctx, open.inventory, node, nil)
+	if err != nil {
+		return moves, err
+	}
+	if len(held) > 0 && keeps == nil {
+		keeps = map[string]string{}
+	}
+	for m, h := range held {
+		keeps[m] = h.commit
+	}
+	if len(keeps) == 0 {
+		return moves, nil
+	}
 	var out []inventory.CarriedMove
 	for _, mv := range moves {
 		if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) {

cmd/mesh-controller/release_plan.go

@@ -690,7 +690,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 	// being behind. Now every module of the tier whose first machine is the same is sent there in one
 	// send, judged there by one gate — kept on the first of them, the others gated by it — and the rest
 	// of the tier's machines are sent once, together, when the gates have passed.
-	var pending []string
+	var pending, waitsForAPerson []string
 	firstOn := map[string][]string{}
 	runningOf := map[string][]string{}
 	var rest []string
@@ -706,6 +706,20 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 			return false, err
 		}
 		runningOf[m] = running
+		// **A joining build held for a person is not sent** (novox/hq ADR 0328 §8, the re-review of 96e15f86): its
+		// machines keep the build they run until its library is taken (or their engine keeps own groups), so sending
+		// it would send nothing new and judge the old build. It waits for a person, said; the plan stays open, the other
+		// modules are sent and judged, and it is sent on the walk after nothing holds it — no new merge.
+		if state.FirstAt == nil {
+			why, err := heldIn(ctx, inv, m, running)
+			if err != nil {
+				return false, err
+			}
+			if why != "" {
+				waitsForAPerson = append(waitsForAPerson, why)
+				continue
+			}
+		}
 		// **A build whose source is unchanged is never a move** (novox/hq issue 280): made from the source
 		// of the build every machine running it was sent, it is registered with that build's artifacts —
 		// nothing to send, and nothing for a gate to judge.
@@ -868,9 +882,17 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 			strings.Join(sent, ", "))
 		return true, nil
 	}
-	if len(pending) > 0 {
-		note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(pending, "; ") +
-			" to report it applied before the rest are sent"
+	if len(pending) > 0 || len(waitsForAPerson) > 0 {
+		note := "tier " + fmt.Sprint(p.Tier) + " built; "
+		if len(pending) > 0 {
+			note += "waiting for " + strings.Join(pending, "; ") + " to report it applied before the rest are sent"
+		}
+		if len(waitsForAPerson) > 0 {
+			if len(pending) > 0 {
+				note += "; and "
+			}
+			note += "waiting for a person: " + strings.Join(waitsForAPerson, "; ")
+		}
 		changed := p.State != inventory.PlanRolling || p.Note != note
 		p.State = inventory.PlanRolling
 		p.Note = note