Merge pull request 'Limit what a build or check starts on a build node: a per-node build slice (issue 1798)' (#432) from fix/1798-limit-build-resources into main

# Conflicts:
#	internal/broker/testdata/composed.conf
#	internal/link/builds_nats.go

Makefile

@@ -117,7 +117,7 @@ proxy-image:
 # as the script runs them: a bus per test, of the release the mesh runs, makes the suite the same in any
 # order, and the timeout bounds a hang to a failure with a stack, never a stalled gate.
 check: postgres
-	@sh merge-check.sh ; status=$$? ; docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
+	@sh merge-check.sh ; status=$$? ; docker rm -f -v $(PG_CONTAINER) >/dev/null 2>&1 || true ; exit $$status
 
 # The unit level alone (untagged: no bus, no store, no other process), which a developer runs continuously. The tests
 # that need a bus or the store carry `//go:build integration` and run in `check`, not here; without a database the
@@ -135,7 +135,7 @@ fmt:
 	if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
 
 postgres:
-	@docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true
+	@docker rm -f -v $(PG_CONTAINER) >/dev/null 2>&1 || true
 	@docker run -d --name $(PG_CONTAINER) -e POSTGRES_PASSWORD=check \
 		-p 127.0.0.1:$(PG_PORT):5432 $(PG_IMAGE) >/dev/null
 	@printf 'waiting for postgres'
@@ -147,7 +147,7 @@ postgres:
 	echo ' — never came up' ; docker logs $(PG_CONTAINER) | tail -20 ; exit 1
 
 postgres-stop:
-	@docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true
+	@docker rm -f -v $(PG_CONTAINER) >/dev/null 2>&1 || true
 
 clean:
 	rm -rf build/

cmd/mesh-builder/busy.go

@@ -6,6 +6,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 )
 
@@ -73,14 +75,77 @@ func drainedTooLong(now time.Time) error {
 		"within %s (novox/hq ADR 0335)", began.UTC().Format(time.RFC3339), drainTooLong)
 }
 
+// inHand is true while this process has claimed a build and not yet released it. The mark in the temporary directory
+// is only believed while it is: a mark with no build of this process behind it is stale (a crash, a fault in an
+// earlier path) and is removed, because left alone it hands back every ask for ever (issue 1768).
+var inHand atomic.Bool
+
+// markRetry is how long an agent that cannot mark a build in hand takes no work, and markFault when it last failed to.
+// Taking nothing is the point: an ask fetched and handed back at once, many times a second, spends all its deliveries.
+const markRetry = 30 * time.Second
+
+var (
+	markMu    sync.Mutex
+	markFault time.Time
+	markSaid  string
+)
+
+// markFaulty says the agent could not mark a build in hand and has not yet been able to since: it takes no work. Once
+// markRetry has passed it tries the mark itself, with no ask in hand, so that an ask is fetched only when marking works.
+func markFaulty(now time.Time) bool {
+	markMu.Lock()
+	failed := markFault
+	markMu.Unlock()
+	if failed.IsZero() {
+		return false
+	}
+	if now.Sub(failed) < markRetry {
+		return true
+	}
+	f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
+	if err != nil && errors.Is(err, os.ErrExist) && !inHand.Load() {
+		_ = os.Remove(inHandMark())
+		f, err = os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
+	}
+	if err != nil {
+		markFailed(err)
+		return true
+	}
+	_ = f.Close()
+	_ = os.Remove(inHandMark())
+	markMu.Lock()
+	markFault, markSaid = time.Time{}, ""
+	markMu.Unlock()
+	fmt.Fprintf(os.Stderr, "the mark of a build in hand can be made again (%s): taking builds\n", inHandMark())
+	return false
+}
+
+// markFailed notes that a build could not be marked in hand, and says it when it is a new reason.
+func markFailed(err error) {
+	markMu.Lock()
+	defer markMu.Unlock()
+	markFault = time.Now()
+	if said := err.Error(); said != markSaid {
+		markSaid = said
+		fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so this agent takes no build for %s and tries again: %v\n",
+			inHandMark(), markRetry, err)
+	}
+}
+
 // claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes
 // back to the queue.
 func claim() bool {
+	if !inHand.Load() {
+		// Nothing of this process is in hand, so a mark already there is stale.
+		if err := os.Remove(inHandMark()); err == nil {
+			fmt.Fprintf(os.Stderr, "removed the mark of a build in hand (%s): no build of this agent is in hand\n", inHandMark())
+		}
+	}
 	// Made, never written over: one build at a time, and a mark already there is a build in hand.
 	f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
 	if err != nil {
 		// Unmarked, an ask would say idle while this build runs: the build is not taken.
-		fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
+		markFailed(err)
 		return false
 	}
 	_, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n")
@@ -89,7 +154,7 @@ func claim() bool {
 	}
 	if err != nil {
 		_ = os.Remove(inHandMark())
-		fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err)
+		markFailed(err)
 		return false
 	}
 	if drained() {
@@ -97,6 +162,7 @@ func claim() bool {
 		fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back")
 		return false
 	}
+	inHand.Store(true)
 	return true
 }
 
@@ -107,6 +173,7 @@ func holding(what string) {
 
 // released ends the build in hand.
 func released() {
+	inHand.Store(false)
 	if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
 		fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err)
 	}

cmd/mesh-builder/busy_test.go

@@ -160,3 +160,64 @@ func TestAnUnknownCommandFails(t *testing.T) {
 		t.Fatalf("help failed: %v", err)
 	}
 }
+
+// 1768/AC2: a mark naming no build of this agent (left by a crash or a fault) is stale: the next ask removes it and is
+// run, never fetched and handed back in a loop.
+func TestAStaleInHandMarkIsRemovedAndTheAskRuns(t *testing.T) {
+	t.Setenv("TMPDIR", t.TempDir())
+	inHand.Store(false)
+	if err := undrained(); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(inHandMark(), []byte("2026-10-21T18:45:06Z\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if !claim() {
+		t.Fatal("a stale in-hand mark made the agent hand back an ask")
+	}
+	// A live one — this agent's own build in hand — is never taken over.
+	if claim() {
+		t.Fatal("a second build was taken while one is in hand")
+	}
+	if _, err := busy(); err != nil {
+		t.Fatalf("the build in hand was lost: %v", err)
+	}
+	released()
+}
+
+// 1768/AC3: an agent that cannot mark a build in hand for another reason takes no work, instead of fetching asks and
+// handing them back; it takes work again once marking works.
+func TestAnAgentThatCannotMarkABuildTakesNoWorkUntilItCan(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("TMPDIR", dir)
+	inHand.Store(false)
+	markFault, markSaid = time.Time{}, ""
+	t.Cleanup(func() { markFault, markSaid = time.Time{}, "" })
+	if err := undrained(); err != nil {
+		t.Fatal(err)
+	}
+	// The mark's place is a directory that cannot be removed or made over: a directory named as the mark.
+	if err := os.MkdirAll(inHandMark()+"/x", 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if claim() {
+		t.Fatal("an agent that cannot mark a build took it")
+	}
+	if !markFaulty(time.Now()) {
+		t.Fatal("an agent that cannot mark a build still takes work")
+	}
+	// Still failing after the retry time: still takes nothing, and no ask was needed to find out.
+	if !markFaulty(time.Now().Add(2 * markRetry)) {
+		t.Fatal("an agent whose mark still cannot be made takes work")
+	}
+	if err := os.RemoveAll(inHandMark()); err != nil {
+		t.Fatal(err)
+	}
+	if markFaulty(time.Now().Add(4 * markRetry)) {
+		t.Fatal("an agent whose mark can be made again takes no work")
+	}
+	if !claim() {
+		t.Fatal("an agent that can mark takes no build")
+	}
+	released()
+}

cmd/mesh-builder/holder_test.go

@@ -91,7 +91,7 @@ func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
 		t.Errorf("kill said %q", said)
 	}
 	// Removed at the kill and again once the build had ended: a container made in between is caught.
-	if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") {
+	if len(removed) != 6 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[3], "label=mesh.build=build-1") {
 		t.Errorf("removed %v", removed)
 	}
 	if c := h.current(); c.Running != nil {
@@ -160,7 +160,7 @@ func TestAHolderStartingRemovesWhatEarlierBuildsLeftHere(t *testing.T) {
 		}
 		return "", nil
 	}
-	if n := h.removeLeftBehind(); n != 2 || len(removed) != 2 || removed[1] != "docker rm -f c1 c2" {
+	if n := h.removeLeftBehind(); n != 2 || len(removed) != 3 || removed[2] != "docker rm -f -v c1 c2" {
 		t.Fatalf("removed %d: %v", n, removed)
 	}
 	h.remove = func(context.Context, string, string, ...string) (string, error) {

cmd/mesh-builder/main.go

@@ -64,6 +64,8 @@ is dialled except the broker.
   MESH_NPM_TOKEN     the token for it, likewise
   MESH_NPM_SCOPE     the scope it answers for (default: @novox)
   MESH_WORKSPACE     where to clone and build (default: a temporary directory)
+  MESH_BUILD_CGROUP_PARENT  the slice every container a build or check starts joins, which carries the
+                     machine's limits on builds (novox/hq issue 1798; default: the runtime's own)
 
 It also builds one module and stops, which is how a mesh is raised — before there is a
 broker to take work from or a registry to publish into:
@@ -137,6 +139,10 @@ func run() error {
 	if err != nil {
 		return err
 	}
+	// A build and a check run below the machine's own work (novox/hq issue 1798): the agent itself at the
+	// lowest priority, which its git and go children inherit, and every container in the build slice.
+	_ = syscall.Setpriority(syscall.PRIO_PROCESS, 0, 19)
+	builder.CgroupParent = strings.TrimSpace(os.Getenv("MESH_BUILD_CGROUP_PARENT"))
 	workspace := os.Getenv("MESH_WORKSPACE")
 	if workspace == "" {
 		workspace = os.TempDir() + "/mesh-builder"
@@ -184,7 +190,7 @@ func run() error {
 	// Paused or drained, it takes nothing new (novox/hq ADR 0219, ADR 0335): a drained agent finishes the build
 	// in hand and then says idle, and its container is replaced between builds.
 	machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{
-		Paused: func() bool { return h.Paused() || drained() },
+		Paused: func() bool { return h.Paused() || drained() || markFaulty(time.Now()) },
 		Taking: func() { taking(seat) },
 		Claim:  claim,
 	})

cmd/mesh-controller/acts.go

@@ -408,12 +408,21 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
 		return ""
 	}
 	user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
-	if _, reads := m.OwnSecrets["broker"]; !reads {
+	_, reads := m.OwnSecrets["broker"]
+	if !reads && !runsTheProvisionerClient(m) {
+		// A module whose bus accounts are its processes' alone has none of its own: only the processes that
+		// are not issued yet are issued (issue 1226).
+		return processAccountsOnAssign(ctx, inv, m, node)
+	}
+	if !reads {
 		// A module running the provisioner client alone is minted the client's account (issue 917).
 		user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username()
 	}
-	if _, minted, err := inv.BusUserHash(ctx, user); err != nil || minted {
+	if _, minted, err := inv.BusUserHash(ctx, user); err != nil {
 		return ""
+	} else if minted {
+		// Its own account stands and is kept; a process it gained is not issued yet (issue 1226).
+		return processAccountsOnAssign(ctx, inv, m, node)
 	}
 	busAddress, err := broker.BusAddress()
 	if err == nil {
@@ -426,6 +435,16 @@ func issueOnAssign(ctx context.Context, open *stores, node, module string) strin
 	return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
 }
 
+// processAccountsOnAssign issues the process accounts of an assigned module not issued yet, and says what it did.
+func processAccountsOnAssign(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string) string {
+	line, err := issueMissingProcessAccounts(ctx, inv, m, node)
+	if err != nil {
+		return fmt.Sprintf("its process accounts are not issued (%v): `module issue %s --node %s` first — "+
+			"`push %s` refuses to send %s until it is", err, m.Module, node, node, m.Module)
+	}
+	return line
+}
+
 // refusedMovingTheController refuses assigning or unassigning the controller's module through a verb (review of
 // novox/hq ADR 0272): the node it is assigned to is the control-node, and the control-node's operator account is
 // the controller's terminal, so whoever moves the module chooses the terminal. At the terminal alone, as every

cmd/mesh-controller/asker.go

@@ -645,6 +645,10 @@ func doesWords(act conditions.Action) string {
 		return "the delivery is stopped"
 	case act.Verb == broker.ControllerSeat+".plans" && act.Arguments["retry"] != "":
 		return "the failed builds are asked again"
+	case act.Verb == broker.ControllerSeat+".push" && act.Arguments["node"] != "":
+		return "the held builds are sent to " + act.Arguments["node"]
+	case act.Verb == broker.ControllerSeat+".upgrade" && act.Arguments["release-backlog"] != "":
+		return "the builds held after the failed delivery are released"
 	case strings.HasSuffix(act.Verb, ".restart"):
 		return "its service is restarted on " + act.Machine
 	}

cmd/mesh-controller/batches.go

@@ -232,6 +232,8 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
 		fmt.Print(nothingReadsIt(repository, m))
 		return nil
 	}
+	// **A module the merge adds is asked for now** (novox/hq issue 1617), not when the batch is cut.
+	askNewModulesAtHearing(ctx, inv, m, entries, read, facts.from)
 	// **Its delivery group's word, before the plans are held** (ADR 0306 decision 2): a slow answer holds up no walk.
 	own := ownPath(touches)
 	waits, holding, err := groupWaitHolding(ctx, inv, m, deliverySeatHeld(entries))

cmd/mesh-controller/build.go

@@ -768,6 +768,9 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
 		}
 		return manifest, kept, err
 	}
+	// A bus process the new definition gains gets its account now, on each machine assigned the module, so the
+	// next push composes a real account (issue 1226).
+	issueGainedProcessAccounts(ctx, inv, manifest)
 	// Which repository it is registered from, by the forge's own id (novox/hq ADR 0266).
 	if err := inv.SetSourceIdentity(ctx, manifest.Module, repoID); err != nil {
 		return manifest, kept, err

cmd/mesh-controller/busobjects.go

@@ -32,6 +32,13 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra
 	if err := broker.Raise(r, names); err != nil {
 		return nil, err
 	}
+	// And each node's bucket for the files of its declaration that travel by digest (novox/hq ADR 0486 §4), so a node finds its own
+	// whenever the bus is up.
+	if files, ok := r.(interface{ EnsureDeclarationFiles(nodes []string) error }); ok {
+		if err := files.EnsureDeclarationFiles(names); err != nil {
+			return nil, err
+		}
+	}
 	// The work queues of the mesh's own roles (novox/hq ADR 0121). The queue before the holder,
 	// deliberately: work queues until somebody arrives to do it, so assigning a build machine a week
 	// after something started asking for builds flushes the backlog instead of having lost it.

cmd/mesh-controller/checks.go

@@ -594,11 +594,13 @@ func checkedOf(result link.BuildResult) link.Checked {
 		// A delivery group's composed check (novox/hq ADR 0239): every head it judged, this one first.
 		if g := result.Checked; g.Group != "" {
 			c.Group = g.Group
+			// Every member is named with its trunk commit (novox/hq issue 1106): the seat's word, else the one the forge
+			// announced with the head, so the forge's merge tool never meets a member whose verdict names none.
 			c.Members = append(c.Members, link.CheckedMember{Owner: g.Owner, Repo: g.Repo, Number: g.Number,
-				Commit: result.Ref, Trunk: c.Trunk})
+				Commit: result.Ref, Trunk: firstTrunk(c.Trunk, g.Trunk)})
 			for _, m := range g.Members {
 				c.Members = append(c.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Number: m.Number,
-					Commit: m.Ref, Trunk: memberTrunk(result.Check, m)})
+					Commit: m.Ref, Trunk: firstTrunk(memberTrunk(result.Check, m), m.Trunk)})
 			}
 		}
 	}
@@ -627,6 +629,16 @@ func checkedOf(result link.BuildResult) link.Checked {
 	return c
 }
 
+// firstTrunk is the first trunk commit named, as twelve characters; empty when none is.
+func firstTrunk(named ...string) string {
+	for _, n := range named {
+		if n != "" {
+			return twelve(n)
+		}
+	}
+	return ""
+}
+
 // memberTrunk is the trunk commit the build seat merged one of a group's heads into (novox/hq ADR 0323 §6), empty
 // when it said none.
 func memberTrunk(outcome *link.CheckOutcome, m link.GroupMember) string {

cmd/mesh-controller/checks_test.go

@@ -250,6 +250,31 @@ func TestAVerdictNamesTheTrunkCommitItJudged(t *testing.T) {
 	}
 }
 
+// **A group's composed verdict names every member's trunk commit** (novox/hq issue 1106): a member the build seat named no
+// trunk for (an SDK member whose base the seat did not merge, or a seat from before the rule) is named by the trunk the forge
+// announced with its head, and the primary the same, so the merge tool never reads "names no trunk commit" on a member
+// whose trunk did not move.
+func TestAGroupVerdictNamesEveryMembersTrunkTheSDKMemberToo(t *testing.T) {
+	group := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 7, Base: "main", Group: "feat/x",
+		Trunk: "dddddddddddd0000",
+		Members: []link.GroupMember{
+			{Owner: "novox", Repo: "mesh-sdk", Number: 8, Ref: "cccccccccccc0000", Base: "main", Trunk: "eeeeeeeeeeee0000"},
+			{Owner: "novox", Repo: "mesh-host", Number: 9, Ref: "bbbbbbbbbbbb0000", Base: "main", Trunk: "ffffffffffff0000"}}}
+	c := checkedOf(link.BuildResult{ID: "g", Ref: "aaaaaaaaaaaa0000", Checked: group, Check: &link.CheckOutcome{
+		Verdict: "pass", Head: "aaaaaaaaaaaa", Gate: &link.CheckLayer{Verdict: "pass"},
+		Members: []link.CheckedMember{{Owner: "novox", Repo: "mesh-host", Commit: "bbbbbbbbbbbb", Trunk: "222222222222"}}}})
+	want := []string{"dddddddddddd", "eeeeeeeeeeee", "222222222222"}
+	if len(c.Members) != 3 {
+		t.Fatalf("the composed check names its members %+v", c.Members)
+	}
+	for i, m := range c.Members {
+		if m.Trunk != want[i] {
+			t.Fatalf("member %d (%s/%s) is named with trunk %q, want %q (the seat's word first, else the announced one): %+v",
+				i, m.Owner, m.Repo, m.Trunk, want[i], c.Members)
+		}
+	}
+}
+
 // **The merge made for a check is never registered** (novox/hq ADR 0323 §2): a build of it — a commit on no branch
 // of the forge — is refused, naming the trunk. The rule of ADR 0238 §1 stands unchanged; this holds it for the
 // merge commit a check makes.

cmd/mesh-controller/compose_perform.go

@@ -233,6 +233,12 @@ func prepareSend(ctx context.Context, open *stores, node string) ([]string, erro
 					if !slices.Contains(in.BusMinted, "provisioner."+node) {
 						continue
 					}
+				default:
+					// A bus process's account nobody issued still gets the stand-in, until the composition leaves
+					// the module off (issue 1753): the warning names the verb (issue 1226).
+					if line := processAccountWarning(m, node, name, in.BusMinted); line != "" {
+						sayWarning(ctx, line)
+					}
 				}
 				if _, err := inv.SecretForModule(ctx, node, m.Module, name); err == nil {
 					did = true

cmd/mesh-controller/compose_sdk.go

@@ -200,6 +200,7 @@ func composeQuestion(ctx context.Context, open *stores, node string, mode compos
 	}
 	sayFound(ctx, d)
 	out := sendableOf(d)
+	out.fileReferences = readsFileReferences(in.Input, node)
 	if !withOrder {
 		out.Sequence, out.Epoch, out.Generation = 0, 0, 0
 	}
@@ -238,7 +239,11 @@ func markUnresolvable(err error) error {
 // still names anything foreseen is refused, said, and never sent.
 func composeSend(ctx context.Context, open *stores, node string) (sendable, error) {
 	declared, _, err := composeSendValidated(ctx, open, node)
-	return declared, err
+	if err != nil {
+		return declared, err
+	}
+	// Never a declaration naming an image the store does not hold (novox/hq issue 1742): that module is left out.
+	return withoutMissingImages(ctx, open, node, declared), nil
 }
 
 // composeSendValidated is composeSend and what the node-engine's validator refuses of the declaration it composed, for the
@@ -281,6 +286,7 @@ func composeSendValidated(ctx context.Context, open *stores, node string) (senda
 		if len(left) == 0 && len(in.wouldMake) == 0 {
 			sayFound(ctx, d)
 			out := sendableOf(d)
+			out.fileReferences = readsFileReferences(in.Input, node)
 			// The sender stamps the order it allots (order.stamp); a composition for a send carries none of its own.
 			out.Sequence, out.Epoch, out.Generation = 0, 0, 0
 			here, _ := nodeOf(in.Input, node)
@@ -366,7 +372,7 @@ func sayFound(ctx context.Context, d compose.Declared) {
 
 // sendableOf is a composed declaration as the controller's sends know it.
 func sendableOf(d compose.Declared) sendable {
-	out := sendable{Resources: d.Resources, Sequence: d.Sequence, Epoch: d.Epoch, Generation: d.Generation,
+	out := sendable{Resources: compactKept(d.Resources), Sequence: d.Sequence, Epoch: d.Epoch, Generation: d.Generation,
 		ReadsModules: d.ReadsModules, moduleOf: d.ModuleOf, buildIDs: d.BuildIDs, modules: d.Modules,
 		localNetwork: d.LocalNetwork, Received: d.Received, Mesh: d.Mesh, BusUsers: d.BusUsers,
 		LeftOut: d.LeftOut, leftOutWhy: d.LeftOutWhy, withheld: d.Withheld, unbound: d.Unbound, foreseen: d.Foreseen,

cmd/mesh-controller/container_groups_test.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+)
+
+// Issue 1638 (novox/hq ADR 0480 clause 7), taken in with mesh-sdk's container `groups`: the controller sends a container's
+// groups only to a node-engine that reports the capability container-groups, and holds the module on any other, naming it.
+
+const n8nWithAGroup = `{"module":"n8n","version":"1",
+  "config":{"media-gid":{"type":"number","minimum":1,"required":true,"description":"the group that owns the media library"}},
+  "resources":[{"id":"server","type":"container","name":"n8n",
+    "image":"n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0",
+    "groups":["${config:media-gid}"]}]}`
+
+// printedBy runs reportUnhostable and returns what it printed.
+func printedBy(t *testing.T, node string, plan catalogue.Resolution) string {
+	t.Helper()
+	stdout := os.Stdout
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	os.Stdout = w
+	reportUnhostable(node, plan)
+	w.Close()
+	os.Stdout = stdout
+	out, _ := io.ReadAll(r)
+	return string(out)
+}
+
+// 1638/AC1: a node that reports the capability gets the module; one that does not has it held, the capability named.
+func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing.T) {
+	m, err := catalogue.ParseManifest([]byte(n8nWithAGroup))
+	if err != nil {
+		t.Fatal(err)
+	}
+	shelf := catalogue.Shelf{"n8n": m}
+	reads := catalogue.Node{Name: "ace", Capabilities: map[string]bool{"container-runtime": true, "container-groups": true}}
+	old := catalogue.Node{Name: "g14", Capabilities: map[string]bool{"container-runtime": true}}
+
+	got, err := catalogue.Resolve(shelf, []string{"n8n"}, reads, catalogue.World{})
+	if err != nil || len(got.Unhostable) != 0 || len(got.Modules) != 1 {
+		t.Fatalf("a node that reads groups: %v, held %+v", err, got.Unhostable)
+	}
+	if out := printedBy(t, "ace", got); out != "" {
+		t.Errorf("a node that reads groups was told: %q", out)
+	}
+
+	got, err = catalogue.Resolve(shelf, []string{"n8n"}, old, catalogue.World{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got.Modules) != 0 || len(got.Unhostable) != 1 {
+		t.Fatalf("n8n was sent to a node that does not report container-groups: hosted %d, held %+v", len(got.Modules), got.Unhostable)
+	}
+	if out := printedBy(t, "g14", got); !strings.Contains(out, "n8n") || !strings.Contains(out, "container-groups") {
+		t.Errorf("the hold does not name the module and the capability: %q", out)
+	}
+}

cmd/mesh-controller/declaration_capability_integration_test.go

@@ -0,0 +1,110 @@
+//go:build integration
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
+	"git.novox.be/novox/mesh-sdk/go/overlay"
+
+	"git.novox.be/novox/mesh-sdk/go/compose"
+)
+
+// 1696/AC2, ADR 0486 §5: a node is sent a file by digest only when its profile reports the capability. A node whose profile lacks
+// it, and a node the mesh has no profile of, are sent every content inline; a node that reports it is sent the reference. The test
+// goes through composeQuestion and composeSend, the two places readsFileReferences is read (compose_sdk.go).
+func TestAFileTravelsByDigestOnlyToANodeThatReportsTheCapability(t *testing.T) {
+	open := aMesh(t) // anchor and laptop report no file-references
+	ctx := t.Context()
+	big := strings.Repeat("0123456789abcdef", 8<<10) // 128 KiB
+	register(t, open, catalogue.Manifest{Module: "bigfile", Version: "1",
+		Resources: []map[string]any{{"id": "blob", "type": "file", "path": "/etc/bigfile/blob", "mode": "0644", "content": big}}})
+
+	// reader reports the capability; bare has never reported a profile at all.
+	for i, n := range []struct {
+		name    string
+		profile map[string]any
+	}{
+		{"reader", map[string]any{"capabilities": []any{
+			map[string]any{"name": "container-runtime", "present": true},
+			map[string]any{"name": "systemd", "present": true},
+			map[string]any{"name": sdkdecl.FileReferencesCapability, "present": true}}}},
+		{"bare", nil},
+	} {
+		record, err := open.inventory.AddNode(ctx, n.name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := open.inventory.SetPlace(ctx, n.name, n.name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+10)); err != nil {
+			t.Fatal(err)
+		}
+		if n.profile != nil {
+			if err := open.inventory.RecordProfile(ctx, record.ID, n.profile); err != nil {
+				t.Fatal(err)
+			}
+		}
+		if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
+			t.Fatal(err)
+		}
+		if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
+			t.Fatal(err)
+		}
+		if _, err := open.inventory.Assign(ctx, n.name, overlay.Name); err != nil {
+			t.Fatal(err)
+		}
+	}
+	for _, node := range []string{"laptop", "reader", "bare"} {
+		if _, err := assign(ctx, open, node, "bigfile"); err != nil {
+			t.Fatalf("assign %s: %v", node, err)
+		}
+	}
+
+	byDigest := map[string]bool{"laptop": false, "bare": false, "reader": true}
+	for node, want := range byDigest {
+		asked, _, err := composeQuestion(ctx, open, node, compose.Reading, false)
+		if err != nil {
+			t.Fatalf("%s: %v", node, err)
+		}
+		sent, err := composeSend(ctx, open, node)
+		if err != nil {
+			t.Fatalf("%s: %v", node, err)
+		}
+		for what, s := range map[string]sendable{"question": asked, "send": sent} {
+			if s.fileReferences != want {
+				t.Fatalf("%s, %s: fileReferences %v, want %v", node, what, s.fileReferences, want)
+			}
+			body, err := s.Body()
+			if err != nil {
+				t.Fatal(err)
+			}
+			var got struct {
+				Resources []map[string]any `json:"resources"`
+			}
+			if err := json.Unmarshal(body, &got); err != nil {
+				t.Fatal(err)
+			}
+			var found bool
+			for _, r := range got.Resources {
+				if r["path"] != "/etc/bigfile/blob" {
+					continue
+				}
+				found = true
+				inline := r["content"] == big
+				if want && (inline || r["digest"] == nil || r["size"] == nil) {
+					t.Fatalf("%s, %s: a node that reads references is sent the content inline", node, what)
+				}
+				if !want && (!inline || r["digest"] != nil) {
+					t.Fatalf("%s, %s: a node that does not report the capability is sent a reference", node, what)
+				}
+			}
+			if !found {
+				t.Fatalf("%s, %s: the file is not in the declaration", node, what)
+			}
+		}
+	}
+}

cmd/mesh-controller/declaration_files.go

@@ -0,0 +1,253 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sort"
+	"sync"
+
+	"git.novox.be/novox/mesh-sdk/go/compose"
+	sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/conditions"
+)
+
+// **A declaration's large files travel by their digest to a node that can read one** (novox/hq ADR 0486 §1, §5, issue 1696).
+//
+// The declaration to a node that reports sdkdecl.FileReferencesCapability carries, in place of the content of every
+// file resource of more than sdkdecl.FileReferenceThreshold bytes that may travel so (File.TravelsByDigest: plain content or
+// bytes, not sealed, not filled with secrets, not written into another file), its sha256 digest and size. The content is put
+// into the bucket of the bus under the hex of the digest before the declaration is sent. A node that does not report the
+// capability is sent every content inline, as before: an older node-engine refuses a field it does not know, whole.
+
+// readsFileReferences says the node reports the capability of reading a file given by digest (ADR 0486 §5). A node the mesh has
+// no profile of reads none.
+func readsFileReferences(in compose.Input, node string) bool {
+	n, ok := nodeOf(in, node)
+	return ok && n.Capabilities[sdkdecl.FileReferencesCapability]
+}
+
+// referencing is the resources as a node that reads references is sent them, and the content of each file that went by
+// digest, by the name of its object (the hex of its digest). The resources it is given are not changed. A file that does not
+// travel by digest is passed as it is.
+func referencing(resources []map[string]any) ([]map[string]any, map[string][]byte, error) {
+	out := make([]map[string]any, len(resources))
+	var objects map[string][]byte
+	for i, r := range resources {
+		out[i] = r
+		if kind, _ := r["type"].(string); kind != string(sdkdecl.TypeFile) {
+			continue
+		}
+		// Cheap first: only a file with a content or bytes of the size is worth reading whole.
+		content, _ := r["content"].(string)
+		bytesField, _ := r["bytes"].(string)
+		if len(content) <= sdkdecl.FileReferenceThreshold && len(bytesField) <= sdkdecl.FileReferenceThreshold {
+			continue
+		}
+		raw, err := json.Marshal(r)
+		if err != nil {
+			return nil, nil, fmt.Errorf("resource %v: %w", r["id"], err)
+		}
+		var f sdkdecl.File
+		if err := json.Unmarshal(raw, &f); err != nil {
+			return nil, nil, fmt.Errorf("resource %v is not a file the controller can read: %w", r["id"], err)
+		}
+		if !f.TravelsByDigest() {
+			continue
+		}
+		ref, body, err := f.AsReference()
+		if err != nil {
+			return nil, nil, err
+		}
+		name, err := sdkdecl.ObjectName(ref.Digest)
+		if err != nil {
+			return nil, nil, err
+		}
+		with := make(map[string]any, len(r)+2)
+		for k, v := range r {
+			with[k] = v
+		}
+		delete(with, "content")
+		delete(with, "bytes")
+		with["digest"], with["size"] = ref.Digest, ref.Size
+		out[i] = with
+		if objects == nil {
+			objects = map[string][]byte{}
+		}
+		objects[name] = body
+	}
+	return out, objects, nil
+}
+
+// namedObjects are the objects of the bucket a body names: the object name of every file resource given by reference,
+// with its size. It reads a declaration as the node does, so it also reads the latest one the bus holds.
+func namedObjects(body []byte) (map[string]int64, error) {
+	var envelope struct {
+		Resources []struct {
+			Type   string `json:"type"`
+			Digest string `json:"digest"`
+			Size   int64  `json:"size"`
+		} `json:"resources"`
+	}
+	if err := json.Unmarshal(body, &envelope); err != nil {
+		return nil, err
+	}
+	var out map[string]int64
+	for _, r := range envelope.Resources {
+		if r.Type != string(sdkdecl.TypeFile) || r.Digest == "" {
+			continue
+		}
+		name, err := sdkdecl.ObjectName(r.Digest)
+		if err != nil {
+			return nil, err
+		}
+		if out == nil {
+			out = map[string]int64{}
+		}
+		out[name] = r.Size
+	}
+	return out, nil
+}
+
+// objectStore is what putting the objects needs of a node's bucket.
+type objectStore interface {
+	// Has says whether an object of that name is there, complete, and whether it carries the sweep's mark of being named by no
+	// node's latest declaration.
+	Has(ctx context.Context, name string) (have, marked bool, err error)
+	// Put writes the object under that name, replacing one of the same name (and so erasing its mark).
+	Put(ctx context.Context, name string, content []byte) error
+}
+
+// putObjects makes sure every object of the declaration is in the bucket before the declaration that names it is sent: a node
+// that reads the declaration finds each object. An object already there (the same content, the same name) is not written again,
+// **unless the sweep marked it unnamed**: then it is written again, so the mark that would have it removed is gone with the
+// old object (issue 1696, review: a mark read just before a send must not take an object the send relies on). Refuses on the
+// first it cannot put, naming it, so the declaration is not sent.
+func putObjects(ctx context.Context, store objectStore, objects map[string][]byte) error {
+	names := make([]string, 0, len(objects))
+	for name := range objects {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	for _, name := range names {
+		have, marked, err := store.Has(ctx, name)
+		if err != nil {
+			return fmt.Errorf("whether the object %s is in the bucket: %w", name[:12], err)
+		}
+		if have && !marked {
+			continue
+		}
+		if err := store.Put(ctx, name, objects[name]); err != nil {
+			return fmt.Errorf("putting the object %s (%d bytes) in the bucket: %w", name[:12], len(objects[name]), err)
+		}
+	}
+	return nil
+}
+
+// busObjects is a node's bucket on the controller's own connection.
+type busObjects struct{ store jetstream.ObjectStore }
+
+// openBucket opens the bucket of a node's files, making it first if it is not there (it is asserted for every node at the
+// controller's start; a node adopted since is covered here).
+func openBucket(ctx context.Context, conn *nats.Conn, node string) (busObjects, error) {
+	js, err := jetstream.New(conn)
+	if err != nil {
+		return busObjects{}, err
+	}
+	bucket, err := broker.DeclarationFilesBucketOf(node)
+	if err != nil {
+		return busObjects{}, fmt.Errorf("the object bucket of node %q: %w", node, err)
+	}
+	store, err := js.ObjectStore(ctx, bucket)
+	if errors.Is(err, jetstream.ErrBucketNotFound) {
+		if err = broker.EnsureDeclarationFiles(ctx, js, node); err == nil {
+			store, err = js.ObjectStore(ctx, bucket)
+		}
+	}
+	if err != nil {
+		return busObjects{}, fmt.Errorf("the object bucket %s: %w", bucket, err)
+	}
+	return busObjects{store}, nil
+}
+
+func (b busObjects) Has(ctx context.Context, name string) (bool, bool, error) {
+	info, err := b.store.GetInfo(ctx, name)
+	switch {
+	case errors.Is(err, jetstream.ErrObjectNotFound):
+		return false, false, nil
+	case err != nil:
+		return false, false, err
+	}
+	_, marked := info.Metadata[declarationUnnamed]
+	return !info.Deleted, marked, nil
+}
+
+func (b busObjects) Put(ctx context.Context, name string, content []byte) error {
+	_, err := b.store.PutBytes(ctx, name, content)
+	return err
+}
+
+// filesLocks hold a node's bucket for one act at a time in this process: a send puts its objects and publishes the declaration that
+// names them under the node's lock, and the sweep judges the node's bucket under the same one, so the sweep never removes an
+// object between a send's put and its declaration. Another process (a person's push) is covered by the put erasing a mark and by
+// the sweep looking at an object again just before it removes it.
+var filesLocks sync.Map
+
+func lockFilesOf(node string) func() {
+	l, _ := filesLocks.LoadOrStore(node, &sync.Mutex{})
+	mu := l.(*sync.Mutex)
+	mu.Lock()
+	return mu.Unlock
+}
+
+// kindFilesNotPut is a node whose files that travel by digest could not be put in its bucket, send after send.
+const kindFilesNotPut = "declaration-files-not-put"
+
+// filesNotPutAfter is how many sends in a row may fail to put before the condition is raised: the first send to the machine holding
+// the bus fails for want of the grant that very send delivers, and says so, and is not yet a fault.
+const filesNotPutAfter = 2
+
+var (
+	filesFailMu sync.Mutex
+	filesFailed = map[string]int{}
+)
+
+func filesNotPutObservation(node, summary string) conditions.Observation {
+	return conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindFilesNotPut, Token: kindFilesNotPut,
+		Machine: node, Severity: conditions.Warning, Source: "sends", Summary: summary}
+}
+
+// notePut records how putting a send's objects went: a failure counts, the second in a row raises the condition, a success
+// clears the count and the condition.
+func notePut(ctx context.Context, node string, put error) {
+	filesFailMu.Lock()
+	if put == nil {
+		delete(filesFailed, node)
+	} else {
+		filesFailed[node]++
+	}
+	n := filesFailed[node]
+	filesFailMu.Unlock()
+	keeper := conditionsFrom
+	if keeper == nil {
+		return
+	}
+	key := filesNotPutObservation(node, "").Key()
+	switch {
+	case put == nil:
+		if _, err := keeper.Clear(ctx, key, node+" was sent its files by digest"); err != nil {
+			fmt.Printf("the condition %s could not be cleared: %v\n", key, err)
+		}
+	case n >= filesNotPutAfter:
+		summary := fmt.Sprintf("%s's files that travel by digest could not be put in its bucket %d sends in a row, so it is sent them "+
+			"inline: %v", node, n, put)
+		if _, err := keeper.Observe(ctx, filesNotPutObservation(node, summary)); err != nil {
+			fmt.Printf("the condition %s could not be raised: %v\n", key, err)
+		}
+	}
+}

cmd/mesh-controller/declaration_files_integration_test.go

@@ -0,0 +1,246 @@
+//go:build integration
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/link"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+type sigOf struct{}
+
+func (sigOf) Sign(context.Context, []byte) ([]byte, error) { return make([]byte, 64), nil }
+
+// 1696/AC7 (ADR 0486 §1): on a real bus with its default largest message of 1 MiB, a declaration whose files add up to more than
+// 3 MiB is sent as references, its objects are in the bucket, each fetched back whole and verified against the digest and size the
+// declaration names; the same declaration inline is refused by the controller, naming size and limit. A changed byte in an object
+// fails the verification.
+func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T) { // 1696/AC7
+	s := testbus.Start(t)
+	conn, err := nats.Connect(s.ClientURL())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	nc, err := conn.JetStream()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := nc.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"},
+		MaxMsgsPerSubject: 1}); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	api, err := jetstream.New(conn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := broker.EnsureDeclarationFiles(ctx, api, "novox"); err != nil {
+		t.Fatal(err)
+	}
+	// Run twice: an assertion of an existing bucket is no error and no change.
+	if err := broker.EnsureDeclarationFiles(ctx, api, "novox"); err != nil {
+		t.Fatalf("asserting the bucket again: %v", err)
+	}
+
+	var resources []map[string]any
+	for i := 0; i < 3; i++ {
+		resources = append(resources, aFileResource("big-"+string(rune('a'+i)), contentOf("big"+string(rune('a'+i)), 1200<<10)))
+	}
+	resources = append(resources, aFileResource("small", "tiny\n"))
+	inline := sendable{Resources: resources}
+	byDigest := sendable{fileReferences: true, Resources: resources}
+
+	bus := link.OverNATS{Conn: conn, JS: nc}
+	body, err := inline.Body()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := link.Declare(ctx, bus, sigOf{}, "novox", body, 5*time.Second); err == nil ||
+		!strings.Contains(err.Error(), "1048576") || strings.Contains(err.Error(), "maximum payload exceeded") {
+		t.Fatalf("the inline declaration of more than 3 MiB: %v", err)
+	}
+
+	bucket, err := openBucket(ctx, conn, "novox")
+	if err != nil {
+		t.Fatal(err)
+	}
+	objects, err := byDigest.objects()
+	if err != nil || len(objects) != 3 {
+		t.Fatalf("objects: %d %v", len(objects), err)
+	}
+	if err := putObjects(ctx, bucket, objects); err != nil {
+		t.Fatal(err)
+	}
+	if err := putObjects(ctx, bucket, objects); err != nil {
+		t.Fatalf("putting them again: %v", err)
+	}
+	refBody, err := byDigest.Body()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := link.Declare(ctx, bus, sigOf{}, "novox", refBody, 5*time.Second); err != nil {
+		t.Fatalf("the declaration by reference: %v", err)
+	}
+
+	// What the stream holds for the node is the reference form, and it names exactly the three objects.
+	latest, err := latestDeclarations(ctx, conn, []string{"novox", "ace"})
+	if err != nil || len(latest) != 1 {
+		t.Fatalf("the latest declarations: %v %v", len(latest), err)
+	}
+	named, err := namedByLatest("novox", latest["novox"])
+	if err != nil || len(named) != 3 {
+		t.Fatalf("what the latest declaration names: %v %v", named, err)
+	}
+	novoxBucket, err := broker.DeclarationFilesBucketOf("novox")
+	if err != nil {
+		t.Fatal(err)
+	}
+	store, err := api.ObjectStore(ctx, novoxBucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var envelope struct {
+		Resources []sdkdecl.File `json:"resources"`
+	}
+	if err := json.Unmarshal(refBody, &envelope); err != nil {
+		t.Fatal(err)
+	}
+	for _, f := range envelope.Resources {
+		if !f.IsReference() {
+			continue
+		}
+		name, _ := sdkdecl.ObjectName(f.Digest)
+		got, err := store.GetBytes(ctx, name)
+		if err != nil {
+			t.Fatalf("the object of %s: %v", f.ID, err)
+		}
+		if err := f.Verify(got); err != nil {
+			t.Errorf("the object does not verify: %v", err)
+		}
+		got[len(got)/2] ^= 1
+		if err := f.Verify(got); err == nil {
+			t.Errorf("a changed byte in the object of %s passed the verification", f.ID)
+		}
+	}
+
+	// The sweep against the real bucket: all named, nothing marked; a fourth object named by none is marked, kept a day less a
+	// minute, and removed at the day.
+	extra := bytes.Repeat([]byte("x"), 100)
+	extraName := strings.Repeat("e", 64)
+	if err := putObjects(ctx, bucket, map[string][]byte{extraName: extra}); err != nil {
+		t.Fatal(err)
+	}
+	b := busSweptBucket{store}
+	t0 := time.Now()
+	if r, err := sweepDeclarationFiles(ctx, b, named, t0); err != nil || r.Marked != 1 || r.Removed != 0 || r.Kept != 3 {
+		t.Fatalf("first sweep: %+v %v", r, err)
+	}
+	if r, err := sweepDeclarationFiles(ctx, b, named, t0.Add(24*time.Hour-time.Minute)); err != nil || r.Removed != 0 {
+		t.Fatalf("a minute short of a day: %+v %v", r, err)
+	}
+	if r, err := sweepDeclarationFiles(ctx, b, named, t0.Add(24*time.Hour+time.Second)); err != nil || r.Removed != 1 {
+		t.Fatalf("a day after: %+v %v", r, err)
+	}
+	if infos, err := store.List(ctx); err != nil || len(infos) != 3 {
+		t.Fatalf("the bucket holds %d objects (%v), want the 3 the declaration names", len(infos), err)
+	}
+}
+
+// 1696/AC8 on a real bucket: the sweep removes an object only if it still carries the mark it judged it by. An object marked
+// unnamed, then put again by a send (which erases the mark), is kept when the sweep deletes it by the old mark.
+func TestTheSweepsDeleteLeavesAnObjectASendPutAgainAfterItWasMarked(t *testing.T) { // 1696/AC8
+	s := testbus.Start(t)
+	conn, err := nats.Connect(s.ClientURL())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	bucket, err := openBucket(ctx, conn, "novox")
+	if err != nil {
+		t.Fatal(err)
+	}
+	name := strings.Repeat("a", 64)
+	content := map[string][]byte{name: bytes.Repeat([]byte("y"), 100)}
+	if err := putObjects(ctx, bucket, content); err != nil {
+		t.Fatal(err)
+	}
+	b := busSweptBucket{bucket.store}
+	judged := time.Now().Add(-25 * time.Hour).Truncate(time.Second)
+	if err := b.MarkUnnamed(ctx, name, judged); err != nil {
+		t.Fatal(err)
+	}
+	if _, marked, err := bucket.Has(ctx, name); err != nil || !marked {
+		t.Fatalf("the object was not marked: %v %v", marked, err)
+	}
+	// The sweep has read the mark; a send now puts the object again, which erases it.
+	if err := putObjects(ctx, bucket, content); err != nil {
+		t.Fatal(err)
+	}
+	if have, marked, err := bucket.Has(ctx, name); err != nil || !have || marked {
+		t.Fatalf("a send did not put the marked object again: have %v marked %v %v", have, marked, err)
+	}
+	gone, err := b.Delete(ctx, name, judged)
+	if err != nil || gone {
+		t.Fatalf("the sweep deleted by an old mark: gone %v, %v", gone, err)
+	}
+	if have, _, err := bucket.Has(ctx, name); err != nil || !have {
+		t.Fatalf("the object a send relies on is gone: %v %v", have, err)
+	}
+	// And the mark still on it is honoured: marked again and deleted by that mark, it goes.
+	if err := b.MarkUnnamed(ctx, name, judged); err != nil {
+		t.Fatal(err)
+	}
+	if gone, err := b.Delete(ctx, name, judged); err != nil || !gone {
+		t.Fatalf("an object still carrying the mark it was judged by is kept: gone %v, %v", gone, err)
+	}
+}
+
+// 1696/AC9 on a real bus: the sweep of a node's bucket takes the node's lock first, so it waits for a send that has put its objects
+// and not yet published the declaration that names them.
+func TestTheSweepOfANodesBucketHoldsTheNodesLock(t *testing.T) { // 1696/AC9
+	s := testbus.Start(t)
+	conn, err := nats.Connect(s.ClientURL())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	const node = "sweep-lock-node"
+	if _, err := openBucket(ctx, conn, node); err != nil { // the bucket exists, or the sweep would return before the lock
+		t.Fatal(err)
+	}
+	release := lockFilesOf(node) // a send is between its put and its publication
+	done := make(chan struct{})
+	go func() {
+		_, _ = sweepNode(ctx, conn, node)
+		close(done)
+	}()
+	select {
+	case <-done:
+		t.Fatal("the sweep judged the node's bucket while a send held it")
+	case <-time.After(300 * time.Millisecond):
+	}
+	release()
+	select {
+	case <-done:
+	case <-time.After(20 * time.Second):
+		t.Fatal("the sweep did not go on once the send let go")
+	}
+}

cmd/mesh-controller/declaration_files_test.go

@@ -0,0 +1,534 @@
+package main
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// contentOf is a deterministic content of n bytes that does not compress to nothing and holds no character JSON escapes.
+func contentOf(seed string, n int) string {
+	var b strings.Builder
+	for i := 0; b.Len() < n; i++ {
+		fmt.Fprintf(&b, "%s-%07d-%x;", seed, i, i*2654435761%1000003)
+	}
+	return b.String()[:n]
+}
+
+func aFileResource(id, content string) map[string]any {
+	return map[string]any{"id": id, "type": "file", "path": "/etc/" + id, "mode": "0644", "content": content}
+}
+
+// controlNodeShape is a declaration shaped like the control node's on 2026-10-21 (issue 1689), the sizes measured then: a bus users
+// file of 252,678 bytes, the vault's export of 138,475 bytes (compact), the tool runtime 21 KB, the packet filter 19 KB, the launcher
+// 14 KB, and about 600 small resources for the rest (about 286 KB). `more` scales the count of small ones: 1 is the shape, 1.4 forty
+// percent more of them.
+func controlNodeShape(more float64) []map[string]any {
+	res := []map[string]any{
+		aFileResource("nats.bus-users", contentOf("users", 252678)),
+		aFileResource("mesh-vault.kept", contentOf("kept", 138475)),
+		aFileResource("node-tools.runtime", contentOf("runtime", 21<<10)),
+		aFileResource("nftables.filtering", contentOf("filter", 19<<10)),
+		aFileResource("mesh-host.launcher", contentOf("launcher", 14<<10)),
+	}
+	for i := 0; i < int(600*more); i++ {
+		res = append(res, aFileResource(fmt.Sprintf("module-%03d.config", i), contentOf(fmt.Sprintf("small%d", i), 380)))
+	}
+	return res
+}
+
+func wireSizeOf(t *testing.T, s sendable) int {
+	t.Helper()
+	body, err := s.Body()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The signature the controller's key makes is 64 bytes.
+	msg, err := link.Envelope(body, make([]byte, 64))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return len(msg)
+}
+
+// 1696/AC1: for a node that reports the capability, the two big files of a control-node-shaped declaration carry their digest and size
+// and no content, and the declaration is far under the bus's 1 MiB.
+func TestAFileOver64KiBTravelsByDigestToANodeThatReadsOne(t *testing.T) { // 1696/AC1
+	users, kept := contentOf("users", 252678), contentOf("kept", 138475)
+	s := sendable{fileReferences: true, Resources: []map[string]any{
+		aFileResource("nats.bus-users", users), aFileResource("mesh-vault.kept", kept),
+		aFileResource("small", "tiny\n"),
+	}}
+	body, err := s.Body()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got struct {
+		Resources []map[string]any `json:"resources"`
+	}
+	if err := json.Unmarshal(body, &got); err != nil {
+		t.Fatal(err)
+	}
+	for i, want := range []string{users, kept} {
+		r := got.Resources[i]
+		digest := sdkdecl.DigestOf([]byte(want))
+		if _, has := r["content"]; has || r["digest"] != digest || r["size"] != float64(len(want)) {
+			t.Errorf("%v goes with content %v, digest %v, size %v; want no content, %s and %d", r["id"], r["content"] != nil,
+				r["digest"], r["size"], digest, len(want))
+		}
+		if r["path"] == nil || r["mode"] != "0644" {
+			t.Errorf("%v lost its path or mode: %v", r["id"], r)
+		}
+	}
+	if got.Resources[2]["content"] != "tiny\n" || got.Resources[2]["digest"] != nil {
+		t.Errorf("a small file is not inline: %v", got.Resources[2])
+	}
+	if strings.Contains(string(body), users[:200]) || strings.Contains(string(body), kept[:200]) {
+		t.Error("the content of a file that went by digest is in the body")
+	}
+	if n := wireSizeOf(t, s); n > 100<<10 {
+		t.Errorf("the declaration is %d bytes on the wire, want under 100 KB", n)
+	}
+	// The objects to put are the contents, each under the hex of its digest, and a reference verifies against its object.
+	objects, err := s.objects()
+	if err != nil || len(objects) != 2 {
+		t.Fatalf("objects: %v %v", objects, err)
+	}
+	for _, want := range []string{users, kept} {
+		name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(want)))
+		if string(objects[name]) != want {
+			t.Errorf("the object %s does not hold the content", name[:12])
+		}
+	}
+	named, err := namedObjects(body)
+	if err != nil || len(named) != 2 {
+		t.Fatalf("the objects the body names: %v %v", named, err)
+	}
+	for name := range objects {
+		if _, ok := named[name]; !ok {
+			t.Errorf("the body does not name the object %s it was sent with", name[:12])
+		}
+		if _, err := hex.DecodeString(name); err != nil || len(name) != 64 {
+			t.Errorf("the object name %q is not 64 hex characters", name)
+		}
+	}
+}
+
+// 1696/AC2: a node that does not report the capability is sent every content inline, as before, and has no object put.
+func TestANodeWithoutTheCapabilityIsSentEveryContentInline(t *testing.T) { // 1696/AC2
+	users := contentOf("users", 252678)
+	s := sendable{Resources: []map[string]any{aFileResource("nats.bus-users", users)}}
+	body, err := s.Body()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got struct {
+		Resources []map[string]any `json:"resources"`
+	}
+	if err := json.Unmarshal(body, &got); err != nil {
+		t.Fatal(err)
+	}
+	if got.Resources[0]["content"] != users || got.Resources[0]["digest"] != nil || got.Resources[0]["size"] != nil {
+		t.Fatalf("a node that reads no references is not sent the content inline: %v", got.Resources[0]["digest"])
+	}
+	if objects, err := s.objects(); err != nil || len(objects) != 0 {
+		t.Fatalf("objects for a node that reads none: %v %v", len(objects), err)
+	}
+}
+
+// 1696/AC3: only a file that may travel by digest does: one of exactly 64 KiB, a sealed one, one filled with secrets, one written
+// into another file, and a directory stay inline, whatever the node reads.
+func TestOnlyAFileThatMayTravelByDigestDoes(t *testing.T) { // 1696/AC3
+	exact := contentOf("exact", 64<<10)
+	big := contentOf("big", 64<<10+1)
+	sealed := aFileResource("sealed", big)
+	sealed["sealed"], sealed["content"] = "box", ""
+	sealed["content"] = big // a sealed file's content is none; the body must pass it as it is, whatever it holds
+	secret := aFileResource("secret", big)
+	secret["secrets"] = map[string]any{"token": "x"}
+	into := aFileResource("into", big)
+	into["into"] = "/etc/other"
+	s := sendable{fileReferences: true, Resources: []map[string]any{
+		aFileResource("exact", exact), sealed, secret, into,
+		{"id": "dir", "type": "directory", "path": "/etc/d", "content": big},
+		aFileResource("big", big),
+	}}
+	body, err := s.Body()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got struct {
+		Resources []map[string]any `json:"resources"`
+	}
+	if err := json.Unmarshal(body, &got); err != nil {
+		t.Fatal(err)
+	}
+	for i, id := range []string{"exact", "sealed", "secret", "into", "dir"} {
+		if got.Resources[i]["id"] != id || got.Resources[i]["digest"] != nil || got.Resources[i]["content"] == nil {
+			t.Errorf("%s went by digest or lost its content: %v", id, got.Resources[i]["digest"])
+		}
+	}
+	if big := got.Resources[5]; big["id"] != "big" || big["digest"] == nil || big["content"] != nil {
+		t.Errorf("a plain file of 64 KiB and a byte did not go by digest: %v", big["digest"])
+	}
+}
+
+// A file given as bytes goes by digest on its decoded length, with the decoded bytes in the object.
+func TestAFileOfBytesTravelsByDigestOnItsDecodedLength(t *testing.T) {
+	raw := []byte(contentOf("png", 70<<10))
+	r := map[string]any{"id": "icon", "type": "file", "path": "/usr/share/icon.png", "bytes": stdBase64(raw)}
+	s := sendable{fileReferences: true, Resources: []map[string]any{r}}
+	objects, err := s.objects()
+	if err != nil || len(objects) != 1 {
+		t.Fatalf("%v %v", len(objects), err)
+	}
+	name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf(raw))
+	if string(objects[name]) != string(raw) {
+		t.Fatal("the object is not the decoded bytes")
+	}
+	if r["bytes"] == nil {
+		t.Fatal("the composed resource was changed in place")
+	}
+}
+
+type memObjects struct {
+	have   map[string][]byte
+	marked map[string]bool
+	puts   []string
+	fail   map[string]error
+}
+
+func (m *memObjects) Has(_ context.Context, name string) (bool, bool, error) {
+	_, ok := m.have[name]
+	return ok, m.marked[name], nil
+}
+func (m *memObjects) Put(_ context.Context, name string, c []byte) error {
+	if err := m.fail[name]; err != nil {
+		return err
+	}
+	m.puts = append(m.puts, name)
+	m.have[name] = c
+	delete(m.marked, name)
+	return nil
+}
+
+// 1696/AC4: an object already in the bucket is not put again, one absent is, and a put that fails stops the send, naming the object.
+func TestObjectsAreInTheBucketBeforeTheDeclarationIsSent(t *testing.T) { // 1696/AC4
+	objects := map[string][]byte{strings.Repeat("a", 64): []byte("one"), strings.Repeat("b", 64): []byte("two")}
+	b := &memObjects{have: map[string][]byte{strings.Repeat("a", 64): []byte("one")}}
+	if err := putObjects(t.Context(), b, objects); err != nil || len(b.puts) != 1 || b.puts[0] != strings.Repeat("b", 64) {
+		t.Fatalf("put %v (%v), want the one absent", b.puts, err)
+	}
+	// An object the sweep marked unnamed is put again, which erases the mark: a send relies on it.
+	b = &memObjects{have: map[string][]byte{strings.Repeat("a", 64): []byte("one"), strings.Repeat("b", 64): []byte("two")},
+		marked: map[string]bool{strings.Repeat("a", 64): true}}
+	if err := putObjects(t.Context(), b, objects); err != nil || len(b.puts) != 1 || b.puts[0] != strings.Repeat("a", 64) || b.marked[strings.Repeat("a", 64)] {
+		t.Fatalf("a marked object: put %v (%v) marked %v, want it put again and the mark gone", b.puts, err, b.marked)
+	}
+	b = &memObjects{have: map[string][]byte{}, fail: map[string]error{strings.Repeat("a", 64): errors.New("permissions violation")}}
+	err := putObjects(t.Context(), b, objects)
+	if err == nil || !strings.Contains(err.Error(), "aaaaaaaaaaaa") || !strings.Contains(err.Error(), "permissions violation") {
+		t.Fatalf("a failed put: %v", err)
+	}
+}
+
+// fallbackDelivery refuses the first declaration it is sent with the bucket's failure, as a bus that has not yet been given the
+// controller's grant does, and takes the second.
+type fallbackDelivery struct {
+	bodies [][]byte
+	first  error
+}
+
+func (f *fallbackDelivery) grant(context.Context, []readyNode) error { return nil }
+func (f *fallbackDelivery) declare(_ context.Context, _ readyNode, body []byte) (string, error) {
+	f.bodies = append(f.bodies, body)
+	if len(f.bodies) == 1 && f.first != nil {
+		return "", fmt.Errorf("x: %w: %w", errFilesNotPut, f.first)
+	}
+	return "digest", nil
+}
+
+// 1696/AC5: when the bucket cannot be written yet (the grant travels in the declaration of the machine holding the bus), that send goes
+// with every content inline and is made; a send that fails for another reason is not retried inline.
+func TestASendWhoseFilesCannotBePutGoesInlineOnce(t *testing.T) { // 1696/AC5
+	users := contentOf("users", 100<<10)
+	s := readyNode{node: "novox", declared: sendable{fileReferences: true, Resources: []map[string]any{aFileResource("nats.bus-users", users)}}}
+	f := &fallbackDelivery{first: errors.New("timeout")}
+	digests, err := deliver(t.Context(), f, "novox", []readyNode{s})
+	if err != nil || digests["novox"] != "digest" || len(f.bodies) != 2 {
+		t.Fatalf("%v %v %d", digests, err, len(f.bodies))
+	}
+	if strings.Contains(string(f.bodies[0]), users[:100]) || !strings.Contains(string(f.bodies[1]), users[:100]) {
+		t.Error("the first body should name the file by digest and the second carry it inline")
+	}
+	other := &fallbackDelivery{}
+	other.first = nil
+	failing := &failingDelivery{err: errors.New("the bus is down")}
+	if _, err := deliver(t.Context(), failing, "novox", []readyNode{s}); err == nil || failing.calls != 1 {
+		t.Fatalf("a failure of another kind: %v after %d send(s)", err, failing.calls)
+	}
+}
+
+type failingDelivery struct {
+	err   error
+	calls int
+}
+
+func (f *failingDelivery) grant(context.Context, []readyNode) error { return nil }
+func (f *failingDelivery) declare(context.Context, readyNode, []byte) (string, error) {
+	f.calls++
+	return "", f.err
+}
+
+// A bucket for the sweep.
+type sweepBucketFake struct {
+	objects map[string]time.Time // zero: named at the last look
+	deleted []string
+}
+
+func (b *sweepBucketFake) Objects(context.Context) ([]bucketObject, error) {
+	var out []bucketObject
+	for n, at := range b.objects {
+		out = append(out, bucketObject{Name: n, UnnamedSince: at})
+	}
+	return out, nil
+}
+func (b *sweepBucketFake) MarkUnnamed(_ context.Context, n string, at time.Time) error {
+	b.objects[n] = at
+	return nil
+}
+func (b *sweepBucketFake) ClearUnnamed(_ context.Context, n string) error {
+	b.objects[n] = time.Time{}
+	return nil
+}
+func (b *sweepBucketFake) Delete(_ context.Context, n string, markedAt time.Time) (bool, error) {
+	if at, ok := b.objects[n]; !ok || !at.Equal(markedAt) {
+		return false, nil // put again since it was judged: its mark is gone
+	}
+	delete(b.objects, n)
+	b.deleted = append(b.deleted, n)
+	return true, nil
+}
+
+// 1696/AC6: an object a latest declaration still names is not removed, however long; one unnamed is marked at the first look and removed
+// only when the mark is 24 hours old; one named again loses its mark.
+func TestAnObjectIsRemovedADayAfterNoLatestDeclarationNamesIt(t *testing.T) { // 1696/AC6
+	named, unnamed, back := strings.Repeat("1", 64), strings.Repeat("2", 64), strings.Repeat("3", 64)
+	t0 := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
+	b := &sweepBucketFake{objects: map[string]time.Time{named: t0.Add(-100 * time.Hour), unnamed: {}, back: t0.Add(-5 * time.Hour)}}
+	named3 := map[string]bool{named: true, back: true}
+	r, err := sweepDeclarationFiles(t.Context(), b, named3, t0)
+	if err != nil || r.Marked != 1 || r.Cleared != 2 || r.Removed != 0 || len(b.deleted) != 0 {
+		t.Fatalf("first look: %+v %v %v", r, err, b.deleted)
+	}
+	if !b.objects[named].IsZero() || !b.objects[back].IsZero() || !b.objects[unnamed].Equal(t0) {
+		t.Fatalf("marks after the first look: %v", b.objects)
+	}
+	// Still unnamed 23 hours 59 minutes later: kept. A day after the mark: removed. The named ones stay for ever.
+	if r, err = sweepDeclarationFiles(t.Context(), b, map[string]bool{named: true, back: true}, t0.Add(24*time.Hour-time.Minute)); err != nil ||
+		r.Removed != 0 || len(b.deleted) != 0 {
+		t.Fatalf("a minute short of a day: %+v %v %v", r, err, b.deleted)
+	}
+	if r, err = sweepDeclarationFiles(t.Context(), b, map[string]bool{named: true, back: true}, t0.Add(24*time.Hour)); err != nil ||
+		r.Removed != 1 || len(b.deleted) != 1 || b.deleted[0] != unnamed {
+		t.Fatalf("a day after: %+v %v %v", r, err, b.deleted)
+	}
+	if _, ok := b.objects[named]; !ok {
+		t.Fatal("an object a latest declaration names was removed")
+	}
+	// An object named again before the day is up loses its mark and is not removed.
+	b = &sweepBucketFake{objects: map[string]time.Time{unnamed: t0}}
+	if _, err := sweepDeclarationFiles(t.Context(), b, map[string]bool{unnamed: true}, t0.Add(23*time.Hour)); err != nil || !b.objects[unnamed].IsZero() {
+		t.Fatalf("named again: %v %v", b.objects, err)
+	}
+}
+
+// What a node's latest declaration names is read from the signed envelope as stored on the bus; an unreadable one is an error and
+// never "names none", which would mark an object a node needs.
+func TestTheObjectsALatestDeclarationNamesAreReadFromTheBus(t *testing.T) {
+	s := sendable{fileReferences: true, Resources: []map[string]any{aFileResource("a", contentOf("a", 70<<10))}}
+	body, _ := s.Body()
+	msg, _ := link.Envelope(body, []byte("sig"))
+	named, err := namedByLatest("novox", msg)
+	name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(contentOf("a", 70<<10))))
+	if err != nil || len(named) != 1 || !named[name] {
+		t.Fatalf("%v %v", named, err)
+	}
+	if none, err := namedByLatest("ace", []byte(`{"declaration":"e30=","signature":""}`)); err != nil || len(none) != 0 {
+		t.Fatalf("a declaration naming none: %v %v", none, err)
+	}
+	if _, err := namedByLatest("novox", []byte("not json")); err == nil {
+		t.Fatal("an unreadable latest declaration was read as naming none")
+	}
+}
+
+// 1698/AC2 (ADR 0486 §9): a declaration shaped like the control node's, for a node that reads references, stays under half of the
+// bus's largest message on the wire; the inline form of the same does not fit at all; and the same shape with 40 percent more resources
+// fails the test's own limit, so a change that eats the room fails in a pull request and not on the live mesh.
+func TestTheControlNodesDeclarationInTheReferenceFormStaysUnderHalfTheBusLimit(t *testing.T) { // 1698/AC2
+	const limit = 1048576
+	shape := sendable{fileReferences: true, Resources: controlNodeShape(1)}
+	n := wireSizeOf(t, shape)
+	t.Logf("the control node's shape in the reference form: %d bytes on the wire, %.1f%% of %d", n, 100*float64(n)/limit, limit)
+	if n > limit/2 {
+		t.Errorf("the reference form is %d bytes on the wire, over half (%d) of the bus's largest message", n, limit/2)
+	}
+	if inline := wireSizeOf(t, sendable{Resources: controlNodeShape(1)}); inline < limit*3/4 {
+		t.Errorf("the inline form is %d bytes, which is not the shape that was over the limit", inline)
+	}
+	grown := wireSizeOf(t, sendable{fileReferences: true, Resources: controlNodeShape(1.4)})
+	if grown <= limit/2 {
+		t.Errorf("the shape with 40%% more resources is %d bytes, which the test would let pass", grown)
+	}
+}
+
+func stdBase64(b []byte) string { return base64.StdEncoding.EncodeToString(b) }
+
+// A sweep that read an object's mark before a send put it again must not remove it: the object, now without the mark, is left.
+func TestASweepDoesNotRemoveAnObjectASendPutAgainAfterItWasJudged(t *testing.T) { // 1696/AC8
+	n := strings.Repeat("4", 64)
+	t0 := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
+	b := &sweepBucketFake{objects: map[string]time.Time{n: t0}}
+	judged := bucketObject{Name: n, UnnamedSince: t0}
+	b.objects[n] = time.Time{} // the send put it again: no mark
+	if gone, err := b.Delete(t.Context(), judged.Name, judged.UnnamedSince); err != nil || gone || len(b.deleted) != 0 {
+		t.Fatalf("an object put again was removed: %v %v %v", gone, err, b.deleted)
+	}
+}
+
+// One node's bucket is held by one act at a time in this process: the sweep waits for a send that has put its objects and not yet
+// published the declaration that names them (1696/AC9).
+func TestTheSweepOfANodesBucketWaitsForASendThatHasPutItsObjects(t *testing.T) { // 1696/AC9
+	unlock := lockFilesOf("lock-test-node")
+	got := make(chan struct{})
+	go func() {
+		defer lockFilesOf("lock-test-node")()
+		close(got)
+	}()
+	select {
+	case <-got:
+		t.Fatal("the sweep took the node's bucket while a send held it")
+	case <-time.After(50 * time.Millisecond):
+	}
+	other := lockFilesOf("lock-test-other-node") // another node's bucket is not held
+	other()
+	unlock()
+	select {
+	case <-got:
+	case <-time.After(2 * time.Second):
+		t.Fatal("the sweep did not take the node's bucket once the send was done")
+	}
+}
+
+// A put that fails send after send raises a condition; the first failure does not (the grant it waits for travels in that very
+// declaration), and a put that works clears it (1696/AC10).
+func TestAPutThatFailsSendAfterSendRaisesAConditionAndClearsWhenItWorks(t *testing.T) { // 1696/AC10
+	store := conditions.NewInMemory()
+	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
+		Teller: &conditions.Told{}, Now: func() time.Time { return time.Date(2026, 10, 21, 14, 0, 0, 0, time.UTC) }})
+	t.Cleanup(func() { k.Close(t.Context()) })
+	was := conditionsFrom
+	conditionsFrom = k
+	t.Cleanup(func() { conditionsFrom = was })
+	key := filesNotPutObservation("cond-node", "").Key()
+	open := func() bool {
+		_, found, err := conditions.ReadOne(t.Context(), store, key)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return found
+	}
+	boom := errors.New("permissions violation")
+	notePut(t.Context(), "cond-node", boom)
+	if open() {
+		t.Fatal("the first failure raised the condition")
+	}
+	notePut(t.Context(), "cond-node", boom)
+	if !open() {
+		t.Fatal("the second failure in a row did not raise the condition")
+	}
+	notePut(t.Context(), "cond-node", nil)
+	if open() {
+		t.Fatal("a put that worked did not clear the condition")
+	}
+	notePut(t.Context(), "cond-node", boom)
+	if open() {
+		t.Fatal("the count did not start again after a put that worked")
+	}
+}
+
+// aBusThatSays is a bus that keeps the declarations published to it and, at each, asks whether the node's bucket is held.
+type aBusThatSays struct {
+	link.Bus
+	published     chan string
+	heldAtPublish func() bool
+}
+
+func (b *aBusThatSays) PublishDeclaration(_ context.Context, node string, _ []byte) error {
+	if b.heldAtPublish != nil && !b.heldAtPublish() {
+		return errors.New("published while the node's bucket was not held")
+	}
+	b.published <- node
+	return errors.New("stop here: nothing is recorded")
+}
+
+type aSignerOfZeros struct{}
+
+func (aSignerOfZeros) Sign(context.Context, []byte) ([]byte, error) { return make([]byte, 64), nil }
+
+// whileHeld says whether the node's lock is held by someone right now, without keeping it.
+func whileHeld(node string) bool {
+	got := make(chan struct{})
+	go func() {
+		defer lockFilesOf(node)()
+		close(got)
+	}()
+	select {
+	case <-got:
+		return false
+	case <-time.After(100 * time.Millisecond):
+		// held: the goroutine stays waiting and takes and releases the lock once it is let go.
+		return true
+	}
+}
+
+// 1696/AC9: a send holds the node's lock from the put to the publication, so the sweep never removes an object between them. The
+// test goes through overTheBus.declare: while another holds the lock the declaration is not published, and at the publication the
+// lock is held.
+func TestASendHoldsTheNodesLockFromThePutToThePublication(t *testing.T) { // 1696/AC9
+	const node = "lock-send-node"
+	bus := &aBusThatSays{published: make(chan string, 1)}
+	bus.heldAtPublish = func() bool { return whileHeld(node) }
+	send := overTheBus{server: link.SendingOn(bus), signer: aSignerOfZeros{}}
+	release := lockFilesOf(node) // a sweep is judging this node's bucket
+	done := make(chan error, 1)
+	go func() {
+		_, err := send.declare(t.Context(), readyNode{node: node}, []byte(`{}`))
+		done <- err
+	}()
+	select {
+	case n := <-bus.published:
+		t.Fatalf("%s was sent its declaration while the sweep held the node's bucket", n)
+	case <-time.After(150 * time.Millisecond):
+	}
+	release()
+	select {
+	case <-bus.published:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the send did not go on once the sweep let go")
+	}
+	if err := <-done; err == nil || !strings.Contains(err.Error(), "nothing is recorded") {
+		t.Fatalf("the send did not reach the publication (a lock held at it): %v", err)
+	}
+}

cmd/mesh-controller/declaration_room.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// **A declaration is one bus message, and the bus's largest message is a cliff** (novox/hq issue 1689). On 2026-10-21
+// the control node's declaration passed it by 2 KB and nothing reached that node until two assignments were undone.
+// So the size is measured at every send: a warning condition from 75% of the limit (ADR 0486 §8, issue 1698: it was 90% while every file travelled inline), urgent over it, cleared when
+// the declaration is back under 75%. The limit is the connection's own, not a constant.
+
+// kindDeclarationNearLimit is a declaration at or near the bus's largest message.
+const kindDeclarationNearLimit = "declaration-near-limit"
+
+// declarationRoomSource is the condition's source: the sends.
+const declarationRoomSource = "sends"
+
+func declarationRoomObservation(node string, size, limit int64) conditions.Observation {
+	o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindDeclarationNearLimit,
+		Token: kindDeclarationNearLimit, Machine: node, Severity: conditions.Warning, Source: declarationRoomSource}
+	if size > limit {
+		o.Severity = conditions.Urgent
+	}
+	o.Summary = fmt.Sprintf("%s's declaration is %d bytes on the wire, %d%% of the bus's largest message of %d bytes",
+		node, size, size*100/limit, limit)
+	if size > limit {
+		o.Summary += ": over it, so it cannot be sent"
+	}
+	return o
+}
+
+func init() {
+	link.DeclarationMeasured = func(ctx context.Context, node string, size, limit int64) {
+		settleDeclarationRoom(ctx, conditionsFrom, node, size, limit)
+	}
+}
+
+// settleDeclarationRoom raises or clears the condition of a declaration's size. A nil keeper (a tool, not the
+// serving controller) says nothing.
+func settleDeclarationRoom(ctx context.Context, keeper *conditions.Keeper, node string, size, limit int64) {
+	if keeper == nil || limit <= 0 {
+		return
+	}
+	obs := declarationRoomObservation(node, size, limit)
+	if float64(size) < link.NearTheLimit*float64(limit) {
+		if _, err := keeper.Clear(ctx, obs.Key(), fmt.Sprintf("%s's declaration is %d bytes, under the limit's margin", node, size)); err != nil {
+			fmt.Printf("the condition %s could not be cleared: %v\n", obs.Key(), err)
+		}
+		return
+	}
+	fmt.Printf("NEAR THE LIMIT: %s\n", obs.Summary)
+	if _, err := keeper.Observe(ctx, obs); err != nil {
+		fmt.Printf("the condition %s could not be raised: %v\n", obs.Key(), err)
+	}
+}
+
+// compactKept writes the vault's export compact. The composition (the SDK's keptFile) writes it indented, and its 334
+// sealed secrets were 164,763 bytes of it against 138,474 compact: a fifth of the declaration's room, in whitespace,
+// base64-encoded again on the wire. Every reader parses JSON, so the spacing is nobody's. A resource that is not
+// the export, or whose content is not JSON, is left as it is. The resources are copied, never changed in place.
+func compactKept(resources []map[string]any) []map[string]any {
+	out := make([]map[string]any, len(resources))
+	for i, r := range resources {
+		out[i] = r
+		id, _ := r["id"].(string)
+		path, _ := r["path"].(string)
+		content, ok := r["content"].(string)
+		if !ok || (id != "kept" && !strings.HasSuffix(id, ".kept")) || !strings.HasSuffix(path, "/export.json") {
+			continue
+		}
+		var dst bytes.Buffer
+		if json.Compact(&dst, []byte(content)) != nil {
+			continue
+		}
+		if strings.HasSuffix(content, "\n") {
+			dst.WriteByte('\n')
+		}
+		if dst.String() == content {
+			continue
+		}
+		c := make(map[string]any, len(r))
+		for k, v := range r {
+			c[k] = v
+		}
+		c["content"] = dst.String()
+		out[i] = c
+	}
+	return out
+}

cmd/mesh-controller/declaration_room_test.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+)
+
+// The vault's export is sent compact (issue 1689): same document, no indentation, one rewrite.
+func TestTheVaultsExportIsSentCompactAndNothingElseIsTouched(t *testing.T) {
+	indented := "{\n  \"export\": 1,\n  \"kept\": [\n    {\n      \"name\": \"a\"\n    }\n  ]\n}\n"
+	other := "{\n  \"x\": 1\n}\n"
+	in := []map[string]any{
+		{"id": "mesh-vault.kept", "type": "file", "path": "/srv/vault/export.json", "mode": "0600", "content": indented},
+		{"id": "mesh-vault.config", "type": "file", "path": "/srv/vault/config.json", "content": other},
+		{"id": "other.kept", "type": "file", "path": "/srv/other/notes.json", "content": other},
+	}
+	out := compactKept(in)
+	want := "{\"export\":1,\"kept\":[{\"name\":\"a\"}]}\n"
+	if out[0]["content"] != want || out[0]["mode"] != "0600" || out[0]["path"] != "/srv/vault/export.json" {
+		t.Fatalf("the export is %q (%v), expected %q with its mode and path kept", out[0]["content"], out[0], want)
+	}
+	if out[1]["content"] != other || out[2]["content"] != other {
+		t.Fatalf("a file that is not the export was rewritten: %v %v", out[1], out[2])
+	}
+	if in[0]["content"] != indented {
+		t.Fatalf("the composed resource was changed in place")
+	}
+	if again := compactKept(out); again[0]["content"] != want {
+		t.Fatalf("compacting twice changed it: %q", again[0]["content"])
+	}
+}
+
+// From 75% of the bus's largest message (ADR 0486 §8, issue 1698) the declaration is a warning; over it, urgent; back under 75%, cleared.
+func TestADeclarationNearTheBusLimitRaisesAConditionAndClearsWhenSmaller(t *testing.T) {
+	store := conditions.NewInMemory()
+	k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
+		Teller: &conditions.Told{}, Now: func() time.Time { return time.Date(2026, 10, 21, 14, 0, 0, 0, time.UTC) }})
+	t.Cleanup(func() { k.Close(t.Context()) })
+	const limit = 1000
+	key := declarationRoomObservation("novox", 0, limit).Key()
+	read := func() (conditions.Condition, bool) {
+		t.Helper()
+		c, found, err := conditions.ReadOne(t.Context(), store, key)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return c, found
+	}
+
+	settleDeclarationRoom(t.Context(), k, "novox", 749, limit)
+	if _, found := read(); found {
+		t.Fatal("a declaration at 74.9% raised a condition")
+	}
+	settleDeclarationRoom(t.Context(), k, "novox", 750, limit)
+	c, found := read()
+	if !found || c.Severity != conditions.Warning || !strings.Contains(c.Summary, "750 bytes") ||
+		!strings.Contains(c.Summary, "1000") || c.Subject.Machine != "novox" {
+		t.Fatalf("at 75%% the condition is %v %+v", found, c)
+	}
+	settleDeclarationRoom(t.Context(), k, "novox", 1001, limit)
+	if c, _ := read(); c.Severity != conditions.Urgent {
+		t.Fatalf("over the limit the condition is %q, expected urgent", c.Severity)
+	}
+	settleDeclarationRoom(t.Context(), k, "novox", 700, limit)
+	if _, found := read(); found {
+		t.Fatal("the condition stayed after the declaration fell under 75%")
+	}
+}

cmd/mesh-controller/declaration_sweep.go

@@ -0,0 +1,273 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// **An object is removed a day after no node's latest declaration names it** (novox/hq ADR 0486 §3, issue 1696), so a node that is
+// asleep and receives its latest declaration later still finds every object that declaration names. What each node's latest
+// declaration names is read from the bus itself, the last message of that node's subject of the stream NODES: nothing else
+// is kept of it. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed);
+// found named again, that is erased; found unnamed for a day, it is removed. Each node has a bucket of its own, judged against that
+// node's latest declaration alone. Objects are immutable and regenerable from the
+// controller's state, so a removal loses nothing.
+
+// declarationUnnamed is the key of an object's metadata holding when it was first found named by no node's latest declaration.
+const declarationUnnamed = "unnamed-since"
+
+// sweepEvery is how often the bucket is judged: the removal is exact to within this.
+const sweepEvery = time.Hour
+
+// bucketObject is one object of the bucket as the sweep sees it.
+type bucketObject struct {
+	Name string
+	// UnnamedSince is when it was first found named by none, zero when it was named at the last look.
+	UnnamedSince time.Time
+}
+
+// sweptBucket is what the sweep needs of the bucket.
+type sweptBucket interface {
+	Objects(ctx context.Context) ([]bucketObject, error)
+	MarkUnnamed(ctx context.Context, name string, since time.Time) error
+	ClearUnnamed(ctx context.Context, name string) error
+	// Delete removes the object if it still carries the mark it was judged by, and says whether it did: a send that put it again
+	// since has erased the mark, and the object is left.
+	Delete(ctx context.Context, name string, markedAt time.Time) (bool, error)
+}
+
+// filesSweep says what a sweep did.
+type filesSweep struct{ Kept, Marked, Cleared, Removed int }
+
+// sweepDeclarationFiles judges every object against the names of the latest declarations: named ones are kept (and a
+// mark of being unnamed is erased), unnamed ones are marked at the first look and removed once the mark is a day old.
+func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[string]bool, now time.Time) (filesSweep, error) {
+	var r filesSweep
+	objects, err := bucket.Objects(ctx)
+	if err != nil {
+		return r, err
+	}
+	sort.Slice(objects, func(i, j int) bool { return objects[i].Name < objects[j].Name })
+	for _, o := range objects {
+		switch {
+		case named[o.Name]:
+			r.Kept++
+			if !o.UnnamedSince.IsZero() {
+				if err := bucket.ClearUnnamed(ctx, o.Name); err != nil {
+					return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
+				}
+				r.Cleared++
+			}
+		case o.UnnamedSince.IsZero():
+			if err := bucket.MarkUnnamed(ctx, o.Name, now); err != nil {
+				return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
+			}
+			r.Marked++
+		case now.Sub(o.UnnamedSince) >= broker.DeclarationFilesKeptFor:
+			gone, err := bucket.Delete(ctx, o.Name, o.UnnamedSince)
+			if err != nil {
+				return r, fmt.Errorf("the object %s: %w", o.Name[:12], err)
+			}
+			if gone {
+				r.Removed++
+			} else {
+				r.Kept++
+			}
+		default:
+			r.Kept++
+		}
+	}
+	return r, nil
+}
+
+// namedByLatest is the name of every object this latest declaration of a node names. A body that cannot be read is an error, never
+// "names none": an object the node still needs would be marked for removal.
+func namedByLatest(node string, signed []byte) (map[string]bool, error) {
+	var s link.Signed
+	if err := json.Unmarshal(signed, &s); err != nil {
+		return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
+	}
+	names, err := namedObjects(s.Declaration)
+	if err != nil {
+		return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
+	}
+	named := make(map[string]bool, len(names))
+	for name := range names {
+		named[name] = true
+	}
+	return named, nil
+}
+
+// latestDeclarations are the last signed declarations of the nodes, by node, as the stream NODES holds them. A node with none has
+// no entry.
+func latestDeclarations(ctx context.Context, conn *nats.Conn, nodes []string) (map[string][]byte, error) {
+	js, err := conn.JetStream(nats.Context(ctx))
+	if err != nil {
+		return nil, err
+	}
+	out := map[string][]byte{}
+	for _, node := range nodes {
+		msg, err := js.GetLastMsg("NODES", link.DeclareSubject(node), nats.Context(ctx))
+		if errors.Is(err, nats.ErrMsgNotFound) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("the latest declaration of %s: %w", node, err)
+		}
+		out[node] = msg.Data
+	}
+	return out, nil
+}
+
+// busSweptBucket is the bucket on the controller's connection.
+type busSweptBucket struct{ store jetstream.ObjectStore }
+
+func (b busSweptBucket) Objects(ctx context.Context) ([]bucketObject, error) {
+	infos, err := b.store.List(ctx)
+	if errors.Is(err, jetstream.ErrNoObjectsFound) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	out := make([]bucketObject, 0, len(infos))
+	for _, info := range infos {
+		o := bucketObject{Name: info.Name}
+		if at, err := time.Parse(time.RFC3339, info.Metadata[declarationUnnamed]); err == nil {
+			o.UnnamedSince = at
+		}
+		out = append(out, o)
+	}
+	return out, nil
+}
+
+func (b busSweptBucket) meta(ctx context.Context, name string, change func(map[string]string)) error {
+	info, err := b.store.GetInfo(ctx, name)
+	if err != nil {
+		return err
+	}
+	meta := info.ObjectMeta
+	kept := map[string]string{}
+	for k, v := range meta.Metadata {
+		kept[k] = v
+	}
+	change(kept)
+	meta.Metadata = kept
+	return b.store.UpdateMeta(ctx, name, meta)
+}
+
+func (b busSweptBucket) MarkUnnamed(ctx context.Context, name string, since time.Time) error {
+	return b.meta(ctx, name, func(m map[string]string) { m[declarationUnnamed] = since.UTC().Format(time.RFC3339) })
+}
+
+func (b busSweptBucket) ClearUnnamed(ctx context.Context, name string) error {
+	return b.meta(ctx, name, func(m map[string]string) { delete(m, declarationUnnamed) })
+}
+
+func (b busSweptBucket) Delete(ctx context.Context, name string, markedAt time.Time) (bool, error) {
+	info, err := b.store.GetInfo(ctx, name)
+	if errors.Is(err, jetstream.ErrObjectNotFound) {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	if at, perr := time.Parse(time.RFC3339, info.Metadata[declarationUnnamed]); perr != nil || !at.Equal(markedAt) {
+		return false, nil
+	}
+	return true, b.store.Delete(ctx, name)
+}
+
+// sweepingDeclarationFiles judges the bucket every sweepEvery for as long as this controller serves and holds the lease.
+func sweepingDeclarationFiles(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) {
+	tick := time.NewTicker(sweepEvery)
+	defer tick.Stop()
+	failing := ""
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-tick.C:
+		}
+		if epoch, err := theLease.epoch(ctx); err != nil || epoch == 0 {
+			continue
+		}
+		why := sweepOnce(ctx, inv, conn)
+		if why != failing {
+			if why != "" {
+				fmt.Printf("the bucket of declaration files cannot be swept: %s\n", why)
+			} else {
+				fmt.Printf("the bucket of declaration files is swept again\n")
+			}
+			failing = why
+		}
+	}
+}
+
+// sweepOnce judges the bucket of every node against that node's latest declaration, one bucket at a time under the node's lock, with
+// the declaration read again under it: a send that put an object and published its declaration is seen, and one that has not yet
+// is waiting for the lock.
+func sweepOnce(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) string {
+	nodes, err := inv.Nodes(ctx)
+	if err != nil {
+		return fmt.Sprintf("the nodes cannot be read: %v", err)
+	}
+	var problems []string
+	var total filesSweep
+	for _, n := range nodes {
+		r, err := sweepNode(ctx, conn, n.Name)
+		total.Kept, total.Marked, total.Cleared, total.Removed = total.Kept+r.Kept, total.Marked+r.Marked, total.Cleared+r.Cleared, total.Removed+r.Removed
+		if err != nil {
+			problems = append(problems, fmt.Sprintf("%s: %v", n.Name, err))
+		}
+	}
+	if total.Removed+total.Marked+total.Cleared > 0 {
+		fmt.Printf("the buckets of declaration files: %d object(s) kept, %d found unnamed, %d named again, %d removed\n",
+			total.Kept, total.Marked, total.Cleared, total.Removed)
+	}
+	return strings.Join(problems, "; ")
+}
+
+func sweepNode(ctx context.Context, conn *nats.Conn, node string) (filesSweep, error) {
+	working, cancel := context.WithTimeout(ctx, 2*time.Minute)
+	defer cancel()
+	js, err := jetstream.New(conn)
+	if err != nil {
+		return filesSweep{}, err
+	}
+	bucket, err := broker.DeclarationFilesBucketOf(node)
+	if err != nil {
+		return filesSweep{}, fmt.Errorf("the bucket of node %q: %w", node, err)
+	}
+	store, err := js.ObjectStore(working, bucket)
+	if errors.Is(err, jetstream.ErrBucketNotFound) {
+		return filesSweep{}, nil // a node never sent a reference has no bucket yet
+	}
+	if err != nil {
+		return filesSweep{}, fmt.Errorf("the bucket cannot be read: %w", err)
+	}
+	defer lockFilesOf(node)()
+	latest, err := latestDeclarations(working, conn, []string{node})
+	if err != nil {
+		return filesSweep{}, err
+	}
+	named := map[string]bool{}
+	if body, ok := latest[node]; ok {
+		if named, err = namedByLatest(node, body); err != nil {
+			return filesSweep{}, err
+		}
+	}
+	return sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now())
+}

cmd/mesh-controller/delivery.go

@@ -1204,7 +1204,7 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or
 			return "", fmt.Errorf("%s cannot be cloned for the group's check: %w", m.ID, err)
 		}
 		request.Check.Members = append(request.Check.Members, link.GroupMember{Owner: mp.Owner, Repo: mp.Repo,
-			Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths, Base: mp.Base})
+			Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths, Base: mp.Base, Trunk: mp.BaseSHA})
 	}
 	seat := buildSeatHeld(ctx)
 	ask, err := askOverOn(seat)

cmd/mesh-controller/delivery_integration_test.go

@@ -59,6 +59,18 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
 		}
 	}
 
+	// heardOf has this mesh's own view of the plans take in the plan as the bucket holds it. A verb of the
+	// controller (deliveryCommand, plansCommand) opens an inventory of its own and writes the plan there; this
+	// mesh's inventory learns of the write from the bucket's watch, a moment later, and a loaded machine makes the
+	// moment long enough for advanceHeld to read the plan as it was, still waiting for its word, and ask nothing
+	// (issue 1399). Reading the plan from the bucket puts it into the view at once, with no wait to bound.
+	heardOf := func(id string) {
+		t.Helper()
+		if _, err := inv.PlanByID(ctx, id); err != nil {
+			t.Fatal(err)
+		}
+	}
+
 	// No holder on record: the merge starts its walk, as before.
 	p := merge("c1aaaaaaaa", "modules/app/index.ts")
 	if p.Waiting() || len(*asked) != 1 {
@@ -90,6 +102,7 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
 	if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil {
 		t.Fatal("a walk was let go twice")
 	}
+	heardOf(p.ID)
 	advanceHeld(ctx, open)
 	if len(*asked) != 2 {
 		t.Fatalf("the word did not start the walk: %v", *asked)
@@ -119,6 +132,7 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin
 	if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil {
 		t.Fatal(err)
 	}
+	heardOf(p.ID)
 	advanceHeld(ctx, open)
 	got, _ = inv.PlanByID(ctx, p.ID)
 	if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 {

cmd/mesh-controller/forge_wait_test.go

@@ -68,7 +68,7 @@ func TestTheForgeNotAnsweringInTimeIsNotAskedAgain(t *testing.T) {
 // An ask that hangs past the bound is cut off at the bound, not at its own wait, and the failure still says it
 // waited: the receive loop is held no longer than the bound (novox/hq issue 486).
 func TestAnAskHangingPastTheBoundIsCutOffAtIt(t *testing.T) {
-	w := waitOut{bound: 200 * time.Millisecond, first: 10 * time.Millisecond, most: 50 * time.Millisecond,
+	w := waitOut{bound: 100 * time.Millisecond, first: 10 * time.Millisecond, most: 50 * time.Millisecond,
 		now: time.Now, sleep: sleepOrDone}
 	asked := 0
 	began := time.Now()

cmd/mesh-controller/held_moves.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// A move held for a person is asked of the person (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259).
+//
+// **A wait only a person can end is said to that person at once.** ADR 0377 holds a recorded module's move that
+// recreates a container or touches its data until a person pushes the machine. The walk said so in its note and
+// ended "done", `status` named the machine behind, and nobody was asked: on 2026-10-21 mailu's new build waited on
+// novox for 40 minutes behind a plan that read "done, 3 tier(s)", and on 2026-10-19 mssql waited on ace the same way.
+// Row S24 raises one condition per machine that has such a move, for the operator, with the answer **Release**: the
+// push of that machine, which the controller's own `push` verb performs on the operator's warrant (ADR 0259). It ends
+// when the machine no longer waits, whoever pushed it.
+//
+// A hold a person chose is not asked again — a module they set to `record` with a why, and the bus, whose step has
+// its own row (S17).
+
+// kindMovesHeld is S24's kind: machine.<node>.moves-held.
+const kindMovesHeld = "moves-held"
+
+// heldFacts is one machine and the moves of recorded modules that wait there for a person.
+type heldFacts struct {
+	node  string
+	moves []heldMove
+}
+
+// gatherHeldMoves is, per machine that holds at least one move no person chose to hold, all its held moves, by machine.
+func gatherHeldMoves(ctx context.Context, inv *inventory.Inventory) ([]heldFacts, error) {
+	nodes, err := inv.Nodes(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("the machines cannot be read: %w", err)
+	}
+	f, err := readMoveFacts(ctx, inv)
+	if err != nil {
+		return nil, err
+	}
+	var out []heldFacts
+	for _, n := range nodes {
+		moves, err := heldMovesOn(ctx, inv, f, n.Name)
+		if err != nil {
+			return nil, err
+		}
+		h := heldFacts{node: n.Name, moves: moves}
+		if len(h.asked()) > 0 {
+			out = append(out, h)
+		}
+	}
+	return out, nil
+}
+
+// asked are the moves no person chose to hold, and chosen the ones a person did (a module set to record with a why, the bus).
+func (h heldFacts) asked() []heldMove  { return h.byChoice(false) }
+func (h heldFacts) chosen() []heldMove { return h.byChoice(true) }
+
+func (h heldFacts) byChoice(chosen bool) []heldMove {
+	var out []heldMove
+	for _, m := range h.moves {
+		if m.Chosen == chosen {
+			out = append(out, m)
+		}
+	}
+	return out
+}
+
+// modules are the held moves' modules, in order, of all of them.
+func (h heldFacts) modules() []string { return moduleNames(h.moves) }
+
+func moduleNames(moves []heldMove) []string {
+	out := make([]string, 0, len(moves))
+	for _, m := range moves {
+		out = append(out, m.Module)
+	}
+	return out
+}
+
+// releaseAction is the one answer a held move offers: the push of the machine, approved by the operator. The push
+// names every module whose move waits there as one that may move its data: the operator's Approve is the word
+// ADR 0217 asks for, and the ask says so. Offered only where no person's own hold is on the machine too: a push
+// composes every recorded module at its current build, so it would send that move as well, which the ask never named
+// (ADR 0259 §6), and hold the whole machine if that module's data would move.
+func (h heldFacts) releaseAction() conditions.Action {
+	return conditions.Action{Label: "Release", Verb: "mesh-controller.push", Level: conditions.LevelApprove,
+		Arguments: map[string]string{"node": h.node, "move": strings.Join(moduleNames(h.asked()), ","), "why": "",
+			"cause": conditions.CauseOperatorAnswer}}
+}
+
+// heldWords are the plain words of a machine's held moves: which modules wait, why in general, and what an Approve does.
+func heldWords(h heldFacts) (headline, explanation, needs string, resolved string) {
+	names := namesWords(moduleNames(h.asked()), 3)
+	headline = clipWords(conditions.Capital(names)+" waits for you on "+h.node, conditions.HeadlineMax)
+	recreates, data := false, false
+	for _, m := range h.asked() {
+		switch {
+		case strings.Contains(m.Why, "recreates"):
+			recreates = true
+		default:
+			data = true
+		}
+	}
+	why := "would change what it runs"
+	switch {
+	case recreates && data:
+		why = "would restart a container or touch its data"
+	case recreates:
+		why = "would restart one of its containers"
+	case data:
+		why = "would touch its files or data"
+	}
+	explanation = clipWords(fmt.Sprintf("The new version of %s %s on %s, so it does not go out with its delivery; "+
+		"it waits for you. Release sends %s everything it is owed, these among it, and lets their data move where it "+
+		"has to. Nothing is sent until you answer.", names, why, h.node, h.node), conditions.ExplanationMax)
+	needs = "release them " + FromMeshMCPServer
+	if chosen := h.chosen(); len(chosen) > 0 {
+		// A push of the machine would send the move a person chose to hold as well: no answer is offered here.
+		explanation = clipWords(fmt.Sprintf("The new version of %s %s on %s, so it does not go out with its delivery; "+
+			"it waits for you. %s is also held there by your own choice, and a push of %s would send that too, so "+
+			"nothing is offered here: the push is yours at the controller's terminal.",
+			names, why, h.node, conditions.Capital(namesWords(moduleNames(chosen), 3)), h.node), conditions.ExplanationMax)
+		needs = fmt.Sprintf("push %s at the controller's terminal, once you have decided about %s.", h.node,
+			namesWords(moduleNames(chosen), 3))
+	}
+	resolved = "Resolved: " + names + " is no longer waiting on " + h.node
+	return
+}
+
+// watchMovesHeld is S24: a machine that has moves of recorded modules waiting for a person, said to the operator with the
+// answer that sends them. The condition ends when the machine no longer waits: it is no longer among the facts.
+func watchMovesHeld(f *signalFacts) []conditions.Observation {
+	var out []conditions.Observation
+	for _, h := range f.held {
+		var said []string
+		for _, m := range h.moves {
+			said = append(said, fmt.Sprintf("%s %s → %s %s", m.Module, short(m.From), short(m.To), m.Why))
+		}
+		headline, explanation, needs, resolved := heldWords(h)
+		if c := h.chosen(); len(c) > 0 {
+			said = append(said, "held by a person's own choice: "+strings.Join(moduleNames(c), ", "))
+		}
+		var actions []conditions.Action
+		if len(h.chosen()) == 0 {
+			actions = []conditions.Action{h.releaseAction()}
+		}
+		out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: h.node, Kind: kindMovesHeld,
+			Machine: h.node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
+			Summary: fmt.Sprintf("%s has %d move(s) of recorded modules waiting for a person's push: %s — `push %s` sends them",
+				h.node, len(h.moves), strings.Join(h.modules(), ", "), h.node),
+			Said:     strings.Join(said, "; "),
+			Headline: headline, Explanation: explanation, Needs: needs, Resolved: resolved,
+			Actions: actions})
+	}
+	return out
+}
+
+// planHeld is, for a plan that ended or is running, the moves its modules carried that a machine still holds for a
+// person: "module on machine", sorted. The plan's own record says the move waited when it was judged
+// (PlanModule.Recorded); the machines say whether it still does, and for the build this plan carried, so a plan older
+// than the build now held is not named.
+func planHeld(p inventory.Plan, held []heldFacts) []string {
+	var out []string
+	for _, h := range held {
+		for _, m := range h.moves {
+			s := p.Modules[m.Module]
+			if s == nil || !strings.HasPrefix(s.Recorded[h.node], "waits:") || !sameCommit(s.Commit, m.To) {
+				continue
+			}
+			out = append(out, m.Module+" on "+h.node)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+// heldSaid is what a plan's line adds when it carries a move that a machine holds for a person (novox/hq issue 1732): a
+// plan whose module waits for a push is not plainly "done".
+func heldSaid(p inventory.Plan, held []heldFacts) string {
+	waiting := planHeld(p, held)
+	if len(waiting) == 0 {
+		return ""
+	}
+	return fmt.Sprintf(" — %d held for a person: %s (the operator is asked)", len(waiting), strings.Join(waiting, ", "))
+}
+
+// heldNow reads the machines' held moves for a plan's line, nil when they cannot be read: the line then reads as before.
+func heldNow(ctx context.Context, inv *inventory.Inventory) []heldFacts {
+	held, err := gatherHeldMoves(ctx, inv)
+	if err != nil {
+		return nil // empty-on-error: a plan's line then reads as it did before; the watchdog (S24) says it cannot read
+	}
+	return held
+}

cmd/mesh-controller/held_moves_integration_test.go

@@ -0,0 +1,92 @@
+//go:build integration
+
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+func inventoryUpgrade(why string) inventory.Upgrade { return inventory.Upgrade{Why: why, By: "jochen"} }
+
+func mustMoveFacts(t *testing.T, rm *recordedMesh) moveFacts {
+	t.Helper()
+	f, err := readMoveFacts(t.Context(), rm.open.inventory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return f
+}
+
+// 1732/AC8 (integration): a mesh running mailu at c1 on two machines, whose registered c2 recreates a container, is read
+// as two machines each holding mailu's move for a person; a bundle-only c3 on top changes nothing of it; after a machine
+// is sent c2 (as a push records it) it is no longer among them, and plans that carried c2 say nothing more of it.
+func TestMachinesHoldingAMoveForAPersonAreReadFromTheRecords(t *testing.T) {
+	rm := aRecordedMesh(t, aMailu(true))
+	c2 := aMailu(true)
+	resourceOf(&c2, "imap")["image"] = "registry.invalid/imap@sha256:2"
+	rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
+	ctx := t.Context()
+	held, err := gatherHeldMoves(ctx, rm.open.inventory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(held) != 2 || held[0].node != "anchor" || held[1].node != "laptop" || len(held[0].moves) != 1 ||
+		held[0].moves[0].Module != "mailu" || held[0].moves[0].To != "c2" || !strings.Contains(held[0].moves[0].Why, "recreates") {
+		t.Fatalf("held %+v", held)
+	}
+	if err := rm.open.inventory.RecordSent(ctx, nodeID(t, rm.open, "anchor"), "d-anchor-c2", map[string]string{"mailu": "c2"}); err != nil {
+		t.Fatal(err)
+	}
+	held, err = gatherHeldMoves(ctx, rm.open.inventory)
+	if err != nil || len(held) != 1 || held[0].node != "laptop" {
+		t.Fatalf("after anchor was sent c2: %+v (%v)", held, err)
+	}
+}
+
+// 1732/AC9 (integration): a module a person set to record is no ask: they chose the hold.
+func TestAHoldAPersonChoseIsNotAsked(t *testing.T) {
+	rm := aRecordedMesh(t, aMailu(false))
+	c2 := aMailu(false)
+	c2.Bundles[0].Digest = "sha256:bbb"
+	rm.register(t, c2, "c2", time.Now().Add(-time.Minute))
+	if err := rm.open.inventory.SetUpgradeOf(t.Context(), "mailu", inventoryUpgrade("the migration week")); err != nil {
+		t.Fatal(err)
+	}
+	held, err := gatherHeldMoves(t.Context(), rm.open.inventory)
+	if err != nil || len(held) != 0 {
+		t.Fatalf("a machine holding only a person's own hold is asked: %+v (%v)", held, err)
+	}
+	if moves, _ := heldMovesOn(t.Context(), rm.open.inventory, mustMoveFacts(t, rm), "anchor"); len(moves) != 1 || !moves[0].Chosen {
+		t.Fatalf("status still names it, marked as chosen: %+v", moves)
+	}
+}
+
+// 1742/AC4 (integration): a registered build naming an image of the store that the store lacks is read as a missing
+// image on each machine that runs the module; one the store holds is not; and what the store holds is not asked twice.
+func TestARegisteredBuildNamingAnImageTheStoreLacksIsRead(t *testing.T) {
+	m := aMailu(true)
+	resourceOf(&m, "imap")["image"] = "artifact-store://mailu/imap@sha256:" + strings.Repeat("e", 64)
+	rm := aRecordedMesh(t, m)
+	asked := 0
+	lacking := func(_ context.Context, reference string) (bool, error) { asked++; return false, nil }
+	imagesHeld.Lock()
+	imagesHeld.held = map[string]bool{}
+	imagesHeld.Unlock()
+	got, err := gatherMissingImages(t.Context(), rm.open.inventory, lacking)
+	if err != nil || len(got) != 2 || got[0].Module != "mailu" || got[0].Node != "anchor" || got[1].Node != "laptop" {
+		t.Fatalf("missing %+v (%v)", got, err)
+	}
+	holding := func(_ context.Context, reference string) (bool, error) { asked++; return true, nil }
+	if got, err := gatherMissingImages(t.Context(), rm.open.inventory, holding); err != nil || len(got) != 0 {
+		t.Fatalf("an image the store holds is missing: %+v (%v)", got, err)
+	}
+	before := asked
+	if _, err := gatherMissingImages(t.Context(), rm.open.inventory, lacking); err != nil || asked != before {
+		t.Fatalf("an image the store was found to hold was asked about again (%d, %v)", asked-before, err)
+	}
+}

cmd/mesh-controller/held_moves_test.go

@@ -0,0 +1,237 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/asks"
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// **A move held for a person is asked of the person** (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259).
+
+// aHeldMailu is novox holding mailu's new build because it would recreate a container: the walk's own words.
+func aHeldMailu() heldFacts {
+	return heldFacts{node: "novox", moves: []heldMove{{Module: "mailu", From: "b2f5ebb2aaaa", To: "3259765dbbbb",
+		Why: "waits: recreates 1 of mailu's 12 container(s) (with a new image: automx)"}}}
+}
+
+func heldConditionOn(f *signalFacts) conditions.Condition {
+	o := watchMovesHeld(f)[0]
+	return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
+		Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
+}
+
+// 1732/AC1: a machine holding a move of a recorded module for a person raises one condition for the operator, with
+// Release as its only answer: the machine's push, approved, naming the modules whose moves wait there.
+func TestAHeldMoveIsRaisedWithReleaseAsItsOnlyAction(t *testing.T) {
+	f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
+	f.held = []heldFacts{aHeldMailu()}
+	got := watchMovesHeld(f)
+	if len(got) != 1 {
+		t.Fatalf("%+v", got)
+	}
+	o := got[0]
+	if o.Kind != "moves-held" || o.Scope != conditions.ScopeMachine || o.ID != "novox" || o.Severity != conditions.Warning ||
+		o.Resolver != conditions.ResolverOperator || o.Key() != "machine.novox.moves-held" {
+		t.Fatalf("kind, scope, subject, severity or resolver: %+v (%s)", o, o.Key())
+	}
+	if len(o.Actions) != 1 {
+		t.Fatalf("actions: %+v", o.Actions)
+	}
+	a := o.Actions[0]
+	if a.Label != "Release" || a.Verb != "mesh-controller.push" || a.Level != conditions.LevelApprove ||
+		a.Arguments["node"] != "novox" || a.Arguments["move"] != "mailu" || a.Arguments["cause"] != conditions.CauseOperatorAnswer {
+		t.Fatalf("the action is not the push of novox: %+v", a)
+	}
+	if _, takes := a.Arguments["why"]; !takes {
+		t.Fatal("the push takes a why, which the asker fills in with the operator's answer")
+	}
+	for _, want := range []string{"mailu", "novox"} {
+		if !strings.Contains(o.Headline+o.Explanation, want) {
+			t.Errorf("the words do not name %q: %q / %q", want, o.Headline, o.Explanation)
+		}
+	}
+	if !strings.Contains(o.Said, "recreates 1 of mailu's 12") {
+		t.Errorf("the evidence does not carry the walk's own words: %q", o.Said)
+	}
+	if w := (conditions.Observation{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs, Resolved: o.Resolved}); w.Headline == "" || len(o.Headline) > conditions.HeadlineMax {
+		t.Errorf("headline: %q", o.Headline)
+	}
+}
+
+// 1732/AC2: a machine with nothing held raises nothing, so the condition ends the look after the move is sent.
+func TestAMachineHoldingNothingRaisesNoHeldMoveCondition(t *testing.T) {
+	f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
+	if got := watchMovesHeld(f); len(got) != 0 {
+		t.Fatalf("%+v", got)
+	}
+}
+
+// 1732/AC3: the condition is asked on a channel at the approve level in words that say what Release does, and the
+// operator's warrant calls the controller's own push with the machine and the modules, once.
+func TestAHeldMovesReleaseIsAskedAndAWarrantPushesTheMachine(t *testing.T) {
+	f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
+	f.held = []heldFacts{aHeldMailu()}
+	c := heldConditionOn(f)
+	r := newAskerRig(t)
+	r.open = []conditions.Condition{c}
+	if err := r.a.reconcile(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	sent := r.asksSent(t)
+	if len(sent) != 1 || len(sent[0].Options) != 1 {
+		t.Fatalf("asked %+v", sent)
+	}
+	o := sent[0].Options[0]
+	if o.Label != "Release" || o.Level != asks.Approve || o.Does != "the held builds are sent to novox" {
+		t.Errorf("the option is %+v", o)
+	}
+	if err := sent[0].Check(r.now); err != nil {
+		t.Errorf("the ask is refused as made: %v", err)
+	}
+	w := r.warrantFor(t, c.Key, "Release")
+	answerWith(t, r, w)
+	answerWith(t, r, w)
+	if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.push@ ") ||
+		!strings.Contains(r.called[0], `"node":"novox"`) || !strings.Contains(r.called[0], `"move":"mailu"`) ||
+		!strings.Contains(r.called[0], `"cause":"`+conditions.CauseOperatorAnswer+`"`) ||
+		!strings.Contains(r.called[0], "chose Release") {
+		t.Fatalf("called %v", r.called)
+	}
+}
+
+// 1732/AC4: the controller may call its own push and upgrade verbs on a warrant: the grant names them.
+func TestTheControllerMayActOnAWarrantForPushAndUpgrade(t *testing.T) {
+	for _, verb := range []string{"push", "upgrade"} {
+		granted := false
+		for _, v := range broker.VerbsTheControllerActsOnAWarrant {
+			granted = granted || (v.Seat == broker.ControllerSeat && v.Verb == verb)
+		}
+		if !granted {
+			t.Errorf("the controller may not act on a warrant for %s", verb)
+		}
+	}
+}
+
+func aDoneMailuPlan() inventory.Plan {
+	return inventory.Plan{ID: "plan-1792585970446304628", Repository: "novox/mesh-catalog", State: inventory.PlanDone,
+		Tiers: [][]string{{"a"}, {"b"}, {"mailu"}},
+		Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", Commit: "3259765dbbbbcccc",
+			Recorded: map[string]string{"novox": "waits: recreates 1 of mailu's 12 container(s)"}}}}
+}
+
+// 1732/AC5: a done plan that carries a move a machine still holds for a person says so on its line; once the machine
+// no longer holds it, or holds another build of the module, the line says nothing more.
+func TestADonePlanSaysWhatItHoldsForAPerson(t *testing.T) {
+	p := aDoneMailuPlan()
+	line := planLine(p, time.Date(2026, 10, 21, 16, 0, 0, 0, time.UTC)) + heldSaid(p, []heldFacts{aHeldMailu()})
+	if !strings.Contains(line, "done, 3 tier(s)") || !strings.Contains(line, "1 held for a person: mailu on novox") {
+		t.Errorf("the line: %q", line)
+	}
+	if said := heldSaid(p, nil); said != "" {
+		t.Errorf("a plan whose moves were sent says %q", said)
+	}
+	other := aHeldMailu()
+	other.moves[0].To = "9999999ffff"
+	if said := heldSaid(p, []heldFacts{other}); said != "" {
+		t.Errorf("a plan older than the build now held says %q", said)
+	}
+	p.Modules["mailu"].Recorded = map[string]string{"novox": recordedRollsWhy}
+	if said := heldSaid(p, []heldFacts{aHeldMailu()}); said != "" {
+		t.Errorf("a move the plan sent says %q", said)
+	}
+}
+
+// 1732/AC6: the builds held after a failed release are asked of the operator: the held-release condition offers
+// Release, which calls the controller's own `upgrade` with release-backlog and nothing else, once, on a warrant.
+func TestTheBuildsHeldAfterAFailedReleaseAreAskedAndReleasedOnAWarrant(t *testing.T) {
+	saved := backlogNow
+	t.Cleanup(func() { backlogNow = saved })
+	backlogNow.held = "release-1792595552499982311 failed (failed on novox); what waits is released again by a person"
+	backlogNow.waiting = map[string][]inventory.CarriedMove{"novox": {{Module: "indexer"}, {Module: "nextcloud"}}}
+	got := backlogObservation()
+	if len(got) != 1 || len(got[0].Actions) != 1 {
+		t.Fatalf("raised %+v", got)
+	}
+	a := got[0].Actions[0]
+	if a.Label != "Release" || a.Verb != "mesh-controller.upgrade" || a.Level != conditions.LevelApprove ||
+		a.Arguments["release-backlog"] != "true" || a.Arguments["cause"] != conditions.CauseOperatorAnswer {
+		t.Fatalf("the action: %+v", a)
+	}
+	o := got[0]
+	c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
+		Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
+	r := newAskerRig(t)
+	r.open = []conditions.Condition{c}
+	if err := r.a.reconcile(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	sent := r.asksSent(t)
+	if len(sent) != 1 || len(sent[0].Options) != 1 || sent[0].Options[0].Does != "the builds held after the failed delivery are released" {
+		t.Fatalf("asked %+v", sent)
+	}
+	w := r.warrantFor(t, c.Key, "Release")
+	answerWith(t, r, w)
+	answerWith(t, r, w)
+	if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.upgrade@ ") ||
+		!strings.Contains(r.called[0], `"release-backlog":"true"`) || !strings.Contains(r.called[0], "chose Release") {
+		t.Fatalf("called %v", r.called)
+	}
+}
+
+// 1732/AC7: a plan whose send waits for a gate while the builds are held after a failed release reads as waiting for
+// a person to release them, not "building ... LATE"; with nothing held it still reads as the wait it is.
+func TestAPlanWaitingForAGateBehindAFailedReleaseIsNotLate(t *testing.T) {
+	saved := backlogNow
+	t.Cleanup(func() { backlogNow = saved })
+	now := time.Date(2026, 10, 21, 18, 50, 0, 0, time.UTC)
+	p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-sdk", Commit: "c0ffee00", State: inventory.PlanBuilding,
+		Tiers: [][]string{{"indexer"}}, TierEntered: now.Add(-100 * time.Minute), Created: now.Add(-110 * time.Minute),
+		Modules: map[string]*inventory.PlanModule{"indexer": {State: "built"}},
+		Note:    refusalNote(0, errors.New("a build waits there for its gate: novox"))}
+	backlogNow.held = ""
+	if line := planLine(p, now); !strings.Contains(line, "a build waits there for its gate: novox") {
+		t.Errorf("the wait is not said: %q", line)
+	}
+	backlogNow.held = "release-1 failed (failed on novox); what waits is released again by a person"
+	line := planLine(p, now)
+	if strings.Contains(line, "LATE") || strings.Contains(line, "building") ||
+		!strings.Contains(line, "until a person releases it") {
+		t.Errorf("a plan held behind a failed release reads %q", line)
+	}
+}
+
+// 1732/AC10: a machine that also holds a move a person chose to hold gets no Release, since a push would send that move
+// too; the condition names both and says the push is the controller's terminal. A machine with only asked moves keeps it.
+func TestAMachineAlsoHoldingAPersonsOwnHoldIsOfferedNoRelease(t *testing.T) {
+	f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
+	h := aHeldMailu()
+	h.moves = append(h.moves, heldMove{Module: "nftables", From: "n1aaaa", To: "n2bbbb",
+		Why: "waits: a person set it to record, and holds every move of it until they push", Chosen: true})
+	f.held = []heldFacts{h}
+	got := watchMovesHeld(f)
+	if len(got) != 1 || len(got[0].Actions) != 0 {
+		t.Fatalf("%+v", got)
+	}
+	o := got[0]
+	text := o.Headline + o.Explanation + o.Needs + o.Summary + o.Said
+	for _, want := range []string{"mailu", "nftables", "terminal", "push novox"} {
+		if !strings.Contains(text, want) {
+			t.Errorf("does not say %q: %q", want, text)
+		}
+	}
+	w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
+	if why, ok := conditions.PlainWords(w, "novox"); !ok {
+		t.Errorf("not plain: %s", why)
+	}
+	f.held = []heldFacts{aHeldMailu()}
+	if got := watchMovesHeld(f); len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["move"] != "mailu" {
+		t.Fatalf("a machine with only asked moves: %+v", got)
+	}
+}

cmd/mesh-controller/image_missing.go

@@ -0,0 +1,245 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"sort"
+	"strings"
+	"sync"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"github.com/novox/mesh-controller/internal/artifacts"
+	"github.com/novox/mesh-controller/internal/conditions"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// No declaration is sent that names an image the artifact store does not hold (novox/hq issue 1742).
+//
+// On 2026-10-21 at 17:13 a send to novox named `mailu/automx@sha256:c62150b8…`; the build record keeps that digest
+// ("recent-build") and the store did not hold it. The node-engine removed the running container, then could not pull
+// the replacement, and every later apply on novox failed for the next hour while the mail clients' autoconfiguration
+// had no backend. The node-engine's half (pull first, replace after) is its own change; this is the controller's: **a
+// module whose image the store lacks is left out of the declaration**, as a module whose setting nobody gave is
+// (ADR 0163, rule 6) — the node-engine keeps what it has of it and touches none of its containers — and **it is said**,
+// row S25: a condition naming the module, the machine and the image, until the store holds the image again (a new build
+// of the module, or the copy put back).
+//
+// Judged only for an image the store's own address names, from the send's own resources; an image of another registry
+// is no business of the store's. A store that cannot be asked judges nothing: a send is not refused for a question
+// that did not answer.
+
+// kindImageMissing is S25's kind: module.<module>.<machine>.image-missing.
+const kindImageMissing = "image-missing"
+
+// missingImage is a module held out of a machine's declaration because the store lacks an image it names.
+type missingImage struct {
+	Node, Module, Image string
+}
+
+// storeHolds is whether the store holds the manifest an image reference at its address names.
+type storeHolds func(ctx context.Context, reference string) (bool, error)
+
+// imageInStore is the artifact-store reference of an image the store's address names, "" for any other.
+func imageInStore(image, address string) string {
+	if address == "" || !strings.HasPrefix(image, address+"/") || !strings.Contains(image, "@sha256:") {
+		return ""
+	}
+	reference := catalogue.Recorded(image)
+	if _, ours := catalogue.InArtifactStore(reference); !ours {
+		return ""
+	}
+	return reference
+}
+
+// holdBackMissingImages leaves out of the declaration every module that names an image the store does not hold,
+// and answers what it left out. The module's resources go, it joins LeftOut with the reason, and its build stays the
+// one the machine was last sent (`was`).
+func holdBackMissingImages(ctx context.Context, node string, s *sendable, address string, holds storeHolds,
+	was map[string]string) []missingImage {
+	lacks := map[string]bool{} // reference → the store does not hold it; absent: not asked, or no answer
+	asked := map[string]bool{}
+	lacking := map[string]string{} // module → the image
+	for _, r := range s.Resources {
+		if fmt.Sprint(r["type"]) != "container" {
+			continue
+		}
+		image, _ := r["image"].(string)
+		reference := imageInStore(image, address)
+		module := s.moduleOf[fmt.Sprint(r["id"])]
+		if reference == "" || module == "" || lacking[module] != "" {
+			continue
+		}
+		if !asked[reference] {
+			asked[reference] = true
+			held, err := holds(ctx, reference)
+			// empty-on-error: the store did not answer, so nothing is held back for a question unanswered
+			lacks[reference] = err == nil && !held
+		}
+		if lacks[reference] {
+			lacking[module] = image
+		}
+	}
+	if len(lacking) == 0 {
+		return nil
+	}
+	kept := s.Resources[:0:0]
+	for _, r := range s.Resources {
+		if _, out := lacking[s.moduleOf[fmt.Sprint(r["id"])]]; !out {
+			kept = append(kept, r)
+		}
+	}
+	s.Resources = kept
+	if s.leftOutWhy == nil {
+		s.leftOutWhy = map[string]string{}
+	}
+	var found []missingImage
+	for module, image := range lacking {
+		s.leftOutWhy[module] = "its image " + image + " is not in the artifact store"
+		if !slices.Contains(s.LeftOut, module) {
+			s.LeftOut = append(s.LeftOut, module)
+		}
+		if s.Builds != nil {
+			if prior, sent := was[module]; sent {
+				s.Builds[module] = prior
+			} else {
+				delete(s.Builds, module)
+			}
+		}
+		found = append(found, missingImage{Node: node, Module: module, Image: image})
+	}
+	sort.Strings(s.LeftOut)
+	sort.Slice(found, func(i, j int) bool { return found[i].Module < found[j].Module })
+	return found
+}
+
+// withoutMissingImages is composeSend's last step: the store asked about the images the declaration names, the modules
+// whose image it lacks left out and kept in the ledger S25 reads. A store that cannot be found or asked leaves the
+// declaration as composed.
+func withoutMissingImages(ctx context.Context, open *stores, node string, s sendable) sendable {
+	inv := open.inventory
+	shelf, err := inv.Catalogue(ctx)
+	if err != nil {
+		return s
+	}
+	address, err := artifactStoreAddress(ctx, inv, shelf, node)
+	if err != nil || address == "" {
+		return s
+	}
+	sent, known, err := inv.SentBuilds(ctx, node)
+	if err != nil || !known {
+		sent = nil
+	}
+	store := artifacts.Store{Address: address}
+	found := holdBackMissingImages(ctx, node, &s, address, store.HoldsManifest, sent)
+	for _, m := range found {
+		fmt.Printf("%s: %s is left out of the declaration: its image %s is not in the artifact store\n", node, m.Module, m.Image)
+	}
+	return s
+}
+
+// imageFacts are the images S25 raises: a module whose registered build names an image the store lacks, on each machine
+// that runs it.
+type imageFacts struct {
+	missingImage
+}
+
+// imagesHeld remembers the images the store was found to hold: a stored image is named by its digest and stays what it
+// is, so it is asked about once; one it lacks is asked again at every look.
+var imagesHeld = struct {
+	sync.Mutex
+	held map[string]bool
+}{held: map[string]bool{}}
+
+// gatherMissingImages is every machine's module whose registered build — the one a send would carry — names an image of
+// the artifact store that the store does not hold. Read from the catalogue and the store, never from what a send did:
+// a push is a command of its own, and the serving controller's watchdog hears nothing of it.
+func gatherMissingImages(ctx context.Context, inv *inventory.Inventory, holds storeHolds) ([]imageFacts, error) {
+	shelf, err := inv.Catalogue(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
+	}
+	var out []imageFacts
+	for _, module := range sortedKeysOf(modulesOf(shelf)) {
+		for _, r := range shelf[module].Resources {
+			if fmt.Sprint(r["type"]) != "container" {
+				continue
+			}
+			image, _ := r["image"].(string)
+			if _, ours := catalogue.InArtifactStore(image); !ours || !strings.Contains(image, "@sha256:") {
+				continue
+			}
+			imagesHeld.Lock()
+			known := imagesHeld.held[image]
+			imagesHeld.Unlock()
+			if !known {
+				held, err := holds(ctx, image)
+				if err != nil {
+					return nil, fmt.Errorf("the artifact store cannot be asked about %s: %w", image, err)
+				}
+				if held {
+					imagesHeld.Lock()
+					imagesHeld.held[image] = true
+					imagesHeld.Unlock()
+					continue
+				}
+			} else {
+				continue
+			}
+			running, err := inv.Running(ctx, module)
+			if err != nil {
+				return nil, err
+			}
+			for _, node := range running {
+				out = append(out, imageFacts{missingImage{Node: node, Module: module, Image: image}})
+			}
+		}
+	}
+	return out, nil
+}
+
+// modulesOf is the shelf's modules as a map of names, for sortedKeysOf.
+func modulesOf(shelf map[string]catalogue.Manifest) map[string]string {
+	out := make(map[string]string, len(shelf))
+	for m := range shelf {
+		out[m] = m
+	}
+	return out
+}
+
+// watchMissingImages is S25: a module left out of a machine's declaration because the store lacks its image, said at
+// once. The condition ends when the store holds the image, or a send names another.
+func watchMissingImages(f *signalFacts) []conditions.Observation {
+	var out []conditions.Observation
+	for _, m := range f.images {
+		out = append(out, conditions.Observation{Scope: conditions.ScopeModule, ID: m.Module + "." + m.Node, Kind: kindImageMissing,
+			Machine: m.Node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
+			Summary: fmt.Sprintf("%s is left out of every send to %s: its declaration names an image that the artifact store does "+
+				"not hold (the evidence names it); the running container is untouched", m.Module, m.Node),
+			Said:     "image " + m.Image + " is not in the store",
+			Headline: clipWords(conditions.Capital(m.Module)+" is not updated on "+m.Node, conditions.HeadlineMax),
+			Explanation: clipWords(fmt.Sprintf("The new version of %s names a container image that the mesh's store does not "+
+				"hold, so it was not sent to %s. What runs there is untouched, and the rest of %s is sent as usual.",
+				m.Module, m.Node, m.Node), conditions.ExplanationMax),
+			Needs:    fmt.Sprintf("build %s again; the details name the image.", m.Module),
+			Resolved: "Resolved: the store holds the image of " + m.Module + " again"})
+	}
+	return out
+}
+
+// gatherMissingImagesHere is gatherMissingImages asking the mesh's own artifact store; nothing to ask where the mesh has
+// none on its network yet.
+func gatherMissingImagesHere(ctx context.Context, inv *inventory.Inventory) ([]imageFacts, error) {
+	shelf, err := inv.Catalogue(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
+	}
+	address, err := artifactStoreAddress(ctx, inv, shelf, "")
+	if err != nil {
+		return nil, err
+	}
+	if address == "" {
+		return nil, nil
+	}
+	return gatherMissingImages(ctx, inv, artifacts.Store{Address: address}.HoldsManifest)
+}

cmd/mesh-controller/image_missing_test.go

@@ -0,0 +1,125 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/conditions"
+)
+
+// **No declaration is sent that names an image the artifact store does not hold** (novox/hq issue 1742).
+
+const storeAt = "novox.internal:5100"
+
+// automx is the image the send of 2026-10-21 17:13 named and the store lacked; the others are held.
+const (
+	automxImage = storeAt + "/mailu/automx@sha256:c62150b8e2cbcc18cb911b098c2877f928761a1e6251667f39ea18251ea35591"
+	imapImage   = storeAt + "/mailu/imap@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+)
+
+func aMailSend() sendable {
+	return sendable{
+		Resources: []map[string]any{
+			{"id": "mailu.automx", "type": "container", "image": automxImage},
+			{"id": "mailu.automx-config", "type": "file", "path": "/etc/automx.conf"},
+			{"id": "mailu.imap", "type": "container", "image": imapImage},
+			{"id": "other.web", "type": "container", "image": "docker.io/library/nginx@sha256:" + strings.Repeat("b", 64)},
+			{"id": "dns.dnsmasq", "type": "container", "image": storeAt + "/dnsmasq/main@sha256:" + strings.Repeat("c", 64)},
+		},
+		moduleOf: map[string]string{"mailu.automx": "automx", "mailu.automx-config": "automx", "mailu.imap": "mailu",
+			"other.web": "other", "dns.dnsmasq": "dnsmasq"},
+		Builds: map[string]string{"automx": "b2f5ebb2new", "mailu": "m2", "other": "o1", "dnsmasq": "d1"},
+	}
+}
+
+func storeHolding(missing ...string) (storeHolds, *[]string) {
+	var asked []string
+	return func(_ context.Context, reference string) (bool, error) {
+		asked = append(asked, reference)
+		for _, m := range missing {
+			if strings.Contains(reference, m) {
+				return false, nil
+			}
+		}
+		return true, nil
+	}, &asked
+}
+
+// 1742/AC1: given a send naming automx's image, which the store does not hold, when the declaration is judged, then
+// every resource of automx is out of it, automx is among the modules left out with the reason, its build is the one the
+// machine was last sent, and every other module — an image of another registry among them — is as composed.
+func TestAModuleWhoseImageTheStoreLacksIsLeftOutOfTheSend(t *testing.T) {
+	s := aMailSend()
+	holds, asked := storeHolding("mailu/automx@")
+	found := holdBackMissingImages(t.Context(), "novox", &s, storeAt, holds, map[string]string{"automx": "b2f5ebb2old"})
+	if len(found) != 1 || found[0].Module != "automx" || found[0].Node != "novox" || found[0].Image != automxImage {
+		t.Fatalf("found %+v", found)
+	}
+	var ids []string
+	for _, r := range s.Resources {
+		ids = append(ids, r["id"].(string))
+	}
+	if got := strings.Join(ids, ","); got != "mailu.imap,other.web,dns.dnsmasq" {
+		t.Errorf("the declaration keeps %s", got)
+	}
+	if len(s.LeftOut) != 1 || s.LeftOut[0] != "automx" || !strings.Contains(s.leftOutWhy["automx"], "not in the artifact store") {
+		t.Errorf("left out %v because %v", s.LeftOut, s.leftOutWhy)
+	}
+	if s.Builds["automx"] != "b2f5ebb2old" || s.Builds["mailu"] != "m2" || s.Builds["other"] != "o1" {
+		t.Errorf("builds %v", s.Builds)
+	}
+	for _, a := range *asked {
+		if strings.Contains(a, "docker.io") {
+			t.Errorf("the store was asked about an image of another registry: %s", a)
+		}
+	}
+}
+
+// 1742/AC2: given a module never sent to the machine, when its image is missing, then it has no build in the record of the
+// send; given the store does not answer, then nothing is held back for a question unanswered.
+func TestAnUnansweredStoreHoldsNothingBackAndANewModuleHasNoBuild(t *testing.T) {
+	s := aMailSend()
+	holds, _ := storeHolding("mailu/automx@")
+	found := holdBackMissingImages(t.Context(), "novox", &s, storeAt, holds, nil)
+	if len(found) != 1 {
+		t.Fatalf("%+v", found)
+	}
+	if _, has := s.Builds["automx"]; has {
+		t.Errorf("a module never sent is recorded as sent: %v", s.Builds)
+	}
+	s = aMailSend()
+	silent := func(context.Context, string) (bool, error) { return false, errors.New("registry: connection refused") }
+	if found := holdBackMissingImages(t.Context(), "novox", &s, storeAt, silent, nil); len(found) != 0 || len(s.LeftOut) != 0 ||
+		len(s.Resources) != 5 {
+		t.Errorf("an unanswered store held back %+v, left out %v", found, s.LeftOut)
+	}
+}
+
+// 1742/AC3: a module left out for its image raises a condition for the operator naming the module, the machine and the
+// image in the evidence, plain words, and no answer to give; a mesh whose images are all held raises nothing.
+func TestAMissingImageIsRaisedForTheOperator(t *testing.T) {
+	f := calm(time.Date(2026, 10, 21, 17, 40, 0, 0, time.UTC))
+	if got := watchMissingImages(f); len(got) != 0 {
+		t.Fatalf("%+v", got)
+	}
+	f.images = []imageFacts{{missingImage{Node: "novox", Module: "automx", Image: automxImage}}}
+	got := watchMissingImages(f)
+	if len(got) != 1 {
+		t.Fatalf("%+v", got)
+	}
+	o := got[0]
+	if o.Kind != "image-missing" || o.Key() != "module.automx.novox.image-missing" || o.Resolver != conditions.ResolverOperator ||
+		o.Severity != conditions.Warning || o.Machine != "novox" {
+		t.Fatalf("%+v (%s)", o, o.Key())
+	}
+	if !strings.Contains(o.Said, "c62150b8") || !strings.Contains(o.Headline+o.Explanation, "automx") {
+		t.Errorf("words: %q / %q / %q", o.Headline, o.Explanation, o.Said)
+	}
+	w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
+	if why, ok := conditions.PlainWords(w, "novox"); !ok {
+		t.Errorf("not plain: %s", why)
+	}
+}

cmd/mesh-controller/new_module_asked_once_test.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// 1617/AC2 (unit): a new module whose first build a merge's hearing asked is not asked again when its batch is cut,
+// and one whose ask failed, was never handed over or is long lost is.
+func TestAModuleWhoseBuildIsComingIsNotAskedAgain(t *testing.T) {
+	now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
+	source := buildSource{Repository: "http://forge/novox/mesh-catalog.git", Seat: "git"}
+	asked := func(path string, mod func(*inventory.RequestOutcome)) inventory.RequestOutcome {
+		r := inventory.RequestOutcome{BuildRequest: inventory.BuildRequest{ID: "b-" + path, Repository: source.Repository,
+			Seat: "git", Path: path, For: "merge", At: now.Add(-3 * time.Minute)}}
+		if mod != nil {
+			mod(&r)
+		}
+		return r
+	}
+	requests := []inventory.RequestOutcome{
+		asked("modules/go", nil),
+		asked("modules/failed", func(r *inventory.RequestOutcome) { r.Heard, r.Failed = true, "no module.json" }),
+		asked("modules/notasked", func(r *inventory.RequestOutcome) { r.NotAsked = "the bus is silent" }),
+		asked("modules/lost", func(r *inventory.RequestOutcome) { r.At = now.Add(-2 * time.Hour) }),
+		asked("modules/byhand", func(r *inventory.RequestOutcome) { r.For = "build" }),
+	}
+	got := unaskedDirs([]string{"modules/go", "modules/failed", "modules/notasked", "modules/lost", "modules/byhand",
+		"modules/nodejs"}, requests, source, now)
+	want := []string{"modules/failed", "modules/notasked", "modules/lost", "modules/byhand", "modules/nodejs"}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("asked again %v, want %v", got, want)
+	}
+}

cmd/mesh-controller/new_module_on_merge_test.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"sync"
 	"testing"
+	"time"
 
 	"git.novox.be/novox/mesh-sdk/go/catalogue"
 	"github.com/novox/mesh-controller/internal/inventory"
@@ -99,3 +100,38 @@ func TestAMergeBuildsItsNewModuleBesideItsPlan(t *testing.T) {
 		t.Fatalf("a merge adding no module asked for %v", *asked)
 	}
 }
+
+// 1617/AC1 (integration): a merge adding a module to a repository the mesh builds modules from, heard while a batch of
+// that repository is still assembling (its window running), asks the module's first build when it is heard, so the
+// module is registered while the batch waits; cutting the batch asks it no second time (1617/AC2).
+func TestAMergeAddingAModuleAsksItsBuildBeforeItsBatchIsCut(t *testing.T) {
+	open := windowed(t)
+	groupsWaiting(t, "pr/c1")
+	asked := asksWithPaths(t)
+	catalog := func(commit string, made time.Time, paths ...string) link.SourceMoved {
+		return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit,
+			Head: "pr/" + commit, MergedAt: made.Format(time.RFC3339Nano), Paths: paths}
+	}
+	hear(t, open, catalog("c1", t0, "modules/app/index.ts"), t0)
+	if len(*asked) != 0 {
+		t.Fatalf("a merge adding no module asked %v", *asked)
+	}
+	hear(t, open, lone(catalog("c2", t0.Add(10*time.Second), "modules/go/packages.json", "modules/go/module.json")), t0.Add(10*time.Second))
+	if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 {
+		t.Fatalf("the batch should still be assembling: %d walk(s), %d batch(es)", len(ws), len(bs))
+	}
+	want := [3]string{"novox/mesh-catalog", "modules/go", "main"}
+	if len(*asked) != 1 || (*asked)[0] != want {
+		t.Fatalf("heard with its batch assembling, the merge asked %v, not %v", *asked, want)
+	}
+	cutAt(t, open, t0.Add(10*time.Second+90*time.Second))
+	n := 0
+	for _, a := range *asked {
+		if a[1] == "modules/go" {
+			n++
+		}
+	}
+	if n != 1 {
+		t.Fatalf("modules/go was asked %d times: %v", n, *asked)
+	}
+}

cmd/mesh-controller/outcome_off_the_loop_test.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// heldElsewhere is the plans held by another act for longer than any test runs: waiting for them ends only with the
+// context, and a look that does not wait is told at once that they are busy.
+func heldElsewhere(ctx context.Context, wait bool) (func(), error) {
+	if !wait {
+		return nil, inventory.ErrPlansBusy
+	}
+	<-ctx.Done()
+	return nil, inventory.ErrPlansBusy
+}
+
+// 1708/AC1: given the plans held by another act, when the event loop takes a build's outcome, then it returns at
+// once, without waiting the plans' two minutes (on 2026-10-21 at 14:20:45 nodered's outcome held the loop until
+// 14:22:45, and the operator was told the controller stopped listening).
+func TestAnOutcomeDoesNotWaitForThePlansAnotherActHolds(t *testing.T) {
+	ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+	defer cancel()
+	start := time.Now()
+	release, err := holdPlansForOutcome(ctx, heldElsewhere)
+	if took := time.Since(start); took > time.Second {
+		t.Fatalf("taking an outcome waited %s for plans another act holds; the event loop takes nothing meanwhile", took)
+	}
+	if err == nil || release != nil {
+		t.Fatalf("plans another act holds were said to be held: release %v, err %v", release != nil, err)
+	}
+}
+
+// 1708/AC4: given the plans free, when the event loop takes a build's outcome, then it holds them and marks the plan
+// from the outcome directly, as before.
+func TestAnOutcomeHoldsThePlansWhenTheyAreFree(t *testing.T) {
+	let := false
+	release, err := holdPlansForOutcome(t.Context(), func(context.Context, bool) (func(), error) {
+		return func() { let = true }, nil
+	})
+	if err != nil || release == nil {
+		t.Fatalf("free plans were not held: %v", err)
+	}
+	release()
+	if !let {
+		t.Fatal("the hold given was not the one taken")
+	}
+}
+
+// withLooker installs a looker that is never run, and answers it, so a test can read whether a look was asked.
+func withLooker(t *testing.T) *looker {
+	t.Helper()
+	l := newLooker(func(context.Context) bool { return false })
+	lookerMu.Lock()
+	before := theLooker
+	theLooker = l
+	lookerMu.Unlock()
+	t.Cleanup(func() {
+		lookerMu.Lock()
+		theLooker = before
+		lookerMu.Unlock()
+	})
+	return l
+}
+
+// 1708/AC2: given an outcome left to the build records because the plans were held, when it is left, then the look
+// at the plans is asked for, so the records settle it once the holder lets go rather than at the next event.
+func TestAnOutcomeLeftToTheRecordsAsksForTheLook(t *testing.T) {
+	l := withLooker(t)
+	leaveToRecords("nodered", "build-1708-ac2", "", "", inventory.ErrPlansBusy)
+	select {
+	case <-l.signal:
+	default:
+		t.Fatal("an outcome left to the build records asked for no look: the plan waits for whatever event comes next")
+	}
+}
+
+// 1708/AC3: given an outcome left to the records that said what the build did (ADR 0350 §4), when the look settles
+// the plan from the build records by that build's id, then the module keeps those words, once.
+func TestTheRecordsSettleAnOutcomeWithWhatTheBuildDid(t *testing.T) {
+	withLooker(t)
+	asked := time.Date(2026, 10, 21, 12, 15, 0, 0, time.UTC)
+	plan := func() inventory.Plan {
+		return inventory.Plan{ID: "plan-1708", Tiers: [][]string{{"nodered"}},
+			Modules: map[string]*inventory.PlanModule{"nodered": {State: "asked", AskedAt: &asked, Build: "build-1708-ac3"}}}
+	}
+	byID := map[string]inventory.Build{"build-1708-ac3": {ID: "build-1708-ac3", Module: "nodered", Commit: "4efac366",
+		At: asked.Add(5 * time.Minute)}}
+	leaveToRecords("nodered", "build-1708-ac3", "reused from build-1708-check", "", inventory.ErrPlansBusy)
+
+	p := plan()
+	if !settleFromRecords(&p, p.Tiers[0], nil, byID) || p.Modules["nodered"].State != "built" {
+		t.Fatalf("the records did not settle the outcome: %+v", p.Modules["nodered"])
+	}
+	if got := p.Modules["nodered"].Made; got != "reused from build-1708-check" {
+		t.Fatalf("settled from the records, the module says %q of what its build did, not the outcome's words", got)
+	}
+	again := plan()
+	settleFromRecords(&again, again.Tiers[0], nil, byID)
+	if got := again.Modules["nodered"].Made; got != "" {
+		t.Fatalf("an outcome's words were kept for a second settle: %q", got)
+	}
+}
+
+// 1708/AC5: given a serving controller (its look at the plans running), when an outcome was taken into the plans,
+// then moving them (a release's sends, which may retry for minutes) is left to the look, off the event loop
+// (2026-10-21 14:28:40 to 14:32:10: the loop sent okioki's release to g14 after mesh-delivery-builder's outcome).
+func TestAnOutcomeLeavesMovingThePlansToTheLook(t *testing.T) {
+	l := withLooker(t)
+	moved := false
+	advanceAfterOutcome(func() { moved = true })
+	if moved {
+		t.Fatal("the plans were moved on the event loop after an outcome, where a send that retries holds every message")
+	}
+	select {
+	case <-l.signal:
+	default:
+		t.Fatal("the look was not asked to move the plans after an outcome")
+	}
+}
+
+// 1708/AC5: given no look running (a command, which has no event loop), when an outcome was taken, then the plans are
+// moved at once, as before.
+func TestAnOutcomeMovesThePlansWhereNoLookRuns(t *testing.T) {
+	lookerMu.Lock()
+	before := theLooker
+	theLooker = nil
+	lookerMu.Unlock()
+	t.Cleanup(func() {
+		lookerMu.Lock()
+		theLooker = before
+		lookerMu.Unlock()
+	})
+	moved := false
+	advanceAfterOutcome(func() { moved = true })
+	if !moved {
+		t.Fatal("with no look running, an outcome left the plans unmoved")
+	}
+}
+
+// 1708/AC6: given a registration the mesh refused (its build recorded as worked, the outcome failed), left to the
+// records because the plans were held, when the look settles the plan from the records, then the module and the plan
+// are failed, never built: settled as built, the tier would be sent the version from before (reviewer-1708@shanks).
+func TestARefusedRegistrationLeftToTheRecordsFailsThePlan(t *testing.T) {
+	withLooker(t)
+	asked := time.Date(2026, 10, 21, 12, 15, 0, 0, time.UTC)
+	p := inventory.Plan{ID: "plan-1708-ac6", State: inventory.PlanBuilding, Tiers: [][]string{{"nodered"}},
+		Modules: map[string]*inventory.PlanModule{"nodered": {State: "asked", AskedAt: &asked, Build: "build-1708-ac6"}}}
+	byID := map[string]inventory.Build{"build-1708-ac6": {ID: "build-1708-ac6", Module: "nodered", Commit: "4efac366",
+		At: asked.Add(5 * time.Minute)}}
+	leaveToRecords("nodered", "build-1708-ac6", "", "the mesh does not register it: a port is taken", inventory.ErrPlansBusy)
+
+	overlayRefusals(nil, byID, refusalsKeptFor([]string{"build-1708-ac6"}))
+	settleFromRecords(&p, p.Tiers[0], nil, byID)
+	if s := p.Modules["nodered"]; s.State != "failed" || p.State != inventory.PlanFailed {
+		t.Fatalf("a refused registration left to the records settled as %q, the plan %q", s.State, p.State)
+	}
+	if !strings.Contains(p.Modules["nodered"].Why, "a port is taken") {
+		t.Fatalf("the refusal's words were lost: %q", p.Modules["nodered"].Why)
+	}
+}
+
+// 1708/AC6: a refusal kept in the store (a controller replaced before the settle) fails the record as one kept in
+// the process does.
+func TestARefusalFromTheStoreFailsTheRecord(t *testing.T) {
+	recorded := map[string][]inventory.Build{"nodered": {{ID: "b-store", Module: "nodered"}, {ID: "b-other", Module: "nodered"}}}
+	byID := map[string]inventory.Build{"b-store": {ID: "b-store", Module: "nodered"}}
+	overlayRefusals(recorded, byID, map[string]string{"b-store": "not registered"})
+	if recorded["nodered"][0].Worked() || byID["b-store"].Worked() {
+		t.Fatal("a build whose registration was refused still reads as worked")
+	}
+	if !recorded["nodered"][1].Worked() {
+		t.Fatal("a build nobody refused was failed")
+	}
+}
+
+// 1708/AC7: given the records settle a module as deleted at its source, or fail the plan, when the look settles, then
+// it does what taking the outcome directly does: the deleted module is named to be forgotten, and the plan's failure
+// is said (reviewer-1708@shanks).
+func TestSettlingFromTheRecordsDoesWhatTakingTheOutcomeDoes(t *testing.T) {
+	p := inventory.Plan{State: inventory.PlanFailed, Modules: map[string]*inventory.PlanModule{
+		"gone": {State: planDeleted}, "x": {State: "failed"}, "y": {State: "built"}}}
+	before := map[string]string{"gone": "asked", "x": "asked", "y": "built"}
+	deleted, failed := settledEffects(before, &p, inventory.PlanBuilding)
+	if len(deleted) != 1 || deleted[0] != "gone" {
+		t.Fatalf("the module the records found deleted is not forgotten: %v", deleted)
+	}
+	if !failed {
+		t.Fatal("the plan the records failed is not said as failed")
+	}
+	if d, f := settledEffects(map[string]string{"gone": planDeleted}, &p, inventory.PlanFailed); len(d) != 0 || f {
+		t.Fatalf("what was deleted or failed before the settle is acted on again: %v %v", d, f)
+	}
+}

cmd/mesh-controller/plain_words.go

@@ -195,6 +195,29 @@ var plainWordings = map[string]func(conditions.Observation) words{
 				"there yet is sealed by the controller until the vault holds it.", m, m),
 			Resolved: "Resolved: the vault gave " + m + " the core's credentials"}
 	}),
+	kindFilesNotPut: worded(func(o conditions.Observation) words {
+		m := machineOr(o, "a machine")
+		return words{Headline: "The big files for " + m + " cannot be put on the bus",
+			Needs: "look at the bus's permissions for the controller; the details give the reason.",
+			Explanation: fmt.Sprintf("The mesh could not put the large files of %s's instructions in their place on the bus, "+
+				"send after send, so %s is sent them inside the instructions instead, which may make them too big.", m, m),
+			Resolved: "Resolved: the big files for " + m + " reach the bus again"}
+	}),
+	kindDeclarationNearLimit: worded(func(o conditions.Observation) words {
+		m := machineOr(o, "a machine")
+		if o.Severity == conditions.Urgent {
+			return words{Headline: m + " cannot be sent its instructions",
+				Needs: "make the instructions for " + m + " smaller, for example by taking a module off it.",
+				Explanation: fmt.Sprintf("The instructions for %s have grown past the largest message the bus takes, so "+
+					"nothing new reaches %s until they are smaller.", m, m),
+				Resolved: "Resolved: the instructions for " + m + " fit the bus again"}
+		}
+		return words{Headline: "The instructions for " + m + " are nearly too big",
+			Needs: "do not add more to " + m + " until this is dealt with; the details give the size.",
+			Explanation: fmt.Sprintf("The instructions for %s are close to the largest message the bus takes. "+
+				"One more module or secret may stop anything new from reaching %s.", m, m),
+			Resolved: "Resolved: the instructions for " + m + " have room again"}
+	}),
 	kindCoreDowngrade: worded(func(o conditions.Observation) words {
 		m := machineOr(o, "a machine")
 		component := idPart(o, 0)
@@ -434,6 +457,18 @@ var plainWordings = map[string]func(conditions.Observation) words{
 			Explanation: "A delivery did not finish. Nothing is tried again until you say so.",
 			Resolved:    "Resolved: the delivery is no longer failed"}
 	}),
+	kindMovesHeld: worded(func(o conditions.Observation) words {
+		return words{Headline: "An update waits for you",
+			Explanation: "A new version of a module would restart a container or touch its data, so it waits for you.",
+			Resolved:    "Resolved: the update is no longer waiting"}
+	}),
+	kindImageMissing: worded(func(o conditions.Observation) words {
+		return words{Headline: "A module is not updated",
+			Explanation: "The new version of a module names a container image that the mesh's store does not hold, so it " +
+				"was not sent. What runs is untouched.",
+			Needs:    "build the module again; the details name the image.",
+			Resolved: "Resolved: the store holds the image again"}
+	}),
 	"ask-lost": worded(func(o conditions.Observation) words {
 		return words{Headline: "A build has no result",
 			Explanation: "A build was asked for and never answered. It is asked again with the next delivery.",

cmd/mesh-controller/plain_words_test.go

@@ -260,7 +260,7 @@ func TestUpdatesHeldNameWhatWaitsAndWhereItIsReleased(t *testing.T) {
 	plainExample(t, got[0], "Updates of openrazer and sensors wait for your release",
 		"Needs you: release them from the mesh MCP server; this notification cannot do it. Updates of openrazer and "+
 			"sensors on g14 and shanks wait for a person to release them, because the last walk failed. "+
-			"They are not delivered until then, and stay held if you leave them.")
+			"They are not delivered until then, and stay held if you leave them.", "Release")
 }
 
 // **What held them is said in the glossary's words** (2026-10-08 review): the backlog is held after any

cmd/mesh-controller/plan_clock_test.go

@@ -78,3 +78,50 @@ func TestAPlansLineCountsFromItsTierNotFromItsLastSave(t *testing.T) {
 
 // novox/hq issue 296: a plan whose tier stands still is not written again by every advance — its revision,
 // its last save and its `plan-moved` stay where its last change left them.
+
+// novox/hq issue 920: a building plan whose current tier is all built waits on its note (a send refused at the
+// gate), and its line, `status` and the stalled condition say that, not "building for ... LATE".
+func TestAPlanLineSaysTheRefusalOnceItsTierIsBuilt(t *testing.T) {
+	now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
+	built := now.Add(-time.Hour)
+	refusal := "tier 1: a build waits there for its gate: novox (mesh-delivery-indexer d68d7ef3→a5aa69a4) — tried again"
+	mk := func(second string) inventory.Plan {
+		return inventory.Plan{ID: "plan-r", Repository: "novox/mesh-sdk", Commit: "b2f5ebb2", State: inventory.PlanBuilding,
+			Tier: 1, Created: now.Add(-2 * time.Hour), Updated: now, TierEntered: now.Add(-time.Hour), Note: refusal,
+			Tiers: [][]string{{"a"}, {"b", "c"}},
+			Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &built},
+				"b": {State: "built", BuiltAt: &built}, "c": {State: second}}}
+	}
+
+	// 920/AC1
+	p := mk("built")
+	line := planLine(p, now)
+	if !strings.Contains(line, "a build waits there for its gate: novox") || strings.Contains(line, "building for") {
+		t.Errorf("920/AC1: a plan with its tier built and a refusal reads %q", line)
+	}
+	st := planStatuses([]inventory.Plan{p}, now, pauseView{}, nil)
+	if len(st) != 1 || !st[0].Late {
+		t.Errorf("920/AC1: status: %+v", st)
+	}
+
+	// 920/AC2
+	if line := planLine(mk("asked"), now); !strings.Contains(line, "building for 1h0m0s") {
+		t.Errorf("920/AC2: a plan with a tier module not built reads %q", line)
+	}
+}
+
+// 920/AC3: a tier move writes the previous tier's summary into the note; with the new tier all built, the line still
+// says "building for", for that note is no refusal.
+func TestAPlanLineDoesNotHoldOnATiersSummary(t *testing.T) {
+	now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
+	built := now.Add(-time.Hour)
+	for _, note := range []string{"failed or refused its send (apply-failing says it): novox", "tier 0 built; sent mesh-sdk to novox first, in one send"} {
+		p := inventory.Plan{ID: "plan-s", Repository: "novox/mesh-sdk", Commit: "b2f5ebb2", State: inventory.PlanBuilding,
+			Tier: 1, Created: now.Add(-2 * time.Hour), Updated: now, TierEntered: now.Add(-time.Hour), Note: note,
+			Tiers:   [][]string{{"a"}, {"b"}},
+			Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &built}, "b": {State: "built", BuiltAt: &built}}}
+		if line := planLine(p, now); !strings.Contains(line, "building for 1h0m0s") || strings.Contains(line, "held") {
+			t.Errorf("920/AC3: a plan with the note %q reads %q", note, line)
+		}
+	}
+}

cmd/mesh-controller/plan_looks.go

@@ -136,6 +136,18 @@ func askToLook() {
 	}
 }
 
+// askedToLook asks the serving controller's look at the plans, and says whether there was one to ask.
+func askedToLook() bool {
+	lookerMu.RLock()
+	l := theLooker
+	lookerMu.RUnlock()
+	if l == nil {
+		return false
+	}
+	l.ask()
+	return true
+}
+
 // startPlanLooks starts the look at the open plans, and asks for the first: the controller took the lease, and a
 // plan left by a replaced controller resumes from there (ADR 0406 §6). It replaces the rollout's 30 second ticker.
 // It answers a channel that is closed once the look has returned after the context ended, for a test that restores what
@@ -194,6 +206,7 @@ func servingPlanSaved(ctx context.Context, bus link.Bus, open *stores) func(*inv
 	return func(_ *inventory.Inventory, p inventory.Plan) {
 		sayPlanMoved(ctx, bus, open.inventory, p)
 		planKept(p)
+		keepBuildingKey(ctx, p)
 		if p.WalkSaid == inventory.WalkEnded {
 			go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
 		}

cmd/mesh-controller/plan_moved_every_writer_test.go

@@ -31,7 +31,8 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
 				return err
 			}
 			text := string(raw)
-			if write.MatchString(text) {
+			// The regular expression is the costly part of reading every source: only a file that names the table can match.
+			if strings.Contains(strings.ToLower(text), "release_plan") && write.MatchString(text) {
 				found = append(found, path+" writes the plan table")
 			}
 			// A plan record is written by the bucket's raw create and update, which only the plans' own file calls.

cmd/mesh-controller/plans_building.go

@@ -0,0 +1,238 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// The plans building (novox/hq ADR 0483 clause 2, issue 1687).
+//
+// **A key per delivery plan in its build phase, so that the build seat takes no new check while a delivery builds.** The phase
+// runs from the ask of the plan's first build until every build of its last tier is built, or the plan fails, stops or waits
+// on the operator. It is derived from the plan alone (buildPhaseOf), so a controller that restarts finds the same answer. The
+// key is put when the first build is asked (sayBuilding), kept current at every save of the plan (keepBuildingKey) and by a
+// look every link.PlansBuildingRefresh while this controller holds the lease (keepingPlansBuilding), and deleted at the phase's
+// end; the bucket's age removes the key of a controller that stopped, so checks are held at most that long.
+//
+// The layout is documented in internal/link/holds.go.
+
+// buildPhaseOf says whether a plan is in its build phase: open, not waiting for the operator's word, with a build asked and a
+// build of its tiers still to be built. A tier built and sent leaves the plan rolling while later tiers are still to be asked;
+// it is in the phase still, which is what keeps a check from taking a place between two tiers.
+func buildPhaseOf(p inventory.Plan) bool {
+	if !p.Open() || p.Waiting() || p.Release != nil {
+		return false
+	}
+	asked, unbuilt := false, false
+	for _, tier := range p.Tiers {
+		for _, m := range tier {
+			st := p.Modules[m]
+			if st != nil && st.State != "" {
+				asked = true
+			}
+			if st == nil || (st.State != "built" && st.State != planDeleted) {
+				unbuilt = true
+			}
+		}
+	}
+	return asked && unbuilt
+}
+
+// buildingSince is when the plan's first build was asked, or fallback when no module says.
+func buildingSince(p inventory.Plan, fallback time.Time) time.Time {
+	since := time.Time{}
+	for _, st := range p.Modules {
+		if st != nil && st.AskedAt != nil && (since.IsZero() || st.AskedAt.Before(since)) {
+			since = *st.AskedAt
+		}
+	}
+	if since.IsZero() {
+		return fallback
+	}
+	return since
+}
+
+// buildingStore is the bucket of the plans building.
+type buildingStore interface {
+	Put(ctx context.Context, plan string, value []byte) error
+	Delete(ctx context.Context, plan string) error
+	Keys(ctx context.Context) ([]string, error)
+}
+
+// syncBuilding makes the bucket say what the plans say: a key for each plan in its build phase, put again, and none for any
+// other plan, deleted. It returns the faults, each once.
+func syncBuilding(ctx context.Context, store buildingStore, plans []inventory.Plan, now time.Time) []error {
+	var faults []error
+	live := map[string]bool{}
+	for _, p := range plans {
+		if !buildPhaseOf(p) {
+			continue
+		}
+		live[p.ID] = true
+		if err := putBuilding(ctx, store, p.ID, buildingSince(p, now), now); err != nil {
+			faults = append(faults, err)
+		}
+	}
+	keys, err := store.Keys(ctx)
+	if err != nil {
+		return append(faults, err)
+	}
+	for _, key := range keys {
+		if live[key] {
+			continue
+		}
+		if err := store.Delete(ctx, key); err != nil {
+			faults = append(faults, fmt.Errorf("the key of %s could not be taken away: %w", key, err))
+		}
+	}
+	return faults
+}
+
+func putBuilding(ctx context.Context, store buildingStore, plan string, since, now time.Time) error {
+	body, _ := json.Marshal(link.PlanBuilding{Plan: plan, Since: since.UTC(), Seen: now.UTC()})
+	if err := store.Put(ctx, plan, body); err != nil {
+		return fmt.Errorf("the key of %s could not be put: %w", plan, err)
+	}
+	return nil
+}
+
+// busBuilding is the store on the controller's bucket.
+type busBuilding struct {
+	conn *nats.Conn
+	mu   sync.Mutex
+	kv   jetstream.KeyValue
+}
+
+func (b *busBuilding) bucket(ctx context.Context) (jetstream.KeyValue, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.kv != nil {
+		return b.kv, nil
+	}
+	api, err := jetstream.New(b.conn)
+	if err != nil {
+		return nil, err
+	}
+	kv, err := api.KeyValue(ctx, broker.PlansBuildingBucket)
+	if err != nil {
+		return nil, fmt.Errorf("the plans building: %w", err)
+	}
+	b.kv = kv
+	return kv, nil
+}
+
+func (b *busBuilding) Put(ctx context.Context, plan string, value []byte) error {
+	kv, err := b.bucket(ctx)
+	if err != nil {
+		return err
+	}
+	_, err = kv.Put(ctx, plan, value)
+	return err
+}
+
+func (b *busBuilding) Delete(ctx context.Context, plan string) error {
+	kv, err := b.bucket(ctx)
+	if err != nil {
+		return err
+	}
+	if err := kv.Delete(ctx, plan); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
+		return err
+	}
+	return nil
+}
+
+func (b *busBuilding) Keys(ctx context.Context) ([]string, error) {
+	kv, err := b.bucket(ctx)
+	if err != nil {
+		return nil, err
+	}
+	keys, err := kv.Keys(ctx)
+	if errors.Is(err, jetstream.ErrNoKeysFound) {
+		return nil, nil
+	}
+	return keys, err
+}
+
+// buildingKeys is the store of the serving controller; nil in a process that is not it, which says nothing.
+var buildingKeys atomic.Pointer[buildingStore]
+
+func setBuildingStore(s buildingStore) {
+	if s == nil {
+		buildingKeys.Store(nil)
+		return
+	}
+	buildingKeys.Store(&s)
+}
+
+// sayBuilding puts a plan's key: its first build is being asked (or another of its builds). A key that could not be put is
+// said; the look that follows puts it again.
+func sayBuilding(ctx context.Context, plan string, since time.Time) {
+	s := buildingKeys.Load()
+	if s == nil {
+		return
+	}
+	if err := putBuilding(ctx, *s, plan, since, time.Now()); err != nil {
+		fmt.Printf("plans: %v; checks may take a place of %s until the next look\n", err, plan)
+	}
+}
+
+// keepBuildingKey is told each plan this controller keeps: its key put again while it builds and taken away once it does not.
+func keepBuildingKey(ctx context.Context, p inventory.Plan) {
+	s := buildingKeys.Load()
+	if s == nil {
+		return
+	}
+	if buildPhaseOf(p) {
+		sayBuilding(ctx, p.ID, buildingSince(p, time.Now()))
+		return
+	}
+	if err := (*s).Delete(ctx, p.ID); err != nil {
+		fmt.Printf("plans: the key of %s could not be taken away; it ages out: %v\n", p.ID, err)
+	}
+}
+
+// keepingPlansBuilding looks at the plans every link.PlansBuildingRefresh for as long as this controller serves and holds the
+// lease, so a key is put again before its age and the key of a plan that left the phase unseen is taken away.
+func keepingPlansBuilding(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn) {
+	store := &busBuilding{conn: conn}
+	setBuildingStore(store)
+	defer setBuildingStore(nil)
+	tick := time.NewTicker(broker.PlansBuildingRefresh)
+	defer tick.Stop()
+	failing := ""
+	for {
+		if epoch, err := theLease.epoch(ctx); err == nil && epoch != 0 {
+			why := ""
+			if plans, err := inv.OpenPlans(ctx); err != nil {
+				why = fmt.Sprintf("the plans cannot be read: %v", err)
+			} else if faults := syncBuilding(ctx, store, plans, time.Now()); len(faults) > 0 {
+				why = fmt.Sprint(faults)
+			}
+			if why != failing {
+				if why != "" {
+					fmt.Printf("plans: the keys of the plans building cannot be kept: %s\n", why)
+				} else {
+					fmt.Printf("plans: the keys of the plans building are kept again\n")
+				}
+				failing = why
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-tick.C:
+		}
+	}
+}

cmd/mesh-controller/plans_building_test.go

@@ -0,0 +1,175 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// Issue 1687 (ADR 0483 clause 2): a delivery plan's key lives from the ask of its first build until every build of its last
+// tier is built, or the plan fails, stops or waits on the operator.
+
+type fakeBuildingStore struct {
+	mu   sync.Mutex
+	keys map[string]link.PlanBuilding
+	puts map[string]int
+}
+
+func newFakeBuildingStore() *fakeBuildingStore {
+	return &fakeBuildingStore{keys: map[string]link.PlanBuilding{}, puts: map[string]int{}}
+}
+
+func (f *fakeBuildingStore) Put(_ context.Context, plan string, value []byte) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var v link.PlanBuilding
+	if err := json.Unmarshal(value, &v); err != nil {
+		return err
+	}
+	f.keys[plan] = v
+	f.puts[plan]++
+	return nil
+}
+
+func (f *fakeBuildingStore) Delete(_ context.Context, plan string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	delete(f.keys, plan)
+	return nil
+}
+
+func (f *fakeBuildingStore) Keys(context.Context) ([]string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var out []string
+	for k := range f.keys {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out, nil
+}
+
+func aPlan(id, state string, tiers [][]string, states map[string]string) inventory.Plan {
+	p := inventory.Plan{ID: id, State: state, Tiers: tiers, Modules: map[string]*inventory.PlanModule{}}
+	now := time.Now().Add(-time.Minute)
+	for m, st := range states {
+		pm := &inventory.PlanModule{State: st}
+		if st != "" {
+			pm.AskedAt = &now
+		}
+		p.Modules[m] = pm
+	}
+	return p
+}
+
+func TestAPlanIsInItsBuildPhaseFromTheFirstAskToTheLastBuild(t *testing.T) {
+	tiers := [][]string{{"a", "b"}, {"c"}}
+	cases := []struct {
+		name string
+		plan inventory.Plan
+		want bool
+	}{
+		{"no build asked yet", aPlan("p", inventory.PlanBuilding, tiers, nil), false},
+		{"the first tier asked", aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "asked", "b": "asked"}), true},
+		// Between two tiers: the first built and sent (rolling), the second not asked yet.
+		{"between two tiers", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built"}), true},
+		{"the last tier asked", aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "built", "b": "built", "c": "asked"}), true},
+		{"every build built", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built", "c": "built"}), false},
+		// A module the plan deletes is done as far as its tiers go (release_plan.go), so it holds no key through the rollout.
+		{"a deleted module is done", aPlan("p", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built", "c": planDeleted}), false},
+		{"failed", aPlan("p", inventory.PlanFailed, tiers, map[string]string{"a": "failed", "b": "asked"}), false},
+		{"superseded", aPlan("p", inventory.PlanSuperseded, tiers, map[string]string{"a": "asked"}), false},
+		{"a batch whose window is open", aPlan("p", inventory.PlanAssembling, tiers, map[string]string{"a": "asked"}), false},
+	}
+	for _, c := range cases {
+		if got := buildPhaseOf(c.plan); got != c.want {
+			t.Errorf("%s: in the build phase = %v, want %v", c.name, got, c.want)
+		}
+	}
+	waiting := aPlan("p", inventory.PlanBuilding, tiers, map[string]string{"a": "asked"})
+	waiting.Delivery = &inventory.PlanDelivery{Awaits: "the operator's word"}
+	if buildPhaseOf(waiting) {
+		t.Error("a plan that waits on the operator is in its build phase")
+	}
+}
+
+// The key's life: put, refreshed, deleted at the phase's end; a plan that left the phase unseen loses its key at the next look.
+func TestAPlanKeyIsPutRefreshedAndDeletedAtThePhasesEnd(t *testing.T) {
+	ctx := context.Background()
+	store := newFakeBuildingStore()
+	tiers := [][]string{{"a"}, {"b"}}
+	building := aPlan("plan-1", inventory.PlanBuilding, tiers, map[string]string{"a": "asked"})
+
+	if faults := syncBuilding(ctx, store, []inventory.Plan{building}, time.Now()); len(faults) > 0 {
+		t.Fatal(faults)
+	}
+	if _, ok := store.keys["plan-1"]; !ok {
+		t.Fatal("no key while the first build is asked")
+	}
+	first := store.keys["plan-1"]
+	// Refreshed: put again, the phase's start kept.
+	time.Sleep(2 * time.Millisecond)
+	syncBuilding(ctx, store, []inventory.Plan{building}, time.Now())
+	if store.puts["plan-1"] != 2 || !store.keys["plan-1"].Seen.After(first.Seen) || !store.keys["plan-1"].Since.Equal(first.Since) {
+		t.Errorf("the key was not refreshed with its start kept: %d puts, %+v then %+v", store.puts["plan-1"], first, store.keys["plan-1"])
+	}
+	// Between the tiers: still there.
+	between := aPlan("plan-1", inventory.PlanRolling, tiers, map[string]string{"a": "built"})
+	syncBuilding(ctx, store, []inventory.Plan{between}, time.Now())
+	if _, ok := store.keys["plan-1"]; !ok {
+		t.Error("the key was gone between two tiers")
+	}
+	// A key of another plan that is no longer in the phase.
+	store.keys["plan-0"] = link.PlanBuilding{Plan: "plan-0"}
+	done := aPlan("plan-1", inventory.PlanRolling, tiers, map[string]string{"a": "built", "b": "built"})
+	syncBuilding(ctx, store, []inventory.Plan{done}, time.Now())
+	if len(store.keys) != 0 {
+		t.Errorf("keys remain after the last build was built: %v", store.keys)
+	}
+}
+
+// keepBuildingKey, told of each save, puts the key while the plan builds and deletes it when the plan fails.
+func TestASavedPlanPutsOrDeletesItsKey(t *testing.T) {
+	ctx := context.Background()
+	store := newFakeBuildingStore()
+	var s buildingStore = store
+	buildingKeys.Store(&s)
+	defer buildingKeys.Store(nil)
+
+	p := aPlan("plan-7", inventory.PlanBuilding, [][]string{{"a"}}, map[string]string{"a": "asked"})
+	keepBuildingKey(ctx, p)
+	if _, ok := store.keys["plan-7"]; !ok {
+		t.Fatal("the key was not put when the plan was saved building")
+	}
+	p.State = inventory.PlanFailed
+	keepBuildingKey(ctx, p)
+	if _, ok := store.keys["plan-7"]; ok {
+		t.Fatal("the key stayed after the plan failed")
+	}
+}
+
+// The plan's key is there before its first build is asked, so a check cannot take a place between the ask and the plan's save.
+func TestThePlansKeyIsPutBeforeItsFirstBuildIsAsked(t *testing.T) {
+	store := newFakeBuildingStore()
+	var s buildingStore = store
+	buildingKeys.Store(&s)
+	defer buildingKeys.Store(nil)
+	p, byName := aTierOfThree()
+	keyThere := false
+	was := askABuild
+	askABuild = func(context.Context, buildSource, string, string) (string, error) {
+		_, keyThere = store.keys["plan-1"]
+		return "build-x", nil
+	}
+	t.Cleanup(func() { askABuild = was })
+	askModule(t.Context(), p, "a", byName)
+	if !keyThere {
+		t.Fatal("the plan's key was not there when its first build was asked")
+	}
+}

cmd/mesh-controller/plans_follow.go

@@ -153,7 +153,7 @@ func planWaitCommand(ctx context.Context, inv *inventory.Inventory, id, untilAsk
 	line := planLineSaid(p, now, pause, bounds.of(p.Repository), func(module string) []string {
 		on, _ := inv.Running(ctx, module)
 		return on
-	}, waitsOf(ctx, inv, one))
+	}, waitsOf(ctx, inv, one)) + heldSaid(p, heldNow(ctx, inv))
 	doc := planWaitedDoc(p, planStatuses(one, now, pause, bounds)[0], until, came, now.Sub(began), line)
 	return json.NewEncoder(os.Stdout).Encode(doc)
 }

cmd/mesh-controller/process_accounts.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// processUser is the bus user of one process of a module on a node (novox/hq ADR 0436 §1).
+func processUser(node, module, process string) string {
+	return broker.Principal{Kind: broker.KindProcess, Node: node, Module: module, Process: process}.Username()
+}
+
+// processAccountWarning is the warning line for the own secret broker-<process id> of a process that declares bus
+// when nobody issued its account (novox/hq ADR 0436 §2, issue 1226): "" for any other secret and for an issued
+// account. The send goes on with the stand-in until the composition leaves the module off (issue 1753), so the
+// line is the only loud thing about it.
+func processAccountWarning(m catalogue.Manifest, node, secret string, minted []string) string {
+	id, isProcess := strings.CutPrefix(secret, "broker-")
+	if !isProcess {
+		return ""
+	}
+	if _, declared := m.ProcessBuses()[id]; !declared {
+		return ""
+	}
+	user := processUser(node, m.Module, id)
+	for _, u := range minted {
+		if u == user {
+			return ""
+		}
+	}
+	return fmt.Sprintf("%s/%s process %s: its bus account is not issued — mesh-controller.issue %s %s",
+		node, m.Module, id, node, m.Module)
+}
+
+// sayWarning writes a line where the compose's other warnings go: the act's answer when it carries a writer,
+// else the controller's journal.
+func sayWarning(ctx context.Context, line string) {
+	var w io.Writer = os.Stderr
+	if lw, ok := ctx.Value(leftOutKey{}).(io.Writer); ok {
+		w = lw
+	}
+	fmt.Fprintln(w, line)
+}
+
+// processAccountsMissing is each process of m that declares bus and whose account is not minted yet on node.
+func processAccountsMissing(m catalogue.Manifest, node string, minted []string) []broker.ProcessPrincipal {
+	have := map[string]bool{}
+	for _, u := range minted {
+		have[u] = true
+	}
+	var out []broker.ProcessPrincipal
+	for _, pr := range inventory.ProcessesOf(m) {
+		if !have[processUser(node, m.Module, pr.Process)] {
+			out = append(out, pr)
+		}
+	}
+	return out
+}
+
+// issueMissingProcessAccounts issues the accounts of m's bus processes on node that are not minted, and no
+// other: an account already issued is rotated on purpose, never by a later delivery or assignment (issue 1226).
+// It says what it issued, one line, and nothing when nothing was missing.
+func issueMissingProcessAccounts(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string) (string, error) {
+	if len(m.ProcessBuses()) == 0 || mayIssue(m) != nil {
+		return "", nil
+	}
+	minted, err := inv.BusMinted(ctx)
+	if err != nil {
+		return "", err
+	}
+	missing := processAccountsMissing(m, node, minted)
+	if len(missing) == 0 {
+		return "", nil
+	}
+	known, err := broker.FromEnvironment()
+	if err != nil {
+		return "", fmt.Errorf("cannot deliver a credential without knowing where the bus is: %w", err)
+	}
+	reachable, err := brokerReachableAt(ctx, inv, known, node)
+	if err != nil {
+		return "", err
+	}
+	busAddress, err := broker.BusAddress()
+	if err != nil {
+		busAddress = ""
+	}
+	var ids []string
+	for _, pr := range missing {
+		if err := issueProcess(ctx, inv, m, pr, node, busAddress, known, reachable); err != nil {
+			return "", err
+		}
+		ids = append(ids, pr.Process)
+	}
+	return fmt.Sprintf("its process account(s) %s issued and sealed to %s, and arrive with the push",
+		strings.Join(ids, ", "), node), nil
+}
+
+// issueGainedProcessAccounts runs after a module's new definition is registered: on each machine assigned it, the
+// accounts of its bus processes not yet minted are issued, so a process a delivery adds is not pushed with a
+// random stand-in (issue 1226). Never fatal: the build worked and the module is registered, so a failure is said,
+// and the push refuses the module until `mesh-controller.issue` is run.
+func issueGainedProcessAccounts(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest) {
+	if len(m.ProcessBuses()) == 0 {
+		return
+	}
+	nodes, err := inv.Running(ctx, m.Module)
+	if err != nil {
+		fmt.Printf("%s: its machines are not read to issue its process accounts: %v\n", m.Module, err)
+		return
+	}
+	for _, node := range nodes {
+		line, err := issueMissingProcessAccounts(ctx, inv, m, node)
+		if err != nil {
+			fmt.Printf("%s on %s: its process accounts are not issued (%v): `mesh-controller.issue %s %s` first\n",
+				m.Module, node, err, node, m.Module)
+			continue
+		}
+		if line != "" {
+			fmt.Printf("%s on %s: %s\n", m.Module, node, line)
+		}
+	}
+}

cmd/mesh-controller/process_accounts_integration_test.go

@@ -0,0 +1,135 @@
+//go:build integration
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/catalogue"
+	"git.novox.be/novox/mesh-sdk/go/compose"
+
+	"github.com/novox/mesh-controller/internal/broker"
+	"github.com/novox/mesh-controller/internal/inventory"
+	"github.com/novox/mesh-controller/internal/link"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+func gainedAndPlain(t *testing.T) (plain, gained catalogue.Manifest) {
+	t.Helper()
+	plain, err := catalogue.ParseManifest([]byte(`{"module":"mesh-mail","version":"1",
+		"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
+		"resources":[{"id":"account","type":"user","name":"mesh-mail-send"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return plain, mailSenderWithProcess()
+}
+
+// 1226/AC2: given an assigned module whose new definition adds a bus process, when the delivery records it
+// (takeIn, the function that records a build result), the process's account is minted; and 1226/AC3: the same
+// build heard again replaces nothing.
+func TestAGainedProcessIsIssuedWhenTheDeliveryRecordsIt(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+
+	dir := t.TempDir()
+	if err := busCertificate([]string{"--into", dir}); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv(broker.AddressVar, strings.TrimPrefix(testbus.URL(t), "nats://"))
+	t.Setenv(broker.CertificateVar, filepath.Join(dir, "tls.crt"))
+
+	plain, gained := gainedAndPlain(t)
+	register(t, open, plain)
+	if _, err := assign(ctx, open, "laptop", "mesh-mail"); err != nil {
+		t.Fatal(err)
+	}
+	if _, minted, _ := inv.BusUserHash(ctx, "laptop.mesh-mail.send"); minted {
+		t.Fatal("the account was minted before the module gained the process")
+	}
+
+	manifest, _ := json.Marshal(gained)
+	build := func(id, commit string) {
+		t.Helper()
+		if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, link.BuildResult{
+			ID: id, Repository: "http://forge.internal:20000/novox/mesh-mail.git", Path: "modules/mesh-mail",
+			Ref: "main", On: "anchor", Commit: commit, Manifest: manifest,
+			Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-mail"},
+		})); err != nil {
+			t.Fatal(err)
+		}
+	}
+	build("b-1", "aaaaaaa1111")
+	hash, held, err := inv.BusUserHash(ctx, "laptop.mesh-mail.send")
+	if err != nil || !held {
+		t.Fatalf("1226/AC2: recording the gained definition did not mint the process's account: held %v, %v", held, err)
+	}
+	// And the next compose carries the account, with no warning (1226/AC2).
+	var said bytes.Buffer
+	if _, err := prepareSend(leftOutSaidBy(ctx, &said), open, "laptop"); err != nil {
+		t.Fatalf("1226/AC2: the next compose failed: %v", err)
+	}
+	if strings.Contains(said.String(), "bus account is not issued") {
+		t.Fatalf("1226/AC2: the next compose warns of an account just minted: %q", said.String())
+	}
+	// The row the compose reads is the account issueProcess accepted (origin accepted), never a value the mesh
+	// made for it (a stand-in).
+	in, err := inputFor(ctx, open)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var row *compose.OwnSecretRow
+	for i, r := range in.OwnSecrets {
+		if r.Node == "laptop" && r.Module == "mesh-mail" && r.Name == "broker-send" {
+			row = &in.OwnSecrets[i]
+		}
+	}
+	if row == nil {
+		t.Fatal("1226/AC2: the next compose carries no row for the process's account")
+	}
+	if row.Origin != inventory.OriginAccepted {
+		t.Fatalf("1226/AC2: the process's account is a stand-in the mesh made (origin %q), not the issued account", row.Origin)
+	}
+	build("b-2", "bbbbbbb2222")
+	again, _, err := inv.BusUserHash(ctx, "laptop.mesh-mail.send")
+	if err != nil || again != hash {
+		t.Fatalf("1226/AC3: recording again replaced the issued account (%v)", err)
+	}
+}
+
+// 1226/AC1 and AC4 against the store: given two modules on a machine, one with an unissued process account, when
+// the machine is composed, the rest of it is composed and sent, and only the warning in the push's answer names
+// the first, with `mesh-controller.issue <node> <module>`.
+func TestAnUnissuedProcessAccountIsWarnedOfAndTheRestIsComposed(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	_, gained := gainedAndPlain(t)
+	register(t, open, gained)
+	other := catalogue.Manifest{Module: "other", Version: "1",
+		OwnSecrets: catalogue.OwnSecrets{"token": {Path: "/var/lib/other/token"}}}
+	register(t, open, other)
+	for _, m := range []string{"mesh-mail", "other"} {
+		if _, err := assign(ctx, open, "laptop", m); err != nil {
+			t.Fatal(err)
+		}
+	}
+	var said bytes.Buffer
+	if _, err := prepareSend(leftOutSaidBy(ctx, &said), open, "laptop"); err != nil {
+		t.Fatalf("1226/AC4: an unissued process account held the machine: %v", err)
+	}
+	want := "laptop/mesh-mail process send: its bus account is not issued — mesh-controller.issue laptop mesh-mail"
+	if !strings.Contains(said.String(), want) {
+		t.Fatalf("1226/AC1: the push says %q, want %q", said.String(), want)
+	}
+	if strings.Contains(said.String(), "other") {
+		t.Fatalf("1226/AC4: the other module is warned of: %q", said.String())
+	}
+	if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "other", "token"); err != nil || !held {
+		t.Fatalf("1226/AC4: the other module's secret was not composed: held %v, %v", held, err)
+	}
+}

cmd/mesh-controller/process_accounts_test.go

@@ -0,0 +1,36 @@
+package main
+
+import "testing"
+
+// 1226/AC1: given a module whose process declares bus and has no issued account, when a machine is composed, the
+// push's warning names issue, and the machine's other modules are composed (AC4: nothing is refused).
+func TestAProcessAccountNobodyIssuedIsWarnedOfNamingIssue(t *testing.T) {
+	m := mailSenderWithProcess()
+	got := processAccountWarning(m, "laptop", "broker-send", nil)
+	want := "laptop/mesh-mail process send: its bus account is not issued — mesh-controller.issue laptop mesh-mail"
+	if got != want {
+		t.Fatalf("1226/AC1: the warning is %q, want %q", got, want)
+	}
+	// Other names are not this rule's.
+	if processAccountWarning(m, "laptop", "broker", nil) != "" || processAccountWarning(m, "laptop", "broker-other", nil) != "" ||
+		processAccountWarning(m, "laptop", "token", nil) != "" {
+		t.Fatal("1226/AC1: a name that is no declared process's account was warned of")
+	}
+	if got := processAccountWarning(m, "laptop", "broker-send", []string{"laptop.mesh-mail.send"}); got != "" {
+		t.Fatalf("1226/AC3: an issued account was warned of: %q", got)
+	}
+}
+
+// 1226/AC3: an account already issued is not issued again by a later delivery or assignment.
+func TestAnIssuedProcessAccountIsNotIssuedAgain(t *testing.T) {
+	m := mailSenderWithProcess()
+	if got := processAccountsMissing(m, "laptop", nil); len(got) != 1 || got[0].Process != "send" {
+		t.Fatalf("1226/AC3: an account not issued is not missing: %v", got)
+	}
+	if got := processAccountsMissing(m, "laptop", []string{"laptop.mesh-mail.send"}); len(got) != 0 {
+		t.Fatalf("1226/AC3: an issued account would be issued again: %v", got)
+	}
+	if got := processAccountsMissing(m, "ace", []string{"laptop.mesh-mail.send"}); len(got) != 1 {
+		t.Fatalf("1226/AC3: another machine's account stands in for this one's: %v", got)
+	}
+}

cmd/mesh-controller/push.go

@@ -282,6 +282,10 @@ func serve(ctx context.Context) (err error) {
 	// assignment.
 	inv.SetProvisioningPlans(inv.SentProvisioningPlans)
 	go mirroringRecords(ctx, inv, bus.Conn, bus)
+	// And the bucket of the files that travel by digest, judged against what each machine's latest declaration names (ADR 0486 §3).
+	go sweepingDeclarationFiles(ctx, inv, bus.Conn)
+	// And the key of each plan in its build phase, which holds the build seat's checks while a delivery builds (ADR 0483 clause 2).
+	go keepingPlansBuilding(ctx, inv, bus.Conn)
 	// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
 	// to-be 45 §6).
 	link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
@@ -979,6 +983,18 @@ func deliver(ctx context.Context, d delivery, holder string, sending []readyNode
 			return err
 		}
 		digest, err := d.declare(ctx, s, body)
+		if errors.Is(err, errFilesNotPut) && s.declared.fileReferences {
+			// **The grant that lets the controller write the bucket travels in the very declaration of the machine holding the
+			// bus** (novox/hq ADR 0486 §2): until the bus has it, a put is refused, and the declaration that would deliver the
+			// grant would wait for it. So that send, and only that, goes with every content inline, as to a node that reads no
+			// references, and says so; the next one tries the bucket again.
+			fmt.Printf("%s: the files that travel by digest could not be put in the bucket (%v): sending them inline\n", s.node, err)
+			s.declared.fileReferences = false
+			if body, err = s.declared.Body(); err != nil {
+				return err
+			}
+			digest, err = d.declare(ctx, s, body)
+		}
 		if err != nil {
 			return err
 		}
@@ -1040,6 +1056,35 @@ type overTheBus struct {
 	indent string
 }
 
+// putFiles puts the contents s names by digest in the bucket.
+func (b overTheBus) putFiles(ctx context.Context, s readyNode) error {
+	objects, err := s.declared.objects()
+	if err != nil {
+		return fmt.Errorf("%s: the files that travel by digest cannot be read: %w", s.node, err)
+	}
+	if len(objects) == 0 {
+		return nil
+	}
+	bus, ok := b.server.Bus().(link.OverNATS)
+	if !ok {
+		return fmt.Errorf("%s: the declaration names files by digest and this bus has no object bucket", s.node)
+	}
+	putting, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	store, err := openBucket(putting, bus.Conn, s.node)
+	if err == nil {
+		err = putObjects(putting, store, objects)
+	}
+	notePut(ctx, s.node, err)
+	if err != nil {
+		return fmt.Errorf("%s was not sent its declaration: %w: %w", s.node, errFilesNotPut, err)
+	}
+	return nil
+}
+
+// errFilesNotPut marks a send that stopped because the files it names by digest could not be put in the bucket.
+var errFilesNotPut = errors.New("the files it names by digest are not in the bucket")
+
 func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
 	// The bus's objects first, which every declaration implies (novox/hq issue 208): said and raised
 	// when they cannot be, and never what holds the send back (assertOnSend).
@@ -1052,9 +1097,20 @@ func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
 }
 
 func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
-	if err := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second); err != nil {
+	// The contents the declaration names by digest are in the bucket before it is sent (novox/hq ADR 0486 §1, issue 1696): a node
+	// that applies it finds every one. A node reading no references has none.
+	// **Under the node's lock** from the put to the publication, so the sweep of its bucket cannot remove an object between them.
+	unlock := lockFilesOf(s.node)
+	err := b.putFiles(ctx, s)
+	if err != nil {
+		unlock()
 		return "", err
 	}
+	declared := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second)
+	unlock()
+	if declared != nil {
+		return "", declared
+	}
 	// After it is away, not before. A digest recorded for something that failed to send would make
 	// the machine look current for a declaration it never received.
 	digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch,

cmd/mesh-controller/queue.go

@@ -91,7 +91,9 @@ type queueAnswer struct {
 	Dead       int            `json:"dead"`
 	MaxDeliver int            `json:"max-deliver"`
 	Asks       []queuedAnswer `json:"asks"`
-	Text       string         `json:"text"`
+	// HeldFor are the delivery plans building, while which the waiting checks are held (ADR 0483 clause 5); empty when none.
+	HeldFor []string `json:"held-for,omitempty"`
+	Text    string   `json:"text"`
 }
 
 // queuedAnswer is one ask as data. Never what the request carries beyond its source — not `held`, every
@@ -120,6 +122,9 @@ type queuedAnswer struct {
 	// Delivery is the how-manyth time an ask in flight was handed out; more than 1 is a restart
 	// (novox/hq issue 594).
 	Delivery int `json:"delivery,omitempty"`
+	// LetThrough is a check that waited past 30 minutes and is let through while a delivery builds: the next one, or the one
+	// that runs (ADR 0483 clause 5).
+	LetThrough bool `json:"let-through,omitempty"`
 }
 
 // aCommitRef is a ref that names a commit by its full hash (SHA-1 or SHA-256).
@@ -130,7 +135,8 @@ var aCommitRef = regexp.MustCompile(`^(?:[0-9a-f]{40}|[0-9a-f]{64})$`)
 func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
 	waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
 	out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead),
-		MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)}
+		MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, HeldFor: q.Hold.Plans, Text: queueText(q, now)}
+	letNext, letRunning := q.LetThroughOf(now)
 	one := func(a link.QueuedAsk, position int) queuedAnswer {
 		kind := link.KindOf(a.Request)
 		since := a.AskedAt
@@ -143,7 +149,7 @@ func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
 		}
 		return queuedAnswer{ID: a.ID, State: a.State, Kind: kind, Repository: link.WithoutPassword(a.Repository), Path: a.Path,
 			Ref: a.Ref, Commit: commit, Machine: a.On, WasOn: a.Was, AskedAt: a.AskedAt, Since: since,
-			Position: position, Seq: a.Seq, Delivery: a.Delivery}
+			Position: position, Seq: a.Seq, Delivery: a.Delivery, LetThrough: a.ID != "" && (a.ID == letNext || a.ID == letRunning)}
 	}
 	for _, a := range running {
 		out.Asks = append(out.Asks, one(a, 0))
@@ -179,6 +185,11 @@ func queueText(q link.Queue, now time.Time) string {
 		}
 		return s
 	}
+	letNext, letRunning := q.LetThroughOf(now)
+	const passed = "waited past 30 minutes, let through"
+	if q.HoldFault != "" {
+		fmt.Fprintf(&b, "why checks wait cannot be read, so none is said held: %s\n", q.HoldFault)
+	}
 	if len(running) > 0 {
 		fmt.Fprintln(&b, "\nin flight:")
 		for _, a := range running {
@@ -193,14 +204,27 @@ func queueText(q link.Queue, now time.Time) string {
 			case a.Was != "":
 				where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
 			}
-			fmt.Fprintf(&b, "  %-26s %-8s %s — %s, %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), where,
-				ago(a.AskedAt), a.Seq)
+			note := ""
+			if a.ID != "" && a.ID == letRunning {
+				note = " — " + passed
+			}
+			fmt.Fprintf(&b, "  %-26s %-8s %s — %s, %s (seq %d)%s\n", a.ID, link.KindOf(a.Request), what(a), where,
+				ago(a.AskedAt), a.Seq, note)
 		}
 	}
 	if len(waiting) > 0 {
 		fmt.Fprintln(&b, "\nwaiting, in the order they are handed out:")
+		// **Why the checks wait** (ADR 0483 clause 5): above them, the plans they are held for.
+		if len(q.Hold.Plans) > 0 && hasCheck(waiting) {
+			fmt.Fprintf(&b, "  checks are held while delivery %s builds; the oldest check that waited 30 minutes goes first, one at a time\n",
+				strings.Join(q.Hold.Plans, ", "))
+		}
 		for _, a := range waiting {
-			fmt.Fprintf(&b, "  %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
+			note := ""
+			if a.ID != "" && a.ID == letNext {
+				note = " — " + passed + " next"
+			}
+			fmt.Fprintf(&b, "  %-26s %-8s %s — %s (seq %d)%s\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq, note)
 		}
 	}
 	if len(dead) > 0 {
@@ -218,6 +242,16 @@ func queueText(q link.Queue, now time.Time) string {
 	return b.String()
 }
 
+// hasCheck says a check is among the asks.
+func hasCheck(asks []link.QueuedAsk) bool {
+	for _, a := range asks {
+		if a.Request.Check != nil {
+			return true
+		}
+	}
+	return false
+}
+
 // cancelCommand drops one waiting or dead ask.
 func cancelCommand(ctx context.Context, args []string) error {
 	if len(args) != 1 {

cmd/mesh-controller/queue_held_test.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/link"
+)
+
+// Issue 1688 / 1652 AC5 (ADR 0483 clause 5): the queue names, above the waiting checks, the plan they are held for, and says of a
+// check let through that it waited past 30 minutes.
+
+func aHeldQueue(now time.Time) link.Queue {
+	check := func(id string, ago time.Duration, n int) link.QueuedAsk {
+		return link.QueuedAsk{Seq: uint64(n), ID: id, Repository: "ssh://forge/novox/r", AskedAt: now.Add(-ago), State: link.AskWaiting,
+			Request: link.BuildRequest{ID: id, Check: &link.CheckRequest{Owner: "novox", Repo: "r", Number: n}}}
+	}
+	return link.Queue{Seat: link.TheBuildMachine, Asks: []link.QueuedAsk{
+		check("build-young", 5*time.Minute, 1),
+		check("build-old", 45*time.Minute, 2),
+		check("build-older", 50*time.Minute, 3),
+	}, Hold: link.CheckHold{Plans: []string{"plan-42"}}}
+}
+
+func TestTheQueueNamesThePlanTheChecksAreHeldFor(t *testing.T) {
+	now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
+	text := queueText(aHeldQueue(now), now)
+	heldAt := strings.Index(text, "held while delivery plan-42 builds")
+	firstAt := strings.Index(text, "build-young")
+	if heldAt < 0 || firstAt < 0 || heldAt > firstAt {
+		t.Fatalf("the plan the checks are held for is not named above the waiting checks:\n%s", text)
+	}
+	// Nothing building: nothing held.
+	q := aHeldQueue(now)
+	q.Hold = link.CheckHold{}
+	if strings.Contains(queueText(q, now), "held while") {
+		t.Fatal("checks are said held with no delivery building")
+	}
+}
+
+func TestACheckLetThroughIsSaidToHaveWaitedPastThirtyMinutes(t *testing.T) {
+	now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
+	q := aHeldQueue(now)
+	text := queueText(q, now)
+	var saidOf []string
+	for _, l := range strings.Split(text, "\n") {
+		if strings.Contains(l, "waited past 30 minutes, let through") {
+			saidOf = append(saidOf, strings.Fields(l)[0])
+		}
+	}
+	if len(saidOf) != 1 || saidOf[0] != "build-older" {
+		t.Fatalf("the check let through next (the oldest) is not the one said: %v\n%s", saidOf, text)
+	}
+	answer := queueAnswerOf(q, now)
+	if len(answer.HeldFor) != 1 || answer.HeldFor[0] != "plan-42" {
+		t.Errorf("held-for = %v", answer.HeldFor)
+	}
+	for _, a := range answer.Asks {
+		if a.LetThrough != (a.ID == "build-older") {
+			t.Errorf("%s let-through = %v", a.ID, a.LetThrough)
+		}
+	}
+	// A check in flight under the let-through key is said too.
+	q.Asks = append(q.Asks, link.QueuedAsk{Seq: 9, ID: "build-running", State: link.AskInFlight, On: "ace", AskedAt: now.Add(-time.Hour),
+		Started: now.Add(-time.Minute), Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "r", Number: 9}}})
+	q.Hold.LetThrough = &link.LetThrough{Builder: "ace"}
+	if !strings.Contains(queueText(q, now), "build-running") || !strings.Contains(queueText(q, now), "waited past 30 minutes, let through\n") {
+		t.Errorf("the check in flight let through is not said:\n%s", queueText(q, now))
+	}
+	// Younger than the bound: none is said.
+	young := aHeldQueue(now)
+	young.Asks = young.Asks[:1]
+	if strings.Contains(queueText(young, now), "waited past") {
+		t.Error("a check of 5 minutes is said to have waited past 30")
+	}
+}

cmd/mesh-controller/recorded_moves.go

@@ -181,15 +181,24 @@ func planModuleNames(p inventory.Plan) map[string]string {
 	return out
 }
 
-// recordedWaiting is, on one machine, every move of a recorded module that waits for a person, each said with why
-// (ADR 0377 §6): what `status` names the machine behind for, with `push <node>` as the remedy. A move under a
-// module's own `record` that recreates nothing and moves no data is not among them: it goes with its walk.
-func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]string, error) {
+// heldMove is one move of a recorded module that waits for a person on one machine (ADR 0377 §6).
+type heldMove struct {
+	Module   string
+	From, To string
+	// Why is the walk's own word for it: "waits: " and the recreate words, the data change, or the hold.
+	Why string
+	// Chosen is whether a person's own choice holds it — a module they set to record, or the bus's step — which the
+	// operator is not asked about again: they chose that hold, and the bus has its own condition (S17).
+	Chosen bool
+}
+
+// heldMovesOn is, on one machine, every move of a recorded module that waits for a person, in module order.
+func heldMovesOn(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]heldMove, error) {
 	sent, known, err := inv.SentBuilds(ctx, node)
 	if err != nil || !known {
 		return nil, err
 	}
-	var out []string
+	var out []heldMove
 	for _, m := range sortedKeysOf(sent) {
 		was := sent[m]
 		now, held := f.current[m]
@@ -197,12 +206,25 @@ func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts,
 			continue
 		}
 		if v := f.recordedMove(m, was); !v.rolls {
-			out = append(out, fmt.Sprintf("%s %s → %s %s", m, short(was), short(now.Commit), v.said))
+			out = append(out, heldMove{Module: m, From: was, To: now.Commit, Why: v.said,
+				Chosen: now.From == catalogue.FromPerson || now.From == catalogue.FromBus})
 		}
 	}
 	return out, nil
 }
 
+// recordedWaiting is, on one machine, every move of a recorded module that waits for a person, each said with why
+// (ADR 0377 §6): what `status` names the machine behind for, with `push <node>` as the remedy. A move under a
+// module's own `record` that recreates nothing and moves no data is not among them: it goes with its walk.
+func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]string, error) {
+	moves, err := heldMovesOn(ctx, inv, f, node)
+	var out []string
+	for _, h := range moves {
+		out = append(out, fmt.Sprintf("%s %s → %s %s", h.Module, short(h.From), short(h.To), h.Why))
+	}
+	return out, err
+}
+
 // stringsKeys is a map of lists as a map of its keys, for sortedKeysOf.
 func stringsKeys(m map[string][]string) map[string]string {
 	out := map[string]string{}

cmd/mesh-controller/release.go

@@ -1371,6 +1371,10 @@ func backlogObservation() []conditions.Observation {
 			"them, `upgrade release-backlog --why …` releases them", n, strings.Join(machines, ", "), backlogNow.held)}
 	w := releaseHeldWords(modules, machines)
 	o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs
+	// Asked of the operator, who releases them with an Approve (novox/hq issue 1732): the controller's own
+	// `upgrade release-backlog`, which opens the release plan that walks them through the machines, each judged.
+	o.Actions = []conditions.Action{{Label: "Release", Verb: "mesh-controller.upgrade", Level: conditions.LevelApprove,
+		Arguments: map[string]string{"release-backlog": "true", "why": "", "cause": conditions.CauseOperatorAnswer}}}
 	return []conditions.Observation{o}
 }
 

cmd/mesh-controller/release_plan.go

@@ -577,6 +577,9 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
 	}
 	// **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull
 	// requests it answers, whose checks name the part of a differing key.
+	// **The plan's key is there before its first build is** (ADR 0483 clause 2): a check taken between the ask and the plan's save
+	// would hold a place for its whole run.
+	sayBuilding(ctx, p.ID, now)
 	id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, p.ID, reuseAskOf(p), mergedHeadsOf(p))), source,
 		e.Source.Path, ref)
 	if err != nil {
@@ -625,16 +628,124 @@ func planBuilt(ctx context.Context, open *stores, module, commit, failed string,
 	planBuiltSaying(ctx, open, module, commit, failed, asked, id, "")
 }
 
+// holdPlansForOutcome is how an outcome taken on the event loop holds the plans: **without waiting for them**
+// (novox/hq issue 1708). The loop takes one message at a time, and the plans' holder may keep them for minutes while
+// it sends machines their declarations; an outcome that waited its turn (up to inventory.HoldWaitFor, two minutes)
+// held every report, build and merge behind it for as long, and the watchdog said the controller stopped listening
+// (2026-10-21 14:20:45 to 14:22:45). Busy, the outcome is left to the build records, where it already is, and the
+// look settles the plan from them (leaveToRecords).
+func holdPlansForOutcome(ctx context.Context, hold func(context.Context, bool) (func(), error)) (func(), error) {
+	return hold(ctx, false)
+}
+
+// leaveToRecords says an outcome was not taken into the plans, which settle it from the build records (novox/hq issue
+// 214). When another act holds the plans it asks for the look, which waits for them off the event loop and settles
+// the plan from the records once they are let go (issue 1708); what the build did is kept for that settle by the
+// build's id, so the walk's module says it as it would have (ADR 0350 §4).
+func leaveToRecords(module, id, made, failed string, err error) {
+	fmt.Printf("plans: %s's outcome is left to the build records: %v\n", module, err)
+	if !errors.Is(err, inventory.ErrPlansBusy) {
+		return
+	}
+	keepMadeFor(id, made, time.Now())
+	// A refused outcome (failed, its build recorded as worked) is kept as refused, or the records settle it built.
+	keepRefusalFor(id, failed, time.Now())
+	askToLook()
+}
+
+// refusalsKeptFor is the refusals kept in this process for these builds (keepRefusalFor), by id.
+func refusalsKeptFor(ids []string) map[string]string {
+	out := map[string]string{}
+	refusedLeftMu.Lock()
+	defer refusedLeftMu.Unlock()
+	for _, id := range ids {
+		if k, ok := refusedLeft[id]; ok {
+			out[id] = k.made
+		}
+	}
+	return out
+}
+
+// refusedLeft is why each outcome left to the records was refused, by the build's id: kept in the process beside the
+// store's record (RecordRefused), so a store that did not take it still fails the plan while this controller serves.
+var (
+	refusedLeftMu sync.Mutex
+	refusedLeft   = map[string]madeKept{}
+)
+
+// keepRefusalFor keeps why an outcome left to the records was refused, forgetting refusals older than madeLeftFor.
+func keepRefusalFor(id, why string, now time.Time) {
+	if id == "" || why == "" {
+		return
+	}
+	refusedLeftMu.Lock()
+	defer refusedLeftMu.Unlock()
+	for k, v := range refusedLeft {
+		if now.Sub(v.at) > madeLeftFor {
+			delete(refusedLeft, k)
+		}
+	}
+	refusedLeft[id] = madeKept{made: why, at: now}
+}
+
+// madeLeft is what each build left to the records did, by the build's id, until a settle takes it: kept in the
+// process, so a controller replaced before the settle loses only these words, never the outcome.
+var (
+	madeLeftMu sync.Mutex
+	madeLeft   = map[string]madeKept{}
+)
+
+type madeKept struct {
+	made string
+	at   time.Time
+}
+
+// madeLeftFor is how long words nobody settled are kept: a plan that ended or moved on never asks for them.
+const madeLeftFor = time.Hour
+
+// keepMadeFor keeps what a build left to the records did for the settle that takes it, forgetting words older than
+// madeLeftFor.
+func keepMadeFor(id, made string, now time.Time) {
+	if id == "" || made == "" {
+		return
+	}
+	madeLeftMu.Lock()
+	defer madeLeftMu.Unlock()
+	for k, v := range madeLeft {
+		if now.Sub(v.at) > madeLeftFor {
+			delete(madeLeft, k)
+		}
+	}
+	madeLeft[id] = madeKept{made: made, at: now}
+}
+
+// takeMadeOf is what the build of this id did, if its outcome was left to the records with words; taken once.
+func takeMadeOf(id string) string {
+	madeLeftMu.Lock()
+	defer madeLeftMu.Unlock()
+	k, ok := madeLeft[id]
+	if !ok {
+		return ""
+	}
+	delete(madeLeft, id)
+	return k.made
+}
+
 // planBuiltSaying is planBuilt keeping what the build did — "reused from <check build>" or "built: <why>" (novox/hq
 // ADR 0350 §4) — on the module of the walk it answers.
 func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed string, asked time.Time, id, made string) {
 	inv := open.inventory
-	// One controller works the plans at a time (novox/hq issue 213); an outcome waits its turn rather
-	// than write over what the holder is about to save. Not taken, it is still in the build records,
-	// which the holder settles the plan from (issue 214).
-	release, err := inv.HoldPlans(ctx, true)
+	// One controller works the plans at a time (novox/hq issue 213). Not taken, the outcome is still in the build
+	// records, which the holder settles the plan from (issue 214).
+	release, err := holdPlansForOutcome(ctx, inv.HoldPlans)
 	if err != nil {
-		fmt.Printf("plans: %s's outcome is left to the build records: %v\n", module, err)
+		if failed != "" && id != "" && errors.Is(err, inventory.ErrPlansBusy) {
+			// In the store too, so a controller replaced before the look settles still fails the plan (issue 1708).
+			if err := inv.RecordRefused(ctx, id, module, failed); err != nil {
+				fmt.Printf("plans: %s's refusal could not be kept in the store, only in this controller: %v\n", module, err)
+			}
+		}
+		leaveToRecords(module, id, made, failed, err)
 		return
 	}
 	defer release()
@@ -696,7 +807,19 @@ func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed s
 			fmt.Printf("%s: %s; the tiers after it are not asked\n", p.ID, p.Note)
 		}
 	}
-	advanceHeld(ctx, open)
+	advanceAfterOutcome(func() { advanceHeld(ctx, open) })
+}
+
+// advanceAfterOutcome moves the plans after an outcome was taken into them: **in the serving controller, by asking
+// the look** (novox/hq issue 1708). The outcome is taken on the event loop, and moving the plans may send machines
+// their declarations, a release's send retried for minutes among them; done here it held every report, build and
+// merge behind it (2026-10-21 14:28:40 to 14:32:10). The look runs off the loop, holding the plans itself. Where no
+// look runs (a command, which has no event loop) the plans are moved at once, as before.
+func advanceAfterOutcome(advance func()) {
+	if askedToLook() {
+		return
+	}
+	advance()
 }
 
 // startedBeside is the id of a started walk open beside this one that it is related to — a merge's walk past
@@ -731,6 +854,23 @@ func startedBeside(ctx context.Context, inv *inventory.Inventory, p inventory.Pl
 	return behind, why, nil
 }
 
+// refusalNotePrefix and refusalNoteSuffix frame the note of a step the walk was refused: "tier N: <refusal> — tried
+// again". Only a note so framed is a hold (novox/hq issue 920); a tier's summary left in the note is not.
+const (
+	refusalNotePrefix = "tier "
+	refusalNoteSuffix = " — tried again"
+)
+
+// refusalNote is the note of a plan whose step at tier was refused with err.
+func refusalNote(tier int, err error) string {
+	return refusalNotePrefix + fmt.Sprint(tier) + ": " + err.Error() + refusalNoteSuffix
+}
+
+// refused says a plan's note is a refusal the walk wrote.
+func refused(p inventory.Plan) bool {
+	return strings.HasPrefix(p.Note, refusalNotePrefix) && strings.HasSuffix(p.Note, refusalNoteSuffix)
+}
+
 // deferredNote begins the note of a walk deferred behind another.
 const deferredNote = "let go; starts once "
 
@@ -743,6 +883,20 @@ func deferred(p inventory.Plan) bool {
 		p.Delivery.Awaits != "" && p.Delivery.Go != nil && strings.HasPrefix(p.Note, deferredNote)
 }
 
+// currentTierBuilt says every module of a plan's current tier is built (or deleted), so no build is what it waits on.
+func currentTierBuilt(p inventory.Plan) bool {
+	if p.Tier < 0 || p.Tier >= len(p.Tiers) || len(p.Tiers[p.Tier]) == 0 {
+		return false
+	}
+	for _, m := range p.Tiers[p.Tier] {
+		s := p.Modules[m]
+		if s == nil || (s.State != "built" && s.State != planDeleted) {
+			return false
+		}
+	}
+	return true
+}
+
 // askedAny says a plan asked any module.
 func askedAny(p inventory.Plan) bool {
 	for _, s := range p.Modules {
@@ -807,7 +961,7 @@ func advanceHeld(ctx context.Context, open *stores) (failed bool) {
 				// Kept in the plan, so `plans` says why it has not moved rather than the log alone;
 				// the state is left as it was and the step is tried again by the next look. Said and
 				// kept when it is new: the same refusal on every tick is one fact, not one per tick.
-				p.Note = "tier " + fmt.Sprint(p.Tier) + ": " + err.Error() + " — tried again"
+				p.Note = refusalNote(p.Tier, err)
 				// A refusal for the bus's planned step is a person's to end: S17 says it from its first moment.
 				if refusedForTheBus(err) {
 					busRefusedFirst.mark(p.ID, time.Now())
@@ -921,7 +1075,17 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
 	if err != nil {
 		return false, err
 	}
+	before, was := statesOf(p), p.State
 	if settleFromRecords(p, tier, recorded, byID) {
+		// As taking the outcome directly does (planBuiltSaying): a module deleted at its source is forgotten, and a
+		// failed plan says what it built and never sent (issue 1708).
+		deleted, failed := settledEffects(before, p, was)
+		for _, m := range deleted {
+			forgetDeleted(ctx, inv, m, p)
+		}
+		if failed {
+			sayUnsent(p, rollsOut)
+		}
 		return true, nil
 	}
 	// Asked: wait for every build.
@@ -1895,6 +2059,19 @@ func planLineSaid(p inventory.Plan, now time.Time, pause pauseView, bound time.D
 	if p.State == inventory.PlanRolling {
 		what = p.Note
 	}
+	if p.State == inventory.PlanBuilding && refused(p) && currentTierBuilt(p) {
+		// Every build of the tier is in, so the plan waits on something else, and its note says what: a send
+		// refused ("a build waits there for its gate: <machine> (...)") is kept there. "building" with LATE
+		// hid that reason from `plans`, `status` and the stalled condition (novox/hq issue 920).
+		what = p.Note
+		// And when the builds are held after a failed release, they wait for a person, not for a build, and the
+		// operator is asked to release them (novox/hq issue 1732): said so, and not late.
+		if strings.Contains(p.Note, errUngated.Error()) && backlogNow.held != "" {
+			what += " — held after a failed delivery until a person releases it (the operator is asked)"
+			late = ""
+		}
+		return fmt.Sprintf("%s · %s, held: %s, for %s%s", name, where, what, since, late)
+	}
 	return fmt.Sprintf("%s · %s, %s for %s%s", name, where, what, since, late)
 }
 
@@ -2100,7 +2277,7 @@ func plansCommand(ctx context.Context, args []string) error {
 			bounds.of(p.Repository), func(module string) []string {
 				on, _ := inv.Running(ctx, module)
 				return on
-			}, waitsOf(ctx, inv, []inventory.Plan{p})))
+			}, waitsOf(ctx, inv, []inventory.Plan{p}))+heldSaid(p, heldNow(ctx, inv)))
 		if r := p.Release; r != nil {
 			// A release plan's walk (ADR 0236): machines done, the one judged, those to come.
 			fmt.Printf("  machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "),
@@ -2295,11 +2472,12 @@ func plansCommand(ctx context.Context, args []string) error {
 		return on
 	}
 	waits := waitsOf(ctx, inv, plans)
+	held := heldNow(ctx, inv)
 	for _, p := range plans {
 		if p.Batch() {
 			continue
 		}
-		fmt.Printf("%-28s %s\n", p.ID, planLineSaid(p, now, pause, bounds.of(p.Repository), running, waits))
+		fmt.Printf("%-28s %s%s\n", p.ID, planLineSaid(p, now, pause, bounds.of(p.Repository), running, waits), heldSaid(p, held))
 	}
 	return nil
 }
@@ -2479,9 +2657,81 @@ func recordsOfAsked(ctx context.Context, inv *inventory.Inventory, p *inventory.
 			}
 		}
 	}
+	refused, err := refusalsOf(ctx, inv, recorded, byID)
+	if err != nil {
+		return nil, nil, err
+	}
+	overlayRefusals(recorded, byID, refused)
 	return recorded, byID, nil
 }
 
+// overlayRefusals marks failed every build record whose registration the mesh refused (novox/hq issue 1708): the record
+// keeps the builder's words, which worked, and settled from it as it is the refused build would be built and sent.
+func overlayRefusals(recorded map[string][]inventory.Build, byID map[string]inventory.Build, refused map[string]string) {
+	if len(refused) == 0 {
+		return
+	}
+	fail := func(b *inventory.Build) {
+		if why, ok := refused[b.ID]; ok && b.Worked() {
+			b.Failed = why
+		}
+	}
+	for m := range recorded {
+		for i := range recorded[m] {
+			fail(&recorded[m][i])
+		}
+	}
+	for id, b := range byID {
+		fail(&b)
+		byID[id] = b
+	}
+}
+
+// refusalsOf is every refusal kept for the builds these records name, in the store and in this process.
+func refusalsOf(ctx context.Context, inv *inventory.Inventory, recorded map[string][]inventory.Build,
+	byID map[string]inventory.Build) (map[string]string, error) {
+	var ids []string
+	for _, bs := range recorded {
+		for _, b := range bs {
+			ids = append(ids, b.ID)
+		}
+	}
+	for id := range byID {
+		ids = append(ids, id)
+	}
+	refused, err := inv.RefusedOf(ctx, ids)
+	if err != nil {
+		return nil, fmt.Errorf("the refused registrations cannot be read: %w", err)
+	}
+	for id, why := range refusalsKeptFor(ids) {
+		refused[id] = why
+	}
+	return refused, nil
+}
+
+// settledEffects is what settling from the records did that needs more than the plan: the modules it found deleted
+// at their source, and whether it failed the plan.
+func settledEffects(before map[string]string, p *inventory.Plan, was string) (deleted []string, failed bool) {
+	for m, s := range p.Modules {
+		if s != nil && s.State == planDeleted && before[m] != planDeleted {
+			deleted = append(deleted, m)
+		}
+	}
+	sort.Strings(deleted)
+	return deleted, p.State == inventory.PlanFailed && was != inventory.PlanFailed
+}
+
+// statesOf is each module's state in a plan, for settledEffects.
+func statesOf(p *inventory.Plan) map[string]string {
+	out := map[string]string{}
+	for m, s := range p.Modules {
+		if s != nil {
+			out[m] = s.State
+		}
+	}
+	return out
+}
+
 // settleFromRecords marks every module of the tier still `asked` built — or failed — from a build
 // recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214).
 // Newest first, as Builds answers: the first record after the ask is the outcome of that ask.
@@ -2531,6 +2781,10 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
 			s.State = "built"
 			s.BuiltAt = &at
 			s.Commit = outcome.Commit
+			// What the build did, when its outcome was left to the records with words (issue 1708).
+			if made := takeMadeOf(outcome.ID); made != "" {
+				s.Made = made
+			}
 		} else {
 			s.State = "failed"
 			s.Why = outcome.Failed

cmd/mesh-controller/sayable_test.go

@@ -152,7 +152,7 @@ func answersRightly(q dnsmessage.Question) *dnsmessage.Message {
 func quickResolvers(t *testing.T, port string) {
 	t.Helper()
 	before, within, pause := resolverPort, resolverWithin, resolverPause
-	resolverPort, resolverWithin, resolverPause = port, 150*time.Millisecond, 10*time.Millisecond
+	resolverPort, resolverWithin, resolverPause = port, 60*time.Millisecond, 5*time.Millisecond
 	t.Cleanup(func() { resolverPort, resolverWithin, resolverPause = before, within, pause })
 }
 

cmd/mesh-controller/seats_test.go

@@ -93,3 +93,27 @@ func TestTheSeatsListingShowsEachSeatsPaths(t *testing.T) {
 		}
 	}
 }
+
+// 1718/AC2: a manifest claiming node-files that serves upload and download passes the controller's module
+// check, judged by the mesh-sdk it vendors; a verb outside the contract is still refused.
+func TestAFilesHolderServingUploadAndDownloadCanHoldTheSeat(t *testing.T) {
+	seat, ok := catalogue.SeatNamed(catalogue.FilesSeat)
+	if !ok {
+		t.Fatal("node-files is not a seat")
+	}
+	serves := []string{"upload", "download"}
+	for _, v := range seat.Serves {
+		if v.Name != "upload" && v.Name != "download" {
+			serves = append(serves, v.Name)
+		}
+	}
+	holder := catalogue.Manifest{Module: "mesh-files", Version: "1", Claims: []catalogue.Claim{{
+		Name: catalogue.FilesSeat, Scope: catalogue.ScopeNode, Serves: serves}}}
+	if err := catalogue.CanHold(holder, seat); err != nil { // 1718/AC2
+		t.Fatalf("a holder serving upload and download is refused: %v", err)
+	}
+	holder.Claims[0].Serves = append(append([]string{}, serves...), "teleport")
+	if err := catalogue.CanHold(holder, seat); err == nil || !strings.Contains(err.Error(), "does not promise") {
+		t.Fatalf("a verb outside the contract must be refused with 'does not promise': %v", err)
+	}
+}

cmd/mesh-controller/sendable.go

@@ -39,6 +39,10 @@ type sendable struct {
 	// ReadsModules says the machine's node-engine reads a module per resource and a build per module (novox/hq issue
 	// 1145): only then are they sent, because an older node-engine refuses a key it does not know, whole.
 	ReadsModules bool
+	// fileReferences says the machine's node-engine reports the capability of reading a file given as a digest and a size
+	// (declaration.FileReferencesCapability, novox/hq ADR 0486 §5): then every file over 64 KiB that may travel so does (Body,
+	// objects). Not on the wire as a field; the bodies differ.
+	fileReferences bool
 	// moduleOf is the composer's owner of each resource id, and buildIDs the build each module carries (its id):
 	// kept beside the resources, and on the wire only to a machine that reads them.
 	moduleOf map[string]string
@@ -115,12 +119,16 @@ type adoptionEnvelope struct {
 
 // Body is the declaration's bytes, as sent and as digested.
 func (s sendable) Body() ([]byte, error) {
-	envelope := map[string]any{"declaration": 1, "resources": s.Resources}
+	resources, _, err := s.byReference()
+	if err != nil {
+		return nil, err
+	}
+	envelope := map[string]any{"declaration": 1, "resources": resources}
 	if s.ReadsModules {
 		// The module of each resource the composer owns (what the mesh adds of its own has none), and the build of each
 		// module (novox/hq issue 1145): to a machine that said it reads them, and no other.
-		named := make([]map[string]any, len(s.Resources))
-		for n, r := range s.Resources {
+		named := make([]map[string]any, len(resources))
+		for n, r := range resources {
 			module, owned := s.moduleOf[fmt.Sprint(r["id"])]
 			if !owned {
 				named[n] = r
@@ -165,3 +173,19 @@ func (s sendable) Body() ([]byte, error) {
 	}
 	return json.Marshal(envelope)
 }
+
+// byReference is the resources as the machine is sent them: with the contents of its large files replaced by their digests when
+// it reports it reads them (issue 1696), and the contents it replaced, by the name of the object that holds each.
+func (s sendable) byReference() ([]map[string]any, map[string][]byte, error) {
+	if !s.fileReferences {
+		return s.Resources, nil, nil
+	}
+	return referencing(s.Resources)
+}
+
+// objects are the contents the body names by digest, by the name of each one's object: what is put in the bucket before the
+// body is sent. Empty for a machine that is sent every content inline.
+func (s sendable) objects() (map[string][]byte, error) {
+	_, objects, err := s.byReference()
+	return objects, err
+}

cmd/mesh-controller/signals.go

@@ -244,6 +244,22 @@ var signalsTable = []signalRow{
 		newest: func(f *signalFacts) time.Time {
 			return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since })
 		}},
+	{Row: "S24", Signal: "a move of a recorded module that waits for a person's push is asked of the operator",
+		Emitter: "controller's plan", Trigger: "each machine with a move of a recorded module held for a person (novox/hq ADR 0377, issue 1732)",
+		Bound: "none: raised on the first look that finds the machine holding such a move, naming the modules and the " +
+			"machine, with Release (the machine's push) as the answer; ended when the machine no longer waits, whoever " +
+			"pushed it. A hold a person chose (a module set to record with a why, the bus) is not asked.",
+		Kind: kindMovesHeld, Severity: conditions.Warning, Phase: 3,
+		needs: func(f *signalFacts) error { return f.heldErr }, watch: watchMovesHeld,
+		newest: func(f *signalFacts) time.Time { return time.Time{} }},
+	{Row: "S25", Signal: "a module left out of a send because the artifact store lacks an image it names",
+		Emitter: "controller's send", Trigger: "each send whose declaration names an image of the store that the store does not hold (novox/hq issue 1742)",
+		Bound: "none: raised on the first look at a registered build that names such an image, naming the module, each " +
+			"machine that runs it and the image; every send leaves that module out; the store is asked again at each " +
+			"look and the condition ends when it holds the image",
+		Kind: kindImageMissing, Severity: conditions.Warning, Phase: 3,
+		needs: func(f *signalFacts) error { return f.imagesErr }, watch: watchMissingImages,
+		newest: func(f *signalFacts) time.Time { return time.Time{} }},
 	{Row: "S20", Signal: "a send refused for the assignment generation it was composed from",
 		Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)",
 		Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " +

cmd/mesh-controller/signals_test.go

@@ -171,6 +171,19 @@ var suppressions = map[string]suppression{
 				since: f.now.Add(-time.Second)}}
 		},
 	},
+	// A machine holding a move of a recorded module for a person is asked at its first look (novox/hq issue 1732).
+	// Inside: nothing waits there.
+	"S24": {
+		inside: func(f *signalFacts) { f.held = nil },
+		past:   func(f *signalFacts) { f.held = []heldFacts{aHeldMailu()} },
+	},
+	// A registered build naming an image the artifact store lacks is raised at the first look (novox/hq issue 1742).
+	"S25": {
+		inside: func(f *signalFacts) { f.images = nil },
+		past: func(f *signalFacts) {
+			f.images = []imageFacts{{missingImage{Node: "anchor", Module: "automx", Image: automxImage}}}
+		},
+	},
 	// A batch queued behind an open walk past that walk's bound, named.
 	"S19": {
 		inside: func(f *signalFacts) {

cmd/mesh-controller/upgrades.go

@@ -415,7 +415,59 @@ func askNewModules(ctx context.Context, inv *inventory.Inventory, m link.SourceM
 	if len(added) == 0 || len(from) == 0 {
 		return nil
 	}
-	return askBuilds(ctx, inv, m, buildSource{Repository: from[0].Source.Repository, Seat: from[0].Source.Seat}, added)
+	source := buildSource{Repository: from[0].Source.Repository, Seat: from[0].Source.Seat}
+	// Asked once (novox/hq issue 1617): a module is asked when its merge is heard, and again here when the batch is cut.
+	var requests []inventory.RequestOutcome
+	for _, dir := range added {
+		if got, err := inv.RequestsNamed(ctx, inventory.BuildRequest{Repository: source.Repository, Path: dir}.Name()); err == nil {
+			requests = append(requests, got...)
+		}
+	}
+	return askBuilds(ctx, inv, m, source, unaskedDirs(added, requests, source, time.Now()))
+}
+
+// askedInFlightFor is how long a merge's build request stands as the build coming for its module: past it, a lost one
+// is asked again.
+const askedInFlightFor = time.Hour
+
+// unaskedDirs are the directories of a new module no build of a merge is on its way for: a kept request of the same
+// repository and directory, asked for a merge and handed over, whose outcome has not come and is not given up on,
+// is the build coming (novox/hq issue 1617).
+func unaskedDirs(added []string, requests []inventory.RequestOutcome, source buildSource, now time.Time) []string {
+	var out []string
+	for _, dir := range added {
+		path := dir
+		if path == "." {
+			path = ""
+		}
+		coming := false
+		for _, r := range requests {
+			if r.For == "merge" && r.Repository == source.Repository && strings.Trim(r.Path, "/") == path &&
+				r.NotAsked == "" && r.OutcomeUnknown == "" && !r.Heard && now.Sub(r.At) < askedInFlightFor {
+				coming = true
+				break
+			}
+		}
+		if !coming {
+			out = append(out, dir)
+		}
+	}
+	return out
+}
+
+// askNewModulesAtHearing asks the first build of the modules a merge adds to a repository the mesh builds modules from,
+// when the merge is heard, not when its batch is cut (novox/hq issue 1617): a batch waits out its window and the walk
+// before it, and a module added by a merge delivered meanwhile could not be assigned. Outside the batch and its plan.
+func askNewModulesAtHearing(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, entries []inventory.Entry,
+	read map[string][]inventory.ReadRepository, from []inventory.Entry) {
+	if len(from) == 0 {
+		return
+	}
+	_, added, _ := touchedBy(from, entries, m, read)
+	if asked := askNewModules(ctx, inv, m, from, added); len(asked) > 0 {
+		fmt.Printf("%s/%s merged into %s (%.8s) adds %s: the first build is asked now, before its batch is cut\n",
+			m.Owner, m.Repo, m.Base, m.Commit, strings.Join(asked, ", "))
+	}
 }
 
 // askBuilds asks the first build of each directory of a merge's repository, as askNewModules does, from the source given.

cmd/mesh-controller/watchdogs.go

@@ -58,6 +58,12 @@ type signalFacts struct {
 	// failed are the delivery plans that failed and are asked of the operator (S23, novox/hq ADR 0449).
 	failed    []failedPlanFacts
 	failedErr error
+	// held are the machines with moves of recorded modules waiting for a person's push (S24, novox/hq issue 1732).
+	held    []heldFacts
+	heldErr error
+	// images are the modules left out of a send because the artifact store lacks an image they name (S25, issue 1742).
+	images    []imageFacts
+	imagesErr error
 	// refusedSends are the sends a machine refused within the hour for the generation they were composed
 	// from, each naming its sender (S20, novox/hq issue 234).
 	refusedSends    []inventory.Send
@@ -357,6 +363,8 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts {
 		f.batches, f.plansErr = gatherBatches(ctx, inv, now)
 	}
 	f.failed, f.failedErr = gatherFailedPlans(ctx, inv, now)
+	f.held, f.heldErr = gatherHeldMoves(ctx, inv)
+	f.images, f.imagesErr = gatherMissingImagesHere(ctx, inv)
 	f.loop, f.loopErr = w.gatherLoop()
 	f.asks, f.asksErr = w.gatherAsks(ctx, inv)
 	if open, err := w.keeper.Open(ctx); err != nil {

go.mod

@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
 go 1.26.0
 
 require (
-	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021105408-c8e35097e861
+	git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021204452-580da61a9107
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/nats-io/nats-server/v2 v2.11.17
 	github.com/nats-io/nats.go v1.54.0

go.sum

@@ -1,7 +1,9 @@
 git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4=
 git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021105408-c8e35097e861 h1:BsqolZMMsC5jLP8X88BwShqFssLgtREcgBIy6OfPSI8=
-git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021105408-c8e35097e861/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021173436-07a65927c362 h1:Ud/YHmhlm0qdB45LyDvxq7l4x6Bg2Ye0siHK5JFE+kk=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021173436-07a65927c362/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021204452-580da61a9107 h1:29NSUkDw7cIEwNcE09JZ9ASSD+cwBDoWRN3vWHKvsoQ=
+git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021204452-580da61a9107/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
 github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

internal/broker/checks.go

@@ -46,6 +46,7 @@ func CheckStreams() []Stream {
 			Retention:         RetentionWorkQueue,
 			MaxAge:            7 * 24 * 60 * 60,
 			MaxMsgsPerSubject: 1,
+			Direct:            true,
 			Why: "the checks of pull requests marked ready, taken after a delivery build and before any other check " +
 				"(novox/hq ADR 0406 §3); one subject a pull request, the newest request kept, so a newer one replaces a " +
 				"request still waiting",
@@ -56,6 +57,7 @@ func CheckStreams() []Stream {
 			Retention:         RetentionWorkQueue,
 			MaxAge:            7 * 24 * 60 * 60,
 			MaxMsgsPerSubject: 1,
+			Direct:            true,
 			Why: "every other check, taken last and in the order asked (novox/hq ADR 0406 §3); one subject a pull " +
 				"request, the newest request kept; one left behind for a head already judged is dropped when taken",
 		},
@@ -98,6 +100,13 @@ func holderCheckGrants() []string {
 		"$KV."+CheckRunningBucket+".>",
 		"$JS.API.STREAM.INFO.KV_"+CheckRunningBucket,
 		"$JS.API.DIRECT.GET.KV_"+CheckRunningBucket+".>")
+	// And, to hold checks while a delivery builds (ADR 0483, issues 1687 and 1688): the bucket of the plans building, **read and
+	// never written** (the controller's alone, so no `$KV.` subject of it is granted), and the oldest waiting ask of each check
+	// queue, read by a direct get that takes nothing from the queue and so counts against no delivery of the ask.
+	pub = append(pub, readGrants(PlansBuildingBucket, "")...)
+	for _, w := range CheckWorkers() {
+		pub = append(pub, "$JS.API.DIRECT.GET."+w.Stream, "$JS.API.DIRECT.GET."+w.Stream+".>")
+	}
 	return pub
 }
 

internal/broker/checks_test.go

@@ -40,3 +40,37 @@ func TestTheBuildAgentReadsAndWatchesThePullRequestsKeys(t *testing.T) {
 		}
 	}
 }
+
+// Issue 1687 (ADR 0483 clause 2): a holder of the build seat reads the bucket of the plans building and writes none of it, and
+// reads the oldest waiting ask of each check queue by a direct get, which takes nothing from the queue.
+func TestTheBuildSeatsHoldersReadThePlansBuildingAndWriteNone(t *testing.T) {
+	seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
+	p := Principal{Kind: KindModule, Node: "one", Module: "build-agent", Holds: []Seat{seat}, PasswordHash: "x"}
+	perms, err := PermissionsFor(p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	stream := "KV_" + PlansBuildingBucket
+	for _, s := range []string{
+		"$JS.API.STREAM.INFO." + stream,
+		"$JS.API.DIRECT.GET." + stream + ".>",
+		"$JS.API.CONSUMER.CREATE." + stream + ".>",
+		"$JS.API.DIRECT.GET.CHECKS_READY",
+		"$JS.API.DIRECT.GET.CHECKS_OTHER",
+	} {
+		has(t, perms.Publish, s)
+	}
+	for _, s := range perms.Publish {
+		if SubjectsOverlap(s, "$KV."+PlansBuildingBucket+".plan-1") {
+			t.Errorf("a holder of the build seat may write %s through %q", PlansBuildingBucket, s)
+		}
+	}
+	// And the controller alone writes it, as the writers table says.
+	for _, row := range WritersTable {
+		for _, s := range row.Subjects {
+			if s == "$KV."+PlansBuildingBucket+".>" && row.Writes(p, "$KV."+PlansBuildingBucket+".plan-1") {
+				t.Errorf("%q lets the build seat's holder write the plans building", row.State)
+			}
+		}
+	}
+}

internal/broker/controller_buckets.go

@@ -63,6 +63,26 @@ var (
 	// repository: the controller reads the ask's warrant from the router and accepts only the operator's Approve of
 	// those heads. Written by the controller alone, from the announcement, when it holds the lease.
 	DeliveryMergesBucket = BucketName(ControllerSeat, "delivery-merges")
+	// PlansBuildingBucket holds one key per delivery plan in its build phase (novox/hq ADR 0483 clause 2, issue 1687): the key
+	// is `<plan id>` (`plan-<digits>`), its value the plan, the time its build phase began and the time of the last refresh.
+	// The controller (lease holder) puts it when the plan's first build is asked, puts it again about every thirty seconds
+	// while the plan builds, and deletes it when the phase ends (every build of the last tier built, or the plan fails, stops
+	// or waits on the operator); the bucket's age, PlansBuildingKeptFor, is how long a key outlives a controller that
+	// stopped. **Read by the build seat's holders and the checker, written by the controller alone**: a holder takes no new
+	// check while any key is there, but for one that waited past ADR 0483's bound. A reader lists the keys and reads none of
+	// their values to decide that (`link.BuildingPlans`).
+	PlansBuildingBucket = BucketName(ControllerSeat, PlansBuilding)
+)
+
+// PlansBuilding is the local name of the bucket of the plans building, as a reader names it in `reads`
+// (`mesh-controller.plans-building`).
+const PlansBuilding = "plans-building"
+
+// PlansBuildingKeptFor is the age of a plan-building key: the controller puts it again at PlansBuildingRefresh, so a
+// controller that stopped leaves checks held at most this long (ADR 0483 clause 2).
+const (
+	PlansBuildingKeptFor = 2 * time.Minute
+	PlansBuildingRefresh = 30 * time.Second
 )
 
 // DeliveryMergesKeptFor is how long a merge mesh-delivery announced is kept: a week, as the events are.
@@ -133,7 +153,7 @@ func ChainBuckets() []string { return []string{BuildsBucket, RolloutsBucket, Ass
 // ControllerMirrors are the local names of the controller's buckets a module may read: the record mirrors and the
 // chain's buckets.
 func ControllerMirrors() []string {
-	return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts, ChecksRunning}
+	return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts, ChecksRunning, PlansBuilding}
 }
 
 // MirrorsWrittenByAnother are the controller's mirrors whose writer is not the controller, each with its reason: the rule
@@ -204,14 +224,14 @@ func IsControllerBucket(bucket string) bool {
 		bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket ||
 		bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket ||
 		bucket == DeliveryMergesBucket || bucket == BuildsBucket || bucket == RolloutsBucket ||
-		bucket == delivery.HaveBucket
+		bucket == PlansBuildingBucket || bucket == delivery.HaveBucket
 }
 
 // ControllerBuckets are the controller's own buckets, in the order they are asserted.
 func ControllerBuckets() []string {
 	return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket,
 		AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket,
-		DeliveryMergesBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket}
+		DeliveryMergesBucket, PlansBuildingBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket}
 }
 
 // ControllerBucketsAsserter is what raising the controller's buckets needs of a connection.
@@ -354,6 +374,22 @@ func (j *JetStream) EnsureControllerBuckets() error {
 	}); err != nil {
 		return fmt.Errorf("asserting bucket %s: %w", CheckRunningBucket, err)
 	}
+	// **The plans building: keys that age** (ADR 0483 clause 2, issue 1687). One message a key, each refresh a new one, so the
+	// age of the bucket is how long a key outlives a controller that stopped refreshing it.
+	if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
+		Bucket: PlansBuildingBucket,
+		Description: "the delivery plans in their build phase (novox/hq ADR 0483): `<plan id>` holds the plan and when its " +
+			"build phase began; put when the plan's first build is asked and about every thirty seconds, deleted when the " +
+			"phase ends, gone within two minutes of a controller that stopped; read by the build seat's holders, which " +
+			"take no new check while a key is there; written by the controller alone",
+		History:      1,
+		TTL:          PlansBuildingKeptFor,
+		MaxValueSize: 4 << 10,
+		MaxBytes:     8 << 20,
+		Storage:      jetstream.FileStorage,
+	}); err != nil {
+		return fmt.Errorf("asserting bucket %s: %w", PlansBuildingBucket, err)
+	}
 	// **The merges mesh-delivery announced: keys that age** (ADR 0420 §7, issue 1025). One message a key; a week.
 	if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{
 		Bucket: DeliveryMergesBucket,

internal/broker/declaration_files.go

@@ -0,0 +1,123 @@
+package broker
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/declaration"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// The object buckets a declaration's large files travel in (novox/hq ADR 0486, issues 1696 and 1697): **one bucket for each
+// node**, `declaration-files-<node>`.
+//
+// **The controller alone writes them, and a node-engine reads an object by its name from its own bucket** (ADR 0486 §2): its
+// account may neither list the bucket nor write it, no account may touch another node's, and no other account has any access.
+// Why a bucket each: a subject pattern cannot tell a name from a wildcard. A grant of `...M.*` matches a publish to the literal
+// subject `...M.*` too (probed on nats-server 2.11.17 by the review of this issue: a node created a consumer filtered on
+// `$O.<bucket>.C.*` and read every object of the bucket), so in one shared bucket a node could read every other node's files.
+// With a bucket each, the same trick reads only the node's own files, which are the files of its own declaration.
+//
+// A bucket of the bus's object store is a stream, OBJ_<bucket>, whose messages are the objects' metadata
+// (`$O.<bucket>.M.<base64 of the name>`) and chunks (`$O.<bucket>.C.<nuid>`).
+
+// DeclarationFilesBucket is the prefix of the buckets' names, which mesh-sdk fixes for the node-engine and the controller alike.
+const DeclarationFilesBucket = declaration.FileBucketPrefix
+
+// DeclarationFilesBucketOf is the bucket a node's files are in: mesh-sdk's FileBucketFor, the one place the name is composed. A
+// node name it refuses (one that could carry a subject wildcard or separator into a grant) is an error, never a bucket.
+func DeclarationFilesBucketOf(node string) (string, error) { return declaration.FileBucketFor(node) }
+
+// DeclarationFilesStreamOf is the stream under a node's bucket: mesh-sdk's FileStreamFor.
+func DeclarationFilesStreamOf(node string) (string, error) { return declaration.FileStreamFor(node) }
+
+// DeclarationFilesMaxBytes is a bucket's size: the objects are immutable and regenerable, and one node's hold well under a hundred
+// megabytes.
+const DeclarationFilesMaxBytes = 1 << 30
+
+// DeclarationFilesKeptFor is how long an object stays after no node's latest declaration names it (ADR 0486 §3).
+const DeclarationFilesKeptFor = 24 * time.Hour
+
+// DeclarationFilesWrites is the one pattern the controller may publish to write a node's bucket: every object's metadata and
+// chunks. The server's API for the stream is the controller's `$JS.API.>`.
+func DeclarationFilesWrites(node string) (string, error) {
+	bucket, err := DeclarationFilesBucketOf(node)
+	if err != nil {
+		return "", err
+	}
+	return "$O." + bucket + ".>", nil
+}
+
+// DeclarationFilesReads is what a node-engine's account may publish to read an object of its own bucket by its name, and nothing
+// more. The set is the one node-engine's own call (mesh-host internal/link FileObject) makes, for B the node's bucket and S its
+// stream (issue 1697, notes 4 and 5):
+//
+//   - the object's metadata, by a direct get of the last message of the one subject that names it. **Only the suffixed
+//     form**: the bare `$JS.API.DIRECT.GET.S` takes a sequence number and would read any message of the stream.
+//   - the ordered consumer on that object's chunks: `$JS.API.CONSUMER.CREATE.S.<name>.$O.B.C.<nuid>`. **Not `S.>`**:
+//     a consumer filtered on `$O.B.M.>` would read every object's name.
+//   - the flow-control replies of that consumer, without which a large object stalls at the flow-control window.
+//   - the consumer's deletion, best effort: the consumer's inactivity threshold removes it if this is refused.
+//
+// **Never granted:** STREAM.INFO (a `subjects_filter` of `>` lists every name), STREAM.MSG.GET (any message by its sequence), a
+// publish to `$O.B.>` (a write), and any subject of another node's stream. A literal `*` in a subject matches the grant's `*`: it
+// reaches the objects of this node's own bucket and no other (above).
+func DeclarationFilesReads(node string) ([]string, error) {
+	bucket, err := DeclarationFilesBucketOf(node)
+	if err != nil {
+		return nil, err
+	}
+	stream, err := DeclarationFilesStreamOf(node)
+	if err != nil {
+		return nil, err
+	}
+	return []string{
+		"$JS.API.DIRECT.GET." + stream + ".$O." + bucket + ".M.*",
+		"$JS.API.CONSUMER.CREATE." + stream + ".*.$O." + bucket + ".C.*",
+		"$JS.API.CONSUMER.DELETE." + stream + ".*",
+		"$JS.FC." + stream + ".>",
+	}, nil
+}
+
+// EnsureDeclarationFiles creates the buckets of these nodes if absent and brings their options to match. A bucket exists whenever the
+// bus does, so a node that is starting finds it (ADR 0486 §4). An update, never a delete.
+func (j *JetStream) EnsureDeclarationFiles(nodes []string) error {
+	js, err := jetstream.New(j.conn)
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	return EnsureDeclarationFiles(ctx, js, nodes...)
+}
+
+// EnsureDeclarationFiles is the same on a JetStream the caller holds.
+func EnsureDeclarationFiles(ctx context.Context, js jetstream.JetStream, nodes ...string) error {
+	for _, node := range nodes {
+		cfg, err := DeclarationFilesConfig(node)
+		if err != nil {
+			return err
+		}
+		if _, err := js.CreateOrUpdateObjectStore(ctx, cfg); err != nil {
+			return fmt.Errorf("asserting the object bucket %s: %w", cfg.Bucket, err)
+		}
+	}
+	return nil
+}
+
+// DeclarationFilesConfig is a node's bucket as the controller asserts it.
+func DeclarationFilesConfig(node string) (jetstream.ObjectStoreConfig, error) {
+	bucket, err := DeclarationFilesBucketOf(node)
+	if err != nil {
+		return jetstream.ObjectStoreConfig{}, err
+	}
+	return jetstream.ObjectStoreConfig{
+		Bucket: bucket,
+		Description: "the contents of the files over 64 KiB of the declaration of " + node + ", each under the hex of its sha256 " +
+			"digest (novox/hq ADR 0486): written by the controller alone, read by the node-engine of that node by name, removed " +
+			"a day after the node's latest declaration no longer names it",
+		MaxBytes: DeclarationFilesMaxBytes,
+		Storage:  jetstream.FileStorage,
+	}, nil
+}

internal/broker/declaration_files_live_test.go

@@ -0,0 +1,316 @@
+//go:build integration
+
+package broker
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"golang.org/x/crypto/bcrypt"
+)
+
+// nodeEnginesFileObject is the node-engine's own call (mesh-host internal/link files.go, OverNATS.FileObject), as the
+// engine makes it: the object's metadata by a direct get of one subject, then its chunks through an ordered consumer
+// bound to the stream. It is copied here so the grants are judged by the call that has to work, and its subjects are the
+// ones the grant names (issue 1697, notes 4 and 5); the node-engine's own integration test holds the other copy.
+func nodeEnginesFileObject(ctx context.Context, nc *nats.Conn, node, name string) ([]byte, error) {
+	js, err := nc.JetStream(nats.MaxWait(10 * time.Second))
+	if err != nil {
+		return nil, err
+	}
+	stream, err := DeclarationFilesStreamOf(node)
+	if err != nil {
+		return nil, err
+	}
+	bucket, err := DeclarationFilesBucketOf(node)
+	if err != nil {
+		return nil, err
+	}
+	meta, err := js.GetLastMsg(stream, "$O."+bucket+".M."+base64.URLEncoding.EncodeToString([]byte(name)),
+		nats.DirectGet(), nats.Context(ctx))
+	if err != nil {
+		return nil, err
+	}
+	var info struct {
+		NUID string `json:"nuid"`
+		Size uint64 `json:"size"`
+	}
+	if err := json.Unmarshal(meta.Data, &info); err != nil {
+		return nil, err
+	}
+	sub, err := js.SubscribeSync("$O."+bucket+".C."+info.NUID, nats.OrderedConsumer(), nats.BindStream(stream))
+	if err != nil {
+		return nil, err
+	}
+	defer sub.Unsubscribe()
+	body := make([]byte, 0, info.Size)
+	for uint64(len(body)) < info.Size {
+		msg, err := sub.NextMsgWithContext(ctx)
+		if err != nil {
+			return nil, err
+		}
+		body = append(body, msg.Data...)
+	}
+	return body, nil
+}
+
+// **The buckets' grants on a real server, judged by the node-engine's own call** (novox/hq ADR 0486 §2, issue 1697, 1697/AC2): the
+// controller's account writes an object into each node's bucket, a node-engine's account reads one of its own bucket by its name,
+// a large object included, and is refused a list of the bucket (a stream info asking for every subject), a message get by
+// sequence, a direct get by sequence, a consumer on the names, and a write; it is refused every one of those and the read by name on
+// another node's bucket, **the literal-`*` forms included** (a published subject `...M.*` or `...C.*` matches the grant's `*`: on
+// its own bucket that reaches its own files, on another's it is refused); another module's account is refused all of them.
+func TestNatsANodeEngineReadsAnObjectOfItsOwnBucketByNameAndNothingElse(t *testing.T) { // 1697/AC2
+	hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
+	if err != nil {
+		t.Fatal(err)
+	}
+	users, err := Users(Records{Nodes: []string{"alpha", "beta"}, Assigned: map[string][]Declared{
+		"alpha": {{Module: RuntimeModule}, {Module: "other-module"}},
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := range users {
+		users[i].PasswordHash = string(hash)
+	}
+	accounts, err := ComposeAccounts(users)
+	if err != nil {
+		t.Fatal(err)
+	}
+	conf := filepath.Join(t.TempDir(), "bus.conf")
+	if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
+		`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	opts, err := server.ProcessConfigFile(conf)
+	if err != nil {
+		t.Fatalf("the composed accounts do not parse: %v", err)
+	}
+	opts.NoLog, opts.NoSigs = true, true
+	s, err := server.NewServer(opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	go s.Start()
+	if !s.ReadyForConnections(10 * time.Second) {
+		t.Fatal("the bus did not come up")
+	}
+	t.Cleanup(s.Shutdown)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+	defer cancel()
+	connect := func(user string) (*nats.Conn, chan error) {
+		t.Helper()
+		refused := make(chan error, 16)
+		nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
+			nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
+		if err != nil {
+			t.Fatalf("%s cannot connect: %v", user, err)
+		}
+		t.Cleanup(nc.Close)
+		return nc, refused
+	}
+
+	// The controller makes the buckets (as it does at its start) and writes into each: for alpha a small object and one of
+	// several chunks, which needs the consumer's flow control; for beta one that holds a marker alpha must never read.
+	controller, _ := connect(ControllerName)
+	cjs, err := jetstream.New(controller)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := EnsureDeclarationFiles(ctx, cjs, "alpha", "beta"); err != nil {
+		t.Fatalf("the controller cannot make the buckets: %v", err)
+	}
+	if err := EnsureDeclarationFiles(ctx, cjs, "alpha", "beta"); err != nil {
+		t.Fatalf("asserting them again: %v", err)
+	}
+	put := func(node string, content []byte) string {
+		t.Helper()
+		store, err := cjs.ObjectStore(ctx, mustBucket(t, node))
+		if err != nil {
+			t.Fatal(err)
+		}
+		sum := sha256.Sum256(content)
+		name := hex.EncodeToString(sum[:])
+		if _, err := store.PutBytes(ctx, name, content); err != nil {
+			t.Fatalf("the controller cannot write the object %s of %s: %v", name[:8], node, err)
+		}
+		return name
+	}
+	small := bytes.Repeat([]byte("small file\n"), 100)
+	large := make([]byte, 900<<10)
+	for i := range large {
+		large[i] = byte(i*7 + i>>8)
+	}
+	secret := bytes.Repeat([]byte("BETA-SECRET "), 50)
+	smallName, largeName, betaName := put("alpha", small), put("alpha", large), put("beta", secret)
+
+	alpha, alphaRefused := connect("node.alpha")
+	for name, want := range map[string][]byte{smallName: small, largeName: large} {
+		got, err := nodeEnginesFileObject(ctx, alpha, "alpha", name)
+		if err != nil {
+			t.Fatalf("a node-engine cannot read the object %s of its own bucket by its name: %v", name[:8], err)
+		}
+		if !bytes.Equal(got, want) {
+			t.Fatalf("the node-engine read %d bytes of the object %s, want %d", len(got), name[:8], len(want))
+		}
+	}
+	beta, _ := connect("node.beta")
+	if got, err := nodeEnginesFileObject(ctx, beta, "beta", betaName); err != nil || !bytes.Equal(got, secret) {
+		t.Fatalf("beta's node-engine reads its own object: %v", err)
+	}
+	select {
+	case err := <-alphaRefused:
+		t.Fatalf("a node-engine's own read was refused something: %v", err)
+	default:
+	}
+
+	// What each account is refused, judged by the server.
+	alphaStream, alphaBucket := mustStream(t, "alpha"), mustBucket(t, "alpha")
+	betaStream, betaBucket := mustStream(t, "beta"), mustBucket(t, "beta")
+	listing := []byte(`{"subjects_filter":">"}`)
+	bySeq := []byte(`{"seq":1}`)
+	type refusal struct {
+		what string
+		do   func(nc *nats.Conn)
+	}
+	onBucket := func(stream, bucket, which string) []refusal {
+		return []refusal{
+			{which + ": a list of the bucket (stream info asking for every subject)", func(nc *nats.Conn) {
+				_, _ = nc.Request("$JS.API.STREAM.INFO."+stream, listing, 300*time.Millisecond)
+			}},
+			{which + ": a message get by sequence", func(nc *nats.Conn) {
+				_, _ = nc.Request("$JS.API.STREAM.MSG.GET."+stream, bySeq, 300*time.Millisecond)
+			}},
+			{which + ": a direct get by sequence", func(nc *nats.Conn) {
+				_, _ = nc.Request("$JS.API.DIRECT.GET."+stream, bySeq, 300*time.Millisecond)
+			}},
+			{which + ": a consumer on every object's name", func(nc *nats.Conn) {
+				js, _ := nc.JetStream(nats.MaxWait(500 * time.Millisecond))
+				if sub, err := js.SubscribeSync("$O."+bucket+".M.>", nats.OrderedConsumer(), nats.BindStream(stream)); err == nil {
+					sub.Unsubscribe()
+				}
+			}},
+			{which + ": a write of an object's metadata", func(nc *nats.Conn) {
+				_ = nc.Publish("$O."+bucket+".M.abc", []byte(`{}`))
+				_ = nc.Flush()
+			}},
+			{which + ": a write of a chunk", func(nc *nats.Conn) {
+				_ = nc.Publish("$O."+bucket+".C.abc", []byte("x"))
+				_ = nc.Flush()
+			}},
+		}
+	}
+	refusedWith := func(who string, nc *nats.Conn, ch chan error, r refusal) {
+		t.Helper()
+		for len(ch) > 0 {
+			<-ch
+		}
+		r.do(nc)
+		_ = nc.Flush()
+		deadline := time.After(3 * time.Second)
+		for {
+			select {
+			case err := <-ch:
+				if errors.Is(err, nats.ErrPermissionViolation) {
+					return
+				}
+			case <-deadline:
+				t.Errorf("the server did not refuse %s %s", who, r.what)
+				return
+			}
+		}
+	}
+	// Alpha's account on its own bucket (list, sequence reads, names, writes) and on beta's (all of that, and the read by name,
+	// and the literal-`*` probes of the review).
+	reachBeta := []refusal{
+		{"beta's bucket: a read of one object by its name", func(nc *nats.Conn) {
+			_, _ = nc.Request("$JS.API.DIRECT.GET."+betaStream+".$O."+betaBucket+".M."+
+				base64.URLEncoding.EncodeToString([]byte(betaName)), nil, 300*time.Millisecond)
+		}},
+		{"beta's bucket: a direct get with a literal `*` for the name", func(nc *nats.Conn) {
+			_, _ = nc.Request("$JS.API.DIRECT.GET."+betaStream+".$O."+betaBucket+".M.*", nil, 300*time.Millisecond)
+		}},
+		{"beta's bucket: a consumer with a literal `*` for the chunks", func(nc *nats.Conn) {
+			js, _ := nc.JetStream(nats.MaxWait(500 * time.Millisecond))
+			if sub, err := js.SubscribeSync("$O."+betaBucket+".C.*", nats.OrderedConsumer(), nats.BindStream(betaStream)); err == nil {
+				sub.Unsubscribe()
+			}
+		}},
+	}
+	all := append(append(onBucket(alphaStream, alphaBucket, "its own bucket"), onBucket(betaStream, betaBucket, "beta's bucket")...), reachBeta...)
+	for _, r := range all {
+		nodeNC, nodeCh := connect("node.alpha")
+		refusedWith("a node-engine", nodeNC, nodeCh, r)
+	}
+	// Another module is refused all of that on both buckets, and the reads by name on both.
+	for _, r := range append(append(onBucket(alphaStream, alphaBucket, "alpha's bucket"), reachBeta...),
+		refusal{"alpha's bucket: a read of one object by its name", func(nc *nats.Conn) {
+			_, _ = nc.Request("$JS.API.DIRECT.GET."+alphaStream+".$O."+alphaBucket+".M."+
+				base64.URLEncoding.EncodeToString([]byte(smallName)), nil, 300*time.Millisecond)
+		}}) {
+		otherNC, otherCh := connect("alpha.other-module")
+		refusedWith("another module", otherNC, otherCh, r)
+	}
+
+	// The literal `*` on alpha's own bucket reaches alpha's own files and never beta's: the chunks it is given hold no marker of beta's.
+	js, err := alpha.JetStream(nats.MaxWait(2 * time.Second))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if sub, err := js.SubscribeSync("$O."+alphaBucket+".C.*", nats.OrderedConsumer(), nats.BindStream(alphaStream)); err == nil {
+		for i := 0; i < 20; i++ {
+			msg, err := sub.NextMsg(300 * time.Millisecond)
+			if err != nil {
+				break
+			}
+			if bytes.Contains(msg.Data, []byte("BETA-SECRET")) {
+				t.Fatal("a literal `*` on alpha's bucket gave it a chunk of beta's file")
+			}
+		}
+		sub.Unsubscribe()
+	}
+
+	// Nothing was written by the refused: each bucket holds what the controller wrote.
+	for node, want := range map[string]int{"alpha": 2, "beta": 1} {
+		store, err := cjs.ObjectStore(ctx, mustBucket(t, node))
+		if err != nil {
+			t.Fatal(err)
+		}
+		infos, err := store.List(ctx)
+		if err != nil || len(infos) != want {
+			t.Errorf("the bucket of %s holds %d objects (%v), want the %d the controller wrote", node, len(infos), err, want)
+		}
+	}
+}
+
+func mustBucket(t *testing.T, node string) string {
+	t.Helper()
+	b, err := DeclarationFilesBucketOf(node)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return b
+}
+
+func mustStream(t *testing.T, node string) string {
+	t.Helper()
+	b, err := DeclarationFilesStreamOf(node)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return b
+}

internal/broker/declaration_files_test.go

@@ -0,0 +1,131 @@
+package broker
+
+import (
+	"slices"
+	"strings"
+	"testing"
+)
+
+// 1697/AC1: the composed accounts give a node-engine exactly the four reads of one object by its name, on its own node's bucket and
+// no other's; the controller the write of every node's bucket; and no other principal anything of any bucket.
+func TestEachNodesBucketIsWrittenByTheControllerAndReadByNameByThatNodesEngine(t *testing.T) { // 1697/AC1
+	node, err := PermissionsFor(Principal{Kind: KindNode, Node: "alpha", PasswordHash: "x"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var mine []string
+	for _, s := range node.Publish {
+		if strings.Contains(s, "declaration-files") {
+			mine = append(mine, s)
+		}
+	}
+	want := []string{
+		"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.*.$O.declaration-files-alpha.C.*",
+		"$JS.API.CONSUMER.DELETE.OBJ_declaration-files-alpha.*",
+		"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha.$O.declaration-files-alpha.M.*",
+		"$JS.FC.OBJ_declaration-files-alpha.>",
+	}
+	if !slices.Equal(mine, want) {
+		t.Fatalf("a node-engine may publish %q of the buckets, want exactly %q", mine, want)
+	}
+	// What would make a list, a write, or a read of another node's files: none of it, in any pattern a node holds. (The real
+	// server's judgement of the wildcard forms is the live test's.)
+	for _, refused := range []string{
+		"$JS.API.STREAM.INFO.OBJ_declaration-files-alpha",
+		"$JS.API.STREAM.MSG.GET.OBJ_declaration-files-alpha",
+		"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha",
+		"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha.$O.declaration-files-alpha.M.abc.def",
+		"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.x.$O.declaration-files-alpha.M.YWJj",
+		"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.x.$O.declaration-files-alpha.M.YWJj.more",
+		"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha",
+		"$O.declaration-files-alpha.M.abc",
+		"$O.declaration-files-alpha.C.abc",
+		// another node's bucket, however it is named
+		"$JS.API.DIRECT.GET.OBJ_declaration-files-beta.$O.declaration-files-beta.M.YWJj",
+		"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-beta.h4sh.$O.declaration-files-beta.C.NUID",
+		"$JS.API.CONSUMER.DELETE.OBJ_declaration-files-beta.h4sh",
+		"$JS.FC.OBJ_declaration-files-beta.x",
+		// and the old shared bucket
+		"$JS.API.DIRECT.GET.OBJ_declaration-files.$O.declaration-files.M.YWJj",
+	} {
+		if MayPublish(node, refused) {
+			t.Errorf("a node-engine may publish %s, which lists or writes a bucket or reads another node's", refused)
+		}
+	}
+	for _, kept := range []string{
+		"$JS.API.DIRECT.GET.OBJ_declaration-files-alpha.$O.declaration-files-alpha.M.YWJj",
+		"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-alpha.h4sh.$O.declaration-files-alpha.C.NUID",
+	} {
+		if !MayPublish(node, kept) {
+			t.Errorf("a node-engine may not publish %s, which reads one object of its own bucket by its name", kept)
+		}
+	}
+
+	controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x", FilesFor: []string{"alpha", "beta"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, s := range []string{"$O.declaration-files-alpha.M.abc", "$O.declaration-files-beta.C.nuid"} {
+		if !MayPublish(controller, s) {
+			t.Errorf("the controller may not write %s", s)
+		}
+	}
+	if MayPublish(controller, "$O.declaration-files-gamma.M.abc") {
+		t.Error("the controller may write the bucket of a node it was not given")
+	}
+
+	for _, p := range []Principal{
+		{Kind: KindModule, Module: "mesh-vault", PasswordHash: "x"},
+		{Kind: KindNodeTools, Node: "alpha", PasswordHash: "x"},
+		{Kind: KindPerson, Module: "jochen", Invokes: []string{"*"}, PasswordHash: "x"},
+	} {
+		perms, err := PermissionsFor(p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, s := range append(append([]string(nil), perms.Publish...), perms.Subscribe...) {
+			if strings.Contains(s, "declaration-files") {
+				t.Errorf("%s holds %s, and no account but the controller's and the node-engines' has any access", p.Username(), s)
+			}
+		}
+	}
+}
+
+// The users the controller composes carry each node's bucket to the controller and to that node alone.
+func TestTheComposedUsersGiveEachNodeItsOwnBucketAndTheControllerAll(t *testing.T) {
+	users, err := Users(Records{Nodes: []string{"alpha", "beta"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var controller Principal
+	for _, u := range users {
+		if u.Kind == KindController {
+			controller = u
+		}
+	}
+	if !slices.Equal(controller.FilesFor, []string{"alpha", "beta"}) {
+		t.Fatalf("the controller writes the buckets of %v", controller.FilesFor)
+	}
+}
+
+// A node name that could carry a subject wildcard or separator into a grant is refused with an error, never composed into a bucket
+// (ADR 0488): the grants of such a name would reach another node's bucket.
+func TestANodeNameThatCarriesAWildcardGetsNoBucketAndNoGrant(t *testing.T) {
+	for _, node := range []string{"", "*", ">", "a.b", "a b", "a*"} {
+		if b, err := DeclarationFilesBucketOf(node); err == nil {
+			t.Errorf("node %q got the bucket %q", node, b)
+		}
+		if _, err := DeclarationFilesReads(node); err == nil {
+			t.Errorf("node %q got read grants", node)
+		}
+		if _, err := DeclarationFilesWrites(node); err == nil {
+			t.Errorf("node %q got write grants", node)
+		}
+		if _, err := DeclarationFilesConfig(node); err == nil {
+			t.Errorf("node %q got a bucket config", node)
+		}
+	}
+	if b, err := DeclarationFilesBucketOf("g14"); err != nil || b != "declaration-files-g14" {
+		t.Errorf("g14: %q %v", b, err)
+	}
+}

internal/broker/nats.go

@@ -104,6 +104,10 @@ type Principal struct {
 	// Username and inbox are <node>.<module>.<process id>.
 	Process string
 
+	// FilesFor are the nodes whose declaration-file buckets this principal writes, for the controller alone (novox/hq ADR 0486 §2):
+	// every node of the mesh, one bucket each.
+	FilesFor []string
+
 	Emits    []string
 	Consumes []string
 	Serves   []string
@@ -357,9 +361,11 @@ var VaultNodeKey = SeatVerb{Seat: "mesh-vault", Verb: "node-key"}
 
 // VerbsTheControllerActsOnAWarrant are the other seat verbs the controller calls when the operator's warrant
 // chooses them (novox/hq ADR 0259): a machine's service restarted, and a walk started or stopped through the
-// controller's own `plans`. Named one by one; a node seat's on any machine, a mesh seat's flat.
+// controller's own `plans`, a machine's held builds sent by its own `push` and the builds held after a failed release
+// sent by its own `upgrade release-backlog` (novox/hq issue 1732). Named one by one; a node seat's on any machine, a
+// mesh seat's flat.
 var VerbsTheControllerActsOnAWarrant = []SeatVerb{{Seat: "node-service-manager", Verb: "restart"},
-	{Seat: ControllerSeat, Verb: "plans"}}
+	{Seat: ControllerSeat, Verb: "plans"}, {Seat: ControllerSeat, Verb: "push"}, {Seat: ControllerSeat, Verb: "upgrade"}}
 
 // VerbsTheControllerAsksTheOperatorChannel are the operator channel's verbs the controller calls (novox/hq
 // ADR 0380): `tell`, one plain message with no condition behind it — a module's own secret given, named,
@@ -383,6 +389,16 @@ type DegradableGrant struct {
 // works. Every other grant stays required — missing one of those is a controller that comes up and is
 // refused on the first thing it tries, with no fallback that covers it.
 var DegradableGrants = []DegradableGrant{
+	{
+		Subject: "mesh.seat." + ControllerSeat + ".tool.push",
+		Reason: "refused: the ask of a held move (Release) is answered, and the refusal told, while the operator's own " +
+			"push at the terminal still sends the machine (issue 1732; ADR 0259)",
+	},
+	{
+		Subject: "mesh.seat." + ControllerSeat + ".tool.upgrade",
+		Reason: "refused: the ask of the builds held after a failed release is answered, and the refusal told, while the " +
+			"operator's own `upgrade release-backlog` at the terminal still releases them (issue 1732; ADR 0259)",
+	},
 	{
 		Subject: "mesh.seat." + VerbsTheControllerAsksTheOperatorChannel[0].Seat + ".tool." + VerbsTheControllerAsksTheOperatorChannel[0].Verb,
 		Reason:  "refused: the controller raises the secret-given condition as before (ADR 0380 §9)",
@@ -408,6 +424,11 @@ var DegradableGrants = []DegradableGrant{
 		Reason: "refused: the controller asserts this bucket through the API and writes none of its keys — the build seat's " +
 			"holders do (ADR 0406 §3, issue 974)",
 	},
+	{
+		Subject: "$KV." + PlansBuildingBucket + ".>",
+		Reason: "refused: the controller then says no plan builds, and no check is held while a delivery builds — a check run " +
+			"wrongly costs a delivery a place, a check held wrongly costs a pull request its turn (ADR 0483 clause 2, issue 1687)",
+	},
 	{
 		Subject: "mesh.seat." + ControllerSeat + ".event.sent",
 		Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " +
@@ -736,6 +757,15 @@ func PermissionsFor(p Principal) (Permissions, error) {
 			}
 			pub = append(pub, "$KV."+bucket+".>")
 		}
+		// **And the object bucket a declaration's large files travel in** (novox/hq ADR 0486 §2, issue 1697): the
+		// controller alone writes it, as a publish to the bucket's own subjects, which `$JS.API.>` does not cover.
+		for _, node := range p.FilesFor {
+			writes, err := DeclarationFilesWrites(node)
+			if err != nil {
+				return Permissions{}, err
+			}
+			pub = append(pub, writes)
+		}
 		// **And what the bus says about itself, read-only** (novox/hq to-be 45 §3, S9): a durable
 		// consumer that gave up on a message, or one that was deleted. The server already publishes
 		// both in the mesh's own account; the controller says each as a condition in the mesh's words.
@@ -835,6 +865,13 @@ func PermissionsFor(p Principal) (Permissions, error) {
 			return Permissions{}, err
 		}
 		pub = append(pub, haveKeys...)
+		// And an object of the declaration bucket by its name, and nothing of the bucket's list or writing
+		// (novox/hq ADR 0486 §2, issue 1697).
+		fileReads, err := DeclarationFilesReads(p.Node)
+		if err != nil {
+			return Permissions{}, err
+		}
+		pub = append(pub, fileReads...)
 		// And, while the vault serves it, the vault's move-in in its own name and nobody else's (issue 676): the
 		// values sealed to this machine, which only it opens, handed in sealed to the vault's key for the call.
 		if p.MovesIntoTheVault {

internal/broker/plans_building_live_test.go

@@ -0,0 +1,134 @@
+//go:build integration
+
+package broker
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"golang.org/x/crypto/bcrypt"
+)
+
+// **A holder of the build seat reads the plans building and the server refuses its write** (novox/hq ADR 0483 clause 2, issue
+// 1687): on a real server whose users are composed as the controller composes them, the controller makes the bucket and puts a
+// plan's key; the build agent's credential lists the keys and gets one, and its put of a key is refused with a permissions
+// violation. It also reads the first waiting ask of a check queue by a direct get, which takes nothing from the queue.
+func TestNatsTheBuildSeatsHoldersReadThePlansBuildingAndTheServerRefusesTheirWrite(t *testing.T) { // 1687
+	hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost)
+	if err != nil {
+		t.Fatal(err)
+	}
+	seat := Seat{Name: TheBuildSeat, Scope: "node", Accepts: []string{"build"}, Emits: []string{"built", "started"}}
+	users, err := Users(Records{Nodes: []string{"alpha"}, Assigned: map[string][]Declared{
+		"alpha": {{Module: RuntimeModule}, {Module: "build-agent", Holds: []Seat{seat}}},
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := range users {
+		users[i].PasswordHash = string(hash)
+	}
+	accounts, err := ComposeAccounts(users)
+	if err != nil {
+		t.Fatal(err)
+	}
+	conf := filepath.Join(t.TempDir(), "bus.conf")
+	if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
+		`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	opts, err := server.ProcessConfigFile(conf)
+	if err != nil {
+		t.Fatalf("the composed accounts do not parse: %v", err)
+	}
+	opts.NoLog, opts.NoSigs = true, true
+	s, err := server.NewServer(opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	go s.Start()
+	if !s.ReadyForConnections(10 * time.Second) {
+		t.Fatal("the bus did not come up")
+	}
+	t.Cleanup(s.Shutdown)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	connect := func(user string) (*nats.Conn, chan error) {
+		t.Helper()
+		refused := make(chan error, 8)
+		nc, err := nats.Connect(s.ClientURL(), nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user),
+			nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refused <- err }))
+		if err != nil {
+			t.Fatalf("%s cannot connect: %v", user, err)
+		}
+		t.Cleanup(nc.Close)
+		return nc, refused
+	}
+	controller, _ := connect(ControllerName)
+	cj := OnConn(controller)
+	if err := AssertMeshStreams(cj); err != nil {
+		t.Fatalf("the controller cannot make the streams: %v", err)
+	}
+	if err := cj.EnsureControllerBuckets(); err != nil {
+		t.Fatalf("the controller cannot make its buckets: %v", err)
+	}
+	cjs, _ := jetstream.New(controller)
+	ckv, err := cjs.KeyValue(ctx, PlansBuildingBucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ckv.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
+		t.Fatalf("the controller cannot put a plan's key: %v", err)
+	}
+
+	holder, refused := connect("alpha.build-agent")
+	hjs, err := jetstream.New(holder)
+	if err != nil {
+		t.Fatal(err)
+	}
+	kv, err := hjs.KeyValue(ctx, PlansBuildingBucket)
+	if err != nil {
+		t.Fatalf("a holder of the build seat cannot bind %s: %v", PlansBuildingBucket, err)
+	}
+	keys, err := kv.Keys(ctx)
+	if err != nil || len(keys) != 1 || keys[0] != "plan-1" {
+		t.Fatalf("the holder lists %v (%v), want plan-1", keys, err)
+	}
+	if e, err := kv.Get(ctx, "plan-1"); err != nil || e == nil {
+		t.Fatalf("the holder cannot get the key: %v", err)
+	}
+	// Its write is refused by the server.
+	if err := holder.Publish("$KV."+PlansBuildingBucket+".plan-2", []byte(`{}`)); err != nil {
+		t.Fatal(err)
+	}
+	_ = holder.Flush()
+	select {
+	case err := <-refused:
+		if !errors.Is(err, nats.ErrPermissionViolation) {
+			t.Errorf("the server said %v, want a permissions violation", err)
+		}
+	case <-time.After(3 * time.Second):
+		t.Error("the server did not refuse the holder's put of a plan's key")
+	}
+	if _, err := ckv.Get(ctx, "plan-2"); !errors.Is(err, jetstream.ErrKeyNotFound) {
+		t.Errorf("the refused put left a key: %v", err)
+	}
+
+	// And the direct get of a check queue's first waiting ask, which is granted.
+	reply, err := holder.RequestWithContext(ctx, "$JS.API.DIRECT.GET."+ChecksOtherStream, []byte(`{"seq":1,"next_by_subj":"`+
+		CheckSubjectPrefix(TheBuildSeat, false)+`>"}`))
+	if err != nil {
+		t.Fatalf("the holder's direct get of %s was not answered: %v", ChecksOtherStream, err)
+	}
+	if reply.Header.Get("Status") != "404" {
+		t.Errorf("an empty queue answered %q, want 404", reply.Header.Get("Status"))
+	}
+}

internal/broker/plans_building_mirror_test.go

@@ -0,0 +1,30 @@
+package broker
+
+import (
+	"slices"
+	"testing"
+)
+
+// Issue 1687 (ADR 0483 clause 2): a module whose manifest reads mesh-controller.plans-building may name it in `reads`, and
+// reads the bucket and writes none; the controller writes it, so it is no mirror written by another.
+func TestAModuleReadingPlansBuildingReadsAndWritesNone(t *testing.T) {
+	if !slices.Contains(ControllerMirrors(), "plans-building") || BucketName(ControllerSeat, PlansBuilding) != PlansBuildingBucket {
+		t.Fatalf("a module cannot name %s in `reads`: %v", PlansBuildingBucket, ControllerMirrors())
+	}
+	if _, other := MirrorsWrittenByAnother[PlansBuilding]; other {
+		t.Error("the controller writes the plans building itself")
+	}
+	perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-delivery-checker", PasswordHash: "x",
+		Reads: []string{"mesh-controller.plans-building"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	stream := "KV_" + PlansBuildingBucket
+	has(t, perms.Publish, "$JS.API.STREAM.INFO."+stream)
+	has(t, perms.Publish, "$JS.API.DIRECT.GET."+stream+".>")
+	for _, p := range perms.Publish {
+		if SubjectsOverlap(p, "$KV."+PlansBuildingBucket+".plan-1") {
+			t.Errorf("the reading module may write %s through %q", PlansBuildingBucket, p)
+		}
+	}
+}

internal/broker/testdata/composed.conf

@@ -29,7 +29,7 @@ accounts {
     ]
     users = [
       { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: {
-        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
+        publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_plans-building.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] }
         subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }
@@ -38,7 +38,7 @@ accounts {
         subscribe: { allow: ["_INBOX.enrol.one.>"] }
       } }
       { user: "node.one", password: "$2a$11$nnnnnnnnnnnnnnnnnnnnnn", permissions: {
-        publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.one.>", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.INFO.NODES.one", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.one.>", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$KV.node-engine_have.one.>", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
+        publish: { allow: ["$JS.ACK.NODES.one.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.one.>", "$JS.API.CONSUMER.CREATE.OBJ_declaration-files-one.*.$O.declaration-files-one.C.*", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.DELETE.OBJ_declaration-files-one.*", "$JS.API.CONSUMER.INFO.NODES.one", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.one.>", "$JS.API.DIRECT.GET.OBJ_declaration-files-one.$O.declaration-files-one.M.*", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$JS.FC.OBJ_declaration-files-one.>", "$KV.node-engine_have.one.>", "$SRV.PING.node-tools.one", "mesh.control.one.>"] }
         subscribe: { allow: ["_DELIVER.one", "_DELIVER.one.>", "_INBOX.node.one.>", "mesh.node.one.ask.hand-over", "mesh.node.one.ask.judge-directories", "mesh.node.one.ask.move-secret", "mesh.node.one.ask.report", "mesh.node.one.ask.setuid-search", "mesh.node.one.declare"] }
         allow_responses: { max: 1, ttl: "1m" }
       } }

internal/broker/users.go

@@ -135,7 +135,7 @@ type Records struct {
 // correct.
 func Users(r Records) ([]Principal, error) {
 	out := []Principal{{Kind: KindController, Uses: asksSeatOf(r), AsksIssued: issuedOf(r), AsksTheVault: vaultVerbsOf(r),
-		AsksTheConfig: servedOf(r, VerbsTheControllerAsksTheConfig)}}
+		AsksTheConfig: servedOf(r, VerbsTheControllerAsksTheConfig), FilesFor: sortedCopy(r.Nodes)}}
 
 	for _, node := range sortedCopy(r.Nodes) {
 		witness := false

internal/broker/writers.go

@@ -177,6 +177,10 @@ var WritersTable = []WriterRow{
 		KeptIn: "key-value " + CheckRunningBucket + ", aged out when not renewed", Others: "the build seat's holders read it to drop a duplicate request",
 		Subjects: kvOf(CheckRunningBucket), Writes: holdsTheBuildSeat,
 		Shared: "every machine holding the build seat puts the key of the check it runs and deletes it when the check ends"},
+	{State: "the plans-building mirror", Writer: "controller (lease holder), at the first build's ask and at each refresh",
+		KeptIn:   "key-value " + PlansBuildingBucket + ", aged out when not refreshed",
+		Others:   "the build seat's holders read it and take no new check while a key is there (ADR 0483)",
+		Subjects: kvOf(PlansBuildingBucket), Writes: isController},
 	{State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)",
 		KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule},
 	// A pull request's head, before it merges (novox/hq to-be 45 §9): the same one announcer.
@@ -219,6 +223,13 @@ var WritersTable = []WriterRow{
 	{State: "a machine's have", Writer: "the node-engine of the machine, under its own keys `<node>.>`",
 		KeptIn: "key-value " + delivery.HaveBucket + ", twenty revisions of each key",
 		Others: "read through `reads` as node-engine.have", Subjects: kvOf(delivery.HaveBucket), Writes: ownNodeKeys},
+	// The contents of a declaration's files over 64 KiB (novox/hq ADR 0486 §2): the controller alone writes the bucket, each
+	// object under the digest of its content; a node-engine reads one by its name, with no list and no write.
+	{State: "a declaration's large files", Writer: "controller, each object under the sha256 digest of its content",
+		KeptIn: "the object bucket " + DeclarationFilesBucket + "-<node> of each node, an object removed a day after the " +
+			"node's latest declaration no longer names it",
+		Others:   "a node-engine reads an object of its own bucket by its name and may neither list the bucket nor write it",
+		Subjects: []string{"$O.*.>"}, Writes: isController},
 	{State: "the declared-config mirror", Writer: "controller (lease holder), from its store, only on a change",
 		KeptIn: "key-value " + DeclaredConfigBucket, Others: "read through `reads` as mesh-controller." + MirrorDeclaredConfig,
 		Subjects: kvOf(DeclaredConfigBucket), Writes: isController},

internal/broker/writers_test.go

@@ -29,6 +29,8 @@ var designRows = []string{
 	"builds and their outcomes",
 	"a pull request's check asked",
 	"a pull request's check running",
+	// The plans in their build phase, a bucket the build seat's holders read (ADR 0483 clause 2, issue 1687).
+	"the plans-building mirror",
 	"a merge announced",
 	"a pull request's head announced",
 	"a pull request's merge check",
@@ -43,6 +45,8 @@ var designRows = []string{
 	"the rollouts of the chain",
 	// What each machine has of each module, written by its node-engine under its own keys (ADR 0429 §1, issue 1136).
 	"a machine's have",
+	// The contents of a declaration's large files, in the bus's object bucket (ADR 0486 §2, issue 1697).
+	"a declaration's large files",
 	"the declared-config mirror",
 	"the predefined-keys mirror",
 	// The provisioner client's word on what it placed (novox/hq ADR 0349 §4, design 57 P6).

internal/builder/check.go

@@ -1773,9 +1773,24 @@ func whatFailed(s string) string {
 			}
 		}
 	}
+	// The controller's module check refuses with a count line and one line per problem, then prints its
+	// warning counts: the refusal is what failed, not the count printed last (novox/hq issue 1310).
+	for i, line := range lines {
+		if line = strings.TrimSpace(line); problemsLine.MatchString(line) {
+			if i+1 < len(lines) {
+				if first := strings.TrimSpace(lines[i+1]); first != "" {
+					return line + ": " + first
+				}
+			}
+			return line
+		}
+	}
 	return lastLine(s)
 }
 
+// problemsLine is the module check's refusal: "mesh-controller: 3 problem(s) in 139 manifest(s)".
+var problemsLine = regexp.MustCompile(`^\S+: [1-9]\d* problem\(s\) in \d+ manifest\(s\)$`)
+
 func lastLine(s string) string {
 	lines := strings.Split(strings.TrimSpace(s), "\n")
 	return strings.TrimSpace(lines[len(lines)-1])

internal/builder/check_test.go

@@ -199,3 +199,29 @@ func TestOnlyATrysSelectionReachesTheScript(t *testing.T) {
 		t.Fatalf("a try's selection: %v", env)
 	}
 }
+
+// **Issue 1310**: the controller's module check prints its warning counts after its refusal; a failed check
+// is said by the refusal, never by the count printed last.
+func TestAFailedModuleCheckIsSaidByItsRefusalNotItsWarningCount(t *testing.T) {
+	log := "mesh-controller: 3 problem(s) in 139 manifest(s)\n" +
+		"mesh-files claims node-files and says it serves upload, download, which that seat's protocol does not promise\n" +
+		"minio claims the seat \"object-store\", which no module declares and the mesh does not define\n" +
+		"only-office claims the seat \"document-converter\", which no module declares and the mesh does not define\n" +
+		"adwaita: ok, 4 tool(s)\nlong-running resources without health: 59\n" +
+		"files asking for a setting without saying whether it is trusted: 36\n"
+	// 1310/AC1
+	want := "mesh-controller: 3 problem(s) in 139 manifest(s): mesh-files claims node-files and says it serves upload, " +
+		"download, which that seat's protocol does not promise"
+	if got := whatFailed(log); got != want {
+		t.Errorf("1310/AC1: the captured log is said as %q, not %q", got, want)
+	}
+	// 1310/AC2
+	if got, want := whatFailed("ok  \tx/a\t1s\n--- FAIL: TestX (0.01s)\n    x_test.go:12: got 1\nFAIL\nlong-running resources without health: 59\n"),
+		"--- FAIL: TestX (0.01s)"; got != want {
+		t.Errorf("1310/AC2: a failing test is said as %q, not %q", got, want)
+	}
+	// 1310/AC3
+	if got, want := whatFailed("starting\nsomething odd happened\n"), "something odd happened"; got != want {
+		t.Errorf("1310/AC3: a log with no failure line is said as %q, not %q", got, want)
+	}
+}

internal/builder/kill.go

@@ -3,6 +3,7 @@ package builder
 import (
 	"context"
 	"os/exec"
+	"regexp"
 	"strings"
 	"syscall"
 	"time"
@@ -41,6 +42,11 @@ func inItsOwnGroup(cmd *exec.Cmd) {
 	cmd.WaitDelay = KillWait
 }
 
+// CgroupParent is the slice every container a build or check starts joins (novox/hq issue 1798), so the node's own
+// weights and cap bind it: the build-agent module declares the slice with the machine's CPU weight, IO weight and
+// memory limits, and names it in MESH_BUILD_CGROUP_PARENT. Empty, a container joins the runtime's default.
+var CgroupParent string
+
 // Labelled is a Runner that marks every container a build starts with the build's id, so a kill
 // finds what outlived the docker client (novox/hq ADR 0219). Applied at the one place every command
 // passes rather than at each `docker run` the builder composes: a run added later is labelled too.
@@ -55,8 +61,11 @@ func LabelledArgs(name string, args []string, id string) []string {
 	if name != "docker" || len(args) == 0 || args[0] != "run" || id == "" {
 		return args
 	}
-	out := make([]string, 0, len(args)+2)
+	out := make([]string, 0, len(args)+4)
 	out = append(out, "run", "--label", BuildLabel+"="+id)
+	if CgroupParent != "" {
+		out = append(out, "--cgroup-parent", CgroupParent)
+	}
 	return append(out, args[1:]...)
 }
 
@@ -71,12 +80,45 @@ func RemoveContainersOf(ctx context.Context, run Runner, id string) (int, error)
 	if len(ids) == 0 {
 		return 0, nil
 	}
-	if _, err := run(ctx, "", "docker", append([]string{"rm", "-f"}, ids...)...); err != nil {
+	if err := removeWithVolumes(ctx, run, ids); err != nil {
 		return 0, err
 	}
 	return len(ids), nil
 }
 
+// MountedVolumes is the template `docker inspect --format` reads the names of a container's volumes with.
+const MountedVolumes = `{{range .Mounts}}{{if eq .Type "volume"}}{{println .Name}}{{end}}{{end}}`
+
+// anonymousVolume is the name docker gives a volume it makes for an image's VOLUME line: 64 hex digits.
+var anonymousVolume = regexp.MustCompile(`^[0-9a-f]{64}$`)
+
+// removeWithVolumes removes containers together with their anonymous volumes (novox/hq issue 1772). A throwaway
+// store declares its data directory a VOLUME, docker makes an anonymous volume for it, and `docker rm -f`
+// without -v leaves that volume for ever: 1,629 of them, 107 GB, on one build node. So the volumes the
+// containers mount are read first, the containers are removed with -v, and an anonymous volume still there
+// afterwards is removed by name. Only a volume one of these containers mounted, and only one with an anonymous
+// volume's name, is touched: a named volume — an operator's own data — never is. A volume already gone is
+// no failure.
+func removeWithVolumes(ctx context.Context, run Runner, ids []string) error {
+	var volumes []string
+	if out, err := run(ctx, "", "docker", append([]string{"inspect", "--format", MountedVolumes}, ids...)...); err == nil {
+		seen := map[string]bool{}
+		for _, name := range strings.Fields(out) {
+			if anonymousVolume.MatchString(name) && !seen[name] {
+				seen[name] = true
+				volumes = append(volumes, name)
+			}
+		}
+	}
+	if _, err := run(ctx, "", "docker", append([]string{"rm", "-f", "-v"}, ids...)...); err != nil {
+		return err
+	}
+	if len(volumes) > 0 {
+		_, _ = run(ctx, "", "docker", append([]string{"volume", "rm"}, volumes...)...)
+	}
+	return nil
+}
+
 // A holder that stops mid-build leaves its containers running (novox/hq issue 306).
 //
 // A build's containers are removed by the build itself, by a kill, or by the next delivery of the same
@@ -106,7 +148,7 @@ func RemoveLeftBehind(ctx context.Context, run Runner) (int, error) {
 	if len(ids) == 0 {
 		return 0, nil
 	}
-	if _, err := run(ctx, "", "docker", append([]string{"rm", "-f"}, ids...)...); err != nil {
+	if err := removeWithVolumes(ctx, run, ids); err != nil {
 		return 0, err
 	}
 	return len(ids), nil

internal/builder/kill_test.go

@@ -2,6 +2,7 @@ package builder
 
 import (
 	"context"
+	"errors"
 	"reflect"
 	"strings"
 	"testing"
@@ -49,7 +50,8 @@ func TestAKillRemovesTheContainersLabelledWithTheBuild(t *testing.T) {
 	if err != nil || n != 2 {
 		t.Fatalf("%d %v", n, err)
 	}
-	if want := []string{"docker ps -aq --filter label=mesh.build=build-3", "docker rm -f c1 c2"}; !reflect.DeepEqual(ran, want) {
+	if want := []string{"docker ps -aq --filter label=mesh.build=build-3", "docker inspect --format " + MountedVolumes + " c1 c2",
+		"docker rm -f -v c1 c2"}; !reflect.DeepEqual(ran, want) {
 		t.Errorf("ran %v", ran)
 	}
 }
@@ -73,7 +75,7 @@ func TestAStartingHolderRemovesWhatEarlierHoldersLeftAndNothingElse(t *testing.T
 	if n != 3 {
 		t.Errorf("removed %d", n)
 	}
-	if len(ran) != 2 || !reflect.DeepEqual(ran[1], []string{"docker", "rm", "-f", "c1", "c2", "c7"}) {
+	if len(ran) != 3 || !reflect.DeepEqual(ran[2], []string{"docker", "rm", "-f", "-v", "c1", "c2", "c7"}) {
 		t.Fatalf("ran %v", ran)
 	}
 	if !strings.Contains(strings.Join(ran[0], " "), "--filter label=mesh.build") {
@@ -90,3 +92,75 @@ func TestAStartingHolderRemovesWhatEarlierHoldersLeftAndNothingElse(t *testing.T
 		t.Errorf("with nothing of the seat's: %d %v %v", n, err, ran)
 	}
 }
+
+// **Issue 1772**: a throwaway store declares its data directory a VOLUME, so docker makes an anonymous volume for
+// it, and `docker rm -f` without -v leaves that volume behind (1,629 of them, 107 GB, on one build node). Every
+// container a build or check removes is removed with its anonymous volumes (-v), and any volume the container
+// mounted that is still there afterwards is removed by name — those, and only those, never a named volume.
+func TestARemovedContainerTakesItsAnonymousVolumesWithIt(t *testing.T) {
+	anon := strings.Repeat("a", 64)
+	other := strings.Repeat("b", 64)
+	var ran []string
+	run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
+		ran = append(ran, name+" "+strings.Join(args, " "))
+		switch args[0] {
+		case "ps":
+			return "c1\n", nil
+		case "inspect":
+			return anon + "\n" + "compose_data\n" + other + "\n", nil
+		}
+		return "", nil
+	}
+	if n, err := RemoveContainersOf(context.Background(), run, "build-4"); err != nil || n != 1 {
+		t.Fatalf("%d %v", n, err)
+	}
+	last := ran[len(ran)-1]
+	if want := "docker volume rm " + anon + " " + other; last != want {
+		t.Errorf("last command %q, want %q", last, want)
+	}
+	if !strings.Contains(strings.Join(ran, "\n"), "docker rm -f -v c1") {
+		t.Errorf("ran %v", ran)
+	}
+	if strings.Contains(last, "compose_data") {
+		t.Errorf("a named volume was removed: %q", last)
+	}
+}
+
+// A volume already gone when the sweep looks (rm -v took it) is no failure.
+func TestAVolumeAlreadyGoneIsNoFailure(t *testing.T) {
+	run := func(_ context.Context, _ string, name string, args ...string) (string, error) {
+		switch args[0] {
+		case "ps":
+			return "c1\n", nil
+		case "inspect":
+			return strings.Repeat("c", 64) + "\n", nil
+		case "volume":
+			return "", errors.New("no such volume")
+		}
+		return "", nil
+	}
+	if n, err := RemoveContainersOf(context.Background(), run, "build-5"); err != nil || n != 1 {
+		t.Fatalf("%d %v", n, err)
+	}
+}
+
+// A node's build slice limits everything a build or check starts (novox/hq issue 1798): with a parent
+// set, every `docker run` joins it, so the node's CPU and IO weights and memory cap bind the container;
+// with none set the arguments are as they always were.
+func TestEveryContainerJoinsTheNodesBuildSlice(t *testing.T) {
+	was := CgroupParent
+	defer func() { CgroupParent = was }()
+	CgroupParent = "meshbuild.slice"
+	got := LabelledArgs("docker", []string{"run", "--rm", "img", "sh"}, "build-1")
+	want := []string{"run", "--label", "mesh.build=build-1", "--cgroup-parent", "meshbuild.slice", "--rm", "img", "sh"}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("1798/AC1: docker run became %v", got)
+	}
+	if got := LabelledArgs("docker", []string{"build", "."}, "build-1"); !reflect.DeepEqual(got, []string{"build", "."}) {
+		t.Errorf("a docker build became %v", got)
+	}
+	CgroupParent = ""
+	if got := LabelledArgs("docker", []string{"run", "img"}, "build-1"); !reflect.DeepEqual(got, []string{"run", "--label", "mesh.build=build-1", "img"}) {
+		t.Errorf("1798/AC1: with no slice, docker run became %v", got)
+	}
+}

internal/builder/registry.go

@@ -59,15 +59,35 @@ func (r Registry) PublishImage(ctx context.Context, localTag, repository string)
 	}); err != nil {
 		return "", err
 	}
-	out, err := r.Run(ctx, "", "docker", "inspect", "--format", "{{index .RepoDigests 0}}", remote)
+	// Every digest the daemon lists for the image, one per line: the image may have been pushed under other names before
+	// (a check's scratch copy, another registry), and the first of them is not the one this push made (novox/hq issue
+	// 1742: a build recorded a digest the store never held).
+	out, err := r.Run(ctx, "", "docker", "inspect", "--format", "{{range .RepoDigests}}{{println .}}{{end}}", remote)
 	if err != nil {
 		return "", fmt.Errorf("pushed %s and cannot read back what it was pinned as: %w", remote, err)
 	}
-	pinned := strings.TrimSpace(out)
-	if !strings.Contains(pinned, "@sha256:") {
+	first, pinned := "", ""
+	for _, line := range strings.Split(out, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		if first == "" {
+			first = line
+		}
+		if strings.HasPrefix(line, remote+"@sha256:") {
+			pinned = line
+			break
+		}
+	}
+	if pinned == "" {
+		if strings.Contains(first, "@sha256:") {
+			return "", fmt.Errorf("pushed %s, but the daemon lists no digest for that name (only %q): the digest of another "+
+				"repository is not what this registry serves", remote, first)
+		}
 		// A tag is not a pin. It can be made to point at something else, and this file is applied
 		// on machines with no mesh to ask about anything (novox/hq ADR 0006).
-		return "", fmt.Errorf("%s came back as %q, which is not pinned by digest", remote, pinned)
+		return "", fmt.Errorf("%s came back as %q, which is not pinned by digest", remote, first)
 	}
 	return pinned, nil
 }

internal/builder/registry_test.go

@@ -252,3 +252,32 @@ func TestAPushedImageComesBackPinned(t *testing.T) {
 		t.Fatalf("it did not tag and then push: %v", ran)
 	}
 }
+
+// 1742: an image pushed under other names before lists their digests too, and the digest recorded is the one of the name
+// this push made — never the first listed, which may be another repository's.
+func TestThePinnedDigestIsTheOneOfTheNamePushed(t *testing.T) {
+	want := "registry.invalid/shell/server@sha256:" + strings.Repeat("b", 64)
+	earlier := "registry.invalid/scratch/shell/server@sha256:" + strings.Repeat("a", 64)
+	r := Registry{Address: "registry.invalid", Run: func(
+		_ context.Context, _, name string, args ...string) (string, error) {
+		if name == "docker" && len(args) > 0 && args[0] == "inspect" {
+			return earlier + "\n" + want + "\n", nil
+		}
+		return "", nil
+	}}
+	got, err := r.PublishImage(context.Background(), "local", "shell/server")
+	if err != nil || got != want {
+		t.Fatalf("pinned %q (%v); want %q", got, err, want)
+	}
+	// Only another repository's digest is listed: the push is not taken for pinned.
+	r.Run = func(_ context.Context, _, name string, args ...string) (string, error) {
+		if name == "docker" && len(args) > 0 && args[0] == "inspect" {
+			return earlier + "\n", nil
+		}
+		return "", nil
+	}
+	if _, err := r.PublishImage(context.Background(), "local", "shell/server"); err == nil ||
+		!strings.Contains(err.Error(), "no digest for that name") {
+		t.Fatalf("a digest of another repository was taken for the push's own: %v", err)
+	}
+}

internal/builder/same_program.go

@@ -105,7 +105,7 @@ func imageContent(ctx context.Context, run Runner, dir, ref string) (string, err
 	if id == "" {
 		return "", errors.New("the container engine named no container for " + ref)
 	}
-	defer func() { _, _ = run(context.WithoutCancel(ctx), dir, "docker", "rm", "--force", id) }()
+	defer func() { _, _ = run(context.WithoutCancel(ctx), dir, "docker", "rm", "--force", "--volumes", id) }()
 	if err := os.MkdirAll(dir, 0o700); err != nil {
 		return "", err
 	}

internal/inventory/hold.go

@@ -100,8 +100,9 @@ var ErrPlansBusy = errors.New("another controller is working the plans")
 // returned function and by the session ending, so a controller that dies holding it holds nothing.
 //
 // wait false gives ErrPlansBusy at once when another holds them — the timer's way: the holder is
-// moving the plans already. wait true looks again every HoldPoll for up to HoldWaitFor — an
-// outcome's or a merge's way, which must be written.
+// moving the plans already; and a build outcome's on the event loop, which the build records keep
+// for the look to settle (novox/hq issue 1708). wait true looks again every HoldPoll for up to
+// HoldWaitFor — a merge's way, which must be written.
 func (i *Inventory) HoldPlans(ctx context.Context, wait bool) (func(), error) {
 	deadline := time.Now().Add(HoldWaitFor)
 	for {

internal/inventory/migrations/0107-a-refused-registration-is-kept.sql

@@ -0,0 +1,10 @@
+-- A build's outcome whose registration the mesh refused, by the build's id (novox/hq issue 1708). The build record says
+-- what the builder said, which worked; an outcome the event loop could not take into the plans (another act held them)
+-- is settled from the records by the look, and without this it would read the refused build as built. Kept in the
+-- store so a controller replaced before the settle still fails the plan.
+create table build_refused (
+    id     text primary key,
+    module text not null,
+    why    text not null,
+    at     timestamptz not null default now()
+);

internal/inventory/refused.go

@@ -0,0 +1,33 @@
+package inventory
+
+import "context"
+
+// RecordRefused keeps that the mesh refused to register a build's outcome, and why (novox/hq issue 1708): the build
+// record says the builder's words, which worked, and the look that settles a plan from the records reads this to fail
+// it instead. The first refusal of an id stands.
+func (i *Inventory) RecordRefused(ctx context.Context, id, module, why string) error {
+	_, err := i.store.Pool().Exec(ctx,
+		`insert into build_refused (id, module, why) values ($1, $2, $3) on conflict (id) do nothing`, id, module, why)
+	return err
+}
+
+// RefusedOf is why each of these builds' registration was refused, by id; a build not refused is absent.
+func (i *Inventory) RefusedOf(ctx context.Context, ids []string) (map[string]string, error) {
+	out := map[string]string{}
+	if len(ids) == 0 {
+		return out, nil
+	}
+	rows, err := i.store.Pool().Query(ctx, `select id, why from build_refused where id = any($1)`, ids)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var id, why string
+		if err := rows.Scan(&id, &why); err != nil {
+			return nil, err
+		}
+		out[id] = why
+	}
+	return out, rows.Err()
+}

internal/inventory/refused_test.go

@@ -0,0 +1,28 @@
+//go:build integration
+
+package inventory
+
+import "testing"
+
+// novox/hq issue 1708 (1708/AC6): a refused registration is kept by its build's id, the first refusal standing, and
+// read back for the builds asked about only.
+func TestARefusedRegistrationIsKeptByItsBuild(t *testing.T) {
+	inv := ForTest(t)
+	ctx := t.Context()
+	if err := inv.RecordRefused(ctx, "build-1708-a", "nodered", "a port is taken"); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordRefused(ctx, "build-1708-a", "nodered", "said again"); err != nil {
+		t.Fatalf("a refusal said twice was refused: %v", err)
+	}
+	got, err := inv.RefusedOf(ctx, []string{"build-1708-a", "build-1708-b"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 1 || got["build-1708-a"] != "a port is taken" {
+		t.Fatalf("the refusals read back are %v", got)
+	}
+	if none, err := inv.RefusedOf(ctx, nil); err != nil || len(none) != 0 {
+		t.Fatalf("no builds asked about answered %v %v", none, err)
+	}
+}

internal/link/admit_test.go

@@ -0,0 +1,28 @@
+package link
+
+import (
+	"context"
+	"testing"
+
+	"github.com/nats-io/nats.go"
+)
+
+// 1768/AC1: a request dropped as a duplicate of one another builder runs marks nothing in hand: the claim is not made,
+// so no mark is left behind to fail every later claim.
+func TestADuplicateDropIsJudgedBeforeTheClaimSoLeavesNoMark(t *testing.T) {
+	claims := 0
+	m := &natsMachine{
+		opts:      MachineOptions{Claim: func() bool { claims++; return true }},
+		duplicate: func(context.Context, BuildRequest) string { return "ace is running it" },
+	}
+	if m.admit(context.Background(), &nats.Msg{}, BuildRequest{ID: "build-1"}) {
+		t.Fatal("a duplicate was admitted")
+	}
+	if claims != 0 {
+		t.Fatalf("a dropped duplicate was claimed %d times, which leaves the in-hand mark behind", claims)
+	}
+	m.duplicate = func(context.Context, BuildRequest) string { return "" }
+	if !m.admit(context.Background(), &nats.Msg{}, BuildRequest{ID: "build-2"}) || claims != 1 {
+		t.Fatalf("a request that is no duplicate was not claimed once: claims=%d", claims)
+	}
+}

internal/link/build.go

@@ -292,6 +292,10 @@ type GroupMember struct {
 	// Base is the branch it merges into: the build seat merges the head into that trunk's head before the
 	// group is composed (novox/hq ADR 0323 §6). A build seat from before it composes the head alone.
 	Base string `json:"base,omitempty"`
+	// Trunk is the base branch's head the forge announced with the head (novox/hq issue 1106): the trunk commit the
+	// composed verdict names for this member when the build seat named none, so a member whose trunk did not move is
+	// never merged-refused for a verdict that names no trunk.
+	Trunk string `json:"trunk,omitempty"`
 }
 
 // Who judges a merge check's gate.

internal/link/builds_nats.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"sync"
 	"time"
 
 	"github.com/nats-io/nats.go"
@@ -162,11 +163,23 @@ type natsMachine struct {
 	// build waits, then the checks of pull requests marked ready, then every other check (novox/hq ADR 0406 §3).
 	subs []*takeQueue
 	opts MachineOptions
+	// duplicate stands in for runningElsewhere in tests; nil is the real one.
+	duplicate func(context.Context, BuildRequest) string
+	// gate is what the loop asks of the bus to hold checks while a delivery builds; nil is the bus's (a test sets a fake).
+	gate checkGate
+	// said is the faults of the hold already said, letKey whether this holder created the let-through key for the check it
+	// took last (ADR 0483 clause 3).
+	said   map[string]bool
+	letKey bool
+	// mu guards the subscriptions of subs, bound and let go of by the taking loop and closed by Close.
+	mu sync.Mutex
 }
 
 // takeQueue is one queue a machine pulls from.
 type takeQueue struct {
 	sub *nats.Subscription
+	// pull is what the queue is fetched through: the subscription once bound (a test sets a fake).
+	pull puller
 	// kind is what its asks are: a build, a ready check, or another check.
 	kind string
 	// worker is the consumer bound; bound is false until a bind succeeded, for a queue the mesh may not have made yet.
@@ -174,6 +187,11 @@ type takeQueue struct {
 	bound  bool
 }
 
+// puller is the fetch of a queue.
+type puller interface {
+	Fetch(batch int, opts ...nats.PullOpt) ([]*nats.Msg, error)
+}
+
 // The kinds of queue a machine takes from, in the order it takes from them.
 const (
 	queueBuilds = "builds"
@@ -195,6 +213,9 @@ type MachineOptions struct {
 	// the request in hand first and reads its drain after, so an ask of `busy` racing the fetch sees one or
 	// the other, never neither. Nil claims every one.
 	Claim func() bool
+	// LetThroughAfter is how long a check waits while a delivery builds before it is let through; zero is ADR 0483's thirty
+	// minutes. Set by a test.
+	LetThroughAfter time.Duration
 }
 
 // MachineOverNATS takes build work from the current build role.
@@ -222,6 +243,8 @@ const (
 )
 
 func (m *natsMachine) Close() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
 	for _, q := range m.subs {
 		if q.sub != nil {
 			_ = q.sub.Unsubscribe()
@@ -244,6 +267,9 @@ func (m *natsMachine) Close() {
 // **In this order** (novox/hq ADR 0406 §3): a delivery build, then a check of a pull request marked ready, then any
 // other check, each queue tried without waiting; when all three are empty the first is waited on for a second at most
 // and the three are tried again, so a delivery build is never much more than a second behind and nothing spins.
+//
+// **Unless a delivery builds** (novox/hq ADR 0483): while the controller's bucket of the plans building holds a key, no check queue
+// is fetched from, but for the one check that waited LetThroughAfter, which goes first (see next).
 func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build)) error {
 	worker, found := broker.HolderConsumerFor(m.on, "build-agent",
 		broker.DeclaredSeat{Name: m.seat, Accepts: []string{"build"}})
@@ -288,6 +314,19 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
 			}
 			continue
 		}
+		// The seat's own queue is bound again when a running ask made the machine let go of it (quiesce): a machine that cannot
+		// is not taking, which is said and tried again.
+		if !m.subs[0].bound {
+			if err := m.bind(m.subs[0]); err != nil {
+				fmt.Fprintf(os.Stderr, "this machine cannot bind %s again after a build, and tries again: %v\n", m.seat, err)
+				select {
+				case <-ctx.Done():
+					return nil
+				case <-time.After(pausedPoll):
+				}
+				continue
+			}
+		}
 		if time.Now().After(bindAgain) {
 			for _, q := range m.subs[1:] {
 				if !q.bound {
@@ -312,7 +351,7 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
 			// say why (2026-10-03, the first build agents). Asked again.
 			continue
 		default:
-			if m.subs[0].sub.IsValid() {
+			if m.subs[0].sub != nil && m.subs[0].sub.IsValid() {
 				// A transient fault in asking — a reconnect, a slow server — is asked past rather
 				// than ending the machine; one that outlasts the ack wait redelivers nothing lost.
 				time.Sleep(time.Second)
@@ -320,62 +359,154 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
 			}
 			return fmt.Errorf("the bus stopped delivering build work: %w", err)
 		}
-		// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
-		// the one that arrived in that moment. Handed back at once for another holder — counted as
-		// a delivery, which a pause landing exactly then costs and nothing else does.
-		if m.opts.Paused != nil && m.opts.Paused() {
-			_ = msg.Nak()
-			continue
-		}
-		var request BuildRequest
-		if err := json.Unmarshal(msg.Data, &request); err != nil {
-			// Unreadable: terminated rather than retried, because the next attempt reads the same
-			// bytes. Nobody waiting hears an answer, which is right — there was no request.
-			_ = msg.Term()
-			continue
-		}
-		// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
-		// id into the seat's cancelled set before deleting the ask, so one taken in between is
-		// ended here, terminated rather than redelivered, and its outcome said as failed — never
-		// built. A set that cannot be read is said and the ask built: a cancel is a person's
-		// exception, and a holder that refused every build while its grant was missing would
-		// stop the mesh building for it.
-		build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
-		if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
-			fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
-		} else if cancelled {
-			endCancelled(ctx, build)
-			continue
-		}
-		// **A check left behind is dropped when it is taken** (novox/hq ADR 0406 §3): a request on the third queue for a
-		// head and a trunk commit its pull request's key already says were judged is acknowledged and not run.
-		if q.kind == queueOther {
-			if why := m.judgedAlready(ctx, request); why != "" {
-				fmt.Fprintf(os.Stderr, "%s was left behind in the check queue and is dropped: %s\n", request.ID, why)
-				_ = msg.Ack()
-				continue
-			}
-		}
-		// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause
-		// landing then is.
-		if m.opts.Claim != nil && !m.opts.Claim() {
-			_ = msg.Nak()
-			continue
-		}
-		// A build outlives the acknowledgement window many times over; said while it runs,
-		// as the controller says it for its own long handlers, so the server neither hands
-		// the ask to a second machine nor counts the wait against its deliveries.
-		working := make(chan struct{})
-		go stillWorking(msg, working, heartbeatWhileWorking)
-		// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
-		// deleted when it ends, whatever the outcome.
-		stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
-		do(ctx, build)
-		stopRunning()
-		close(working)
+		m.handOver(ctx, msg, q, do)
+		// And the queues it let go of are bound again at once, not after the half minute.
+		bindAgain = time.Time{}
 	}
 }
 
+// handOver settles one fetched ask: dropped, handed back or built. **A check let through keeps its key for as long as it
+// stays here** (ADR 0483 clause 3): refreshed while it runs and deleted when it ends, however it ends.
+func (m *natsMachine) handOver(ctx context.Context, msg *nats.Msg, q *takeQueue, do func(context.Context, Build)) {
+	if m.letKey {
+		defer m.keepLetThrough(ctx, heartbeatWhileWorking)()
+	}
+	// **Paused while the pull was answered** (ADR 0219): "takes no new build" holds even for
+	// the one that arrived in that moment. Handed back at once for another holder — counted as
+	// a delivery, which a pause landing exactly then costs and nothing else does.
+	if m.opts.Paused != nil && m.opts.Paused() {
+		_ = msg.Nak()
+		return
+	}
+	var request BuildRequest
+	if err := json.Unmarshal(msg.Data, &request); err != nil {
+		// Unreadable: terminated rather than retried, because the next attempt reads the same
+		// bytes. Nobody waiting hears an answer, which is right — there was no request.
+		_ = msg.Term()
+		return
+	}
+	// **Cancelled in the moment it was fetched** (novox/hq ADR 0219): the controller wrote the
+	// id into the seat's cancelled set before deleting the ask, so one taken in between is
+	// ended here, terminated rather than redelivered, and its outcome said as failed — never
+	// built. A set that cannot be read is said and the ask built: a cancel is a person's
+	// exception, and a holder that refused every build while its grant was missing would
+	// stop the mesh building for it.
+	build := &natsBuild{request: request, msg: msg, on: m.on, js: m.js, seat: m.seat, delivery: deliveryOf(msg)}
+	if cancelled, err := IsCancelled(m.js.Conn(), m.seat, request.ID); err != nil {
+		fmt.Fprintf(os.Stderr, "could not read whether %s was cancelled, so it is built: %v\n", request.ID, err)
+	} else if cancelled {
+		endCancelled(ctx, build)
+		return
+	}
+	// **A check left behind is dropped when it is taken** (novox/hq ADR 0406 §3): a request on the third queue for a
+	// head and a trunk commit its pull request's key already says were judged is acknowledged and not run.
+	if q.kind == queueOther {
+		if why := m.judgedAlready(ctx, request); why != "" {
+			fmt.Fprintf(os.Stderr, "%s was left behind in the check queue and is dropped: %s\n", request.ID, why)
+			_ = msg.Ack()
+			return
+		}
+	}
+	if !m.admit(ctx, msg, request) {
+		return
+	}
+	// A build outlives the acknowledgement window many times over; said while it runs,
+	// as the controller says it for its own long handlers, so the server neither hands
+	// the ask to a second machine nor counts the wait against its deliveries.
+	working := make(chan struct{})
+	go stillWorking(msg, working, heartbeatWhileWorking)
+	// And a check says it is running, to the other builders (ADR 0406 §3): its key put now and at each heartbeat,
+	// deleted when it ends, whatever the outcome.
+	stopRunning := keepRunning(ctx, m.js.Conn(), request, m.on, heartbeatWhileWorking)
+	// **And holds nothing else while it runs** (issue 1747): see quiesce.
+	m.quiesce(q)
+	do(ctx, build)
+	stopRunning()
+	close(working)
+	// **The queue it came from too, once the ask is settled**: a fetch that returned a late message leaves its pull request
+	// open, and the server can deliver one more into the subscription (issue 1747). Take binds every queue again first.
+	m.quiesce(nil)
+}
+
+// quiesce lets go of every queue but the one the running ask came from, for as long as it runs. **A builder never holds a
+// message it does not start** (issue 1747). A message the server sends after a fetch gave up, within the few milliseconds its
+// pull request still lived, lands in that queue's subscription and stays there until the next fetch of it: held, never
+// acknowledged as in progress, for as long as the builder runs its current ask, five or ten minutes for a check, while the
+// holders that could build it are all busy and a delivery waits behind it. The legacy pull subscription cannot be read
+// without sending a new pull request, which could take one more, so it cannot be searched for such a message. It is dropped
+// instead: the subscription is closed, a message in it or on its way is lost to this builder and the server hands it out again
+// when its worker's wait for an acknowledgement ends (a minute), to any holder that is free, and the queue is bound again
+// before the next fetch (Take). The queue the running ask came from keeps its subscription, which its acknowledgements use.
+func (m *natsMachine) quiesce(keep *takeQueue) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	for _, q := range m.subs {
+		if q == keep || !q.bound {
+			continue
+		}
+		if q.sub != nil {
+			_ = q.sub.Unsubscribe()
+		}
+		q.sub, q.pull, q.bound = nil, nil, false
+	}
+}
+
+// admit decides whether a fetched request is run here, and answers the message when it is not.
+//
+// **The duplicate is judged before the request is claimed** (issue 1768): a claim marks the build in hand, and a request
+// dropped after it left the mark behind, so every later claim failed and every ask was handed back at once until all
+// had spent their deliveries. A request dropped here has marked nothing.
+//
+// **Drained in the moment it was fetched** (novox/hq ADR 0335): handed back at once, as a pause landing then is.
+func (m *natsMachine) admit(ctx context.Context, msg *nats.Msg, request BuildRequest) bool {
+	// **A duplicate of a check another builder is running is dropped** (issue 1649): an ask redelivered to this machine
+	// because its first holder did not say, in time, that it had it. The running key names that builder and this same
+	// ask; running it a second time would spend a second place on one check.
+	elsewhere := m.runningElsewhere
+	if m.duplicate != nil {
+		elsewhere = m.duplicate
+	}
+	if why := elsewhere(ctx, request); why != "" {
+		fmt.Fprintf(os.Stderr, "%s is handed out again while another builder runs it and is dropped: %s\n", request.ID, why)
+		_ = msg.Ack()
+		return false
+	}
+	if m.opts.Claim != nil && !m.opts.Claim() {
+		_ = msg.Nak()
+		return false
+	}
+	return true
+}
+
+// runningElsewhere says why an ask need not run here, or nothing: the bucket of the checks running says another builder
+// runs this very ask (its id, its head and its trunk commit). A key that cannot be read runs the check.
+func (m *natsMachine) runningElsewhere(ctx context.Context, r BuildRequest) string {
+	if r.Check == nil {
+		return ""
+	}
+	running, ok, err := ReadRunning(ctx, m.js.Conn(), r)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "could not read whether %s is running elsewhere, so it runs: %v\n", r.ID, err)
+		return ""
+	}
+	return duplicateOf(running, ok, r, m.on)
+}
+
+// runningFreshFor is how recent a running key's last heartbeat must be for its builder to be taken as alive: two missed
+// heartbeats of ten seconds are not yet a dead builder, and a redelivery after a crash comes a minute after the last one.
+const runningFreshFor = 25 * time.Second
+
+// duplicateOf says why an ask is a second hand-out of one another builder is running, or nothing.
+func duplicateOf(running CheckRunning, found bool, r BuildRequest, me string) string {
+	// Seen is the server's time and runningFreshFor is judged against this builder's clock: a skew over about 35 s would matter.
+	// A key not renewed for runningFreshFor is a builder that has gone: its ask is redelivered after the ack wait, and
+	// that is the recovery, not a duplicate (the key itself lives two minutes).
+	if found && running.Build == r.ID && running.Builder != me && running.Same(r) && time.Since(running.Seen) < runningFreshFor {
+		return fmt.Sprintf("%s is running it as %s since %s", running.Builder, running.Build, running.Started.Local().Format("15:04:05"))
+	}
+	return ""
+}
+
 // bind binds a queue's worker, which exists and is not created here.
 //
 // **The consumer's own filter, not the one subject this machine cares about.** The client checks what is asked for
@@ -383,25 +514,39 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
 // subscribing `…accept.build` against a consumer filtered on `…accept.>` is rejected even though it is narrower. Learned
 // twice now, on two different consumers, which is why it is written down here.
 func (m *natsMachine) bind(q *takeQueue) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
 	sub, err := m.js.Context().PullSubscribe(q.worker.Filters[0], q.worker.Name,
 		nats.Bind(q.worker.Stream, q.worker.Name), nats.ManualAck())
 	if err != nil {
 		return err
 	}
-	q.sub, q.bound = sub, true
+	q.sub, q.pull, q.bound = sub, sub, true
 	return nil
 }
 
 // next takes the next request in the order of the queues: each tried for a moment, and when all are empty the first waited
 // on for a second. Nothing, with no error, when there was nothing.
 func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
+	// **While a delivery builds, only delivery builds are taken, but for one check that waited** (novox/hq ADR 0483): the
+	// bucket of the plans building is read before either check queue is fetched from, and a key there holds both. The check
+	// that waited LetThroughAfter goes first, ahead of the builds that wait; the rest of the holders take builds only.
+	if m.seat == broker.TheBuildSeat && len(m.subs) > 1 {
+		if plans := m.building(ctx); len(plans) > 0 {
+			if msg, q := m.letOneThrough(ctx); msg != nil {
+				fmt.Fprintf(os.Stderr, "a check that waited %s is let through while %s builds\n", m.letThroughAfter(), whyHeld(plans))
+				return msg, q, nil
+			}
+			return m.nextBuild(ctx)
+		}
+	}
 	var last error
 	for _, q := range m.subs {
 		if !q.bound {
 			continue
 		}
 		trying, end := context.WithTimeout(ctx, checkTakeWait)
-		fetched, err := q.sub.Fetch(1, nats.Context(trying))
+		fetched, err := q.pull.Fetch(1, nats.Context(trying))
 		end()
 		if len(fetched) > 0 {
 			return fetched[0], q, nil
@@ -417,9 +562,18 @@ func (m *natsMachine) next(ctx context.Context) (*nats.Msg, *takeQueue, error) {
 	if last != nil {
 		return nil, nil, last
 	}
+	return m.nextBuild(ctx)
+}
+
+// nextBuild waits for a second at most for a delivery build, the seat's own queue.
+func (m *natsMachine) nextBuild(ctx context.Context) (*nats.Msg, *takeQueue, error) {
+	if !m.subs[0].bound {
+		// Let go of while an ask ran and not bound again yet (Take binds it before it asks): nothing to take from.
+		return nil, nil, nil
+	}
 	waiting, end := context.WithTimeout(ctx, takeIdleWait)
 	defer end()
-	fetched, err := m.subs[0].sub.Fetch(1, nats.Context(waiting))
+	fetched, err := m.subs[0].pull.Fetch(1, nats.Context(waiting))
 	if len(fetched) > 0 {
 		return fetched[0], m.subs[0], nil
 	}
@@ -582,3 +736,16 @@ func endCancelled(ctx context.Context, b *natsBuild) {
 	}
 	_ = b.msg.Term()
 }
+
+// boundKinds are the kinds of queue this machine holds a subscription to now.
+func (m *natsMachine) boundKinds() []string {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	var out []string
+	for _, q := range m.subs {
+		if q.bound {
+			out = append(out, q.kind)
+		}
+	}
+	return out
+}

internal/link/bus.go

@@ -184,6 +184,14 @@ func (b OverNATS) PublishMembershipAfter(ctx context.Context, node, module strin
 	return nil
 }
 
+// MaxPayload is the largest message the connected server takes, as the server told this connection.
+func (b OverNATS) MaxPayload() int64 {
+	if b.Conn == nil {
+		return 0
+	}
+	return b.Conn.MaxPayload()
+}
+
 func (b OverNATS) PublishDeclaration(ctx context.Context, node string, body []byte) error {
 	_, err := b.JS.Publish(DeclareSubject(node), body, nats.Context(ctx))
 	if err != nil {

internal/link/bus_nats.go

@@ -24,3 +24,6 @@ func ConnectNats(js *broker.JetStream, enroller Enroller, listener Listener) *Se
 // Bus is the controller's outbound, whichever transport it connected over. Callers that send a
 // declaration or ask a tool use this rather than the channel, which one transport does not have.
 func (s *Server) Bus() Bus { return s.bus }
+
+// SendingOn is a server that only sends, over the bus given: for a test of a sender with a bus it holds in its hand.
+func SendingOn(bus Bus) *Server { return &Server{bus: bus} }

internal/link/checks_nats.go

@@ -253,11 +253,25 @@ func watchPullIn(ctx context.Context, buckets []string, open func(ctx context.Co
 
 // checkTakeWait is how long a taking loop waits on one queue for an ask that is not there, and takeIdleWait how long it
 // waits on the first when all three are empty.
+//
+// **A fetch's pull request must end before the fetch does** (issue 1649). nats.go gives the server-side request an expiry a tenth
+// shorter than the fetch's wait (fetchExpiryOf), and the server then answers with a status message at the expiry; a wait so
+// short that the answer can land after the fetch gave up left a status message in the pull's buffer, and anything a request
+// could still deliver would be held by a machine that is busy. Each wait leaves at least minFetchMargin between the two.
 const (
-	checkTakeWait = 30 * time.Millisecond
+	checkTakeWait = 100 * time.Millisecond
 	takeIdleWait  = time.Second
 )
 
+// minFetchMargin is the least time between a fetch's server-side expiry and the end of its wait.
+const minFetchMargin = 10 * time.Millisecond
+
+// fetchExpiryOf is the expiry nats.go gives the pull request of a fetch that waits this long: the wait less a tenth of it, or
+// less five seconds.
+func fetchExpiryOf(wait time.Duration) time.Duration {
+	return wait - min(wait/10, 5*time.Second)
+}
+
 // CheckRunning is what a builder says of the check it is running, in the bucket of the checks running (novox/hq ADR 0406
 // §3): the head and trunk commit it checks, the builder and its build. The key is the pull request's (RunningKeyOf).
 type CheckRunning struct {
@@ -268,15 +282,21 @@ type CheckRunning struct {
 	// Started is when the check started, kept on every heartbeat: the entry's own time is the last heartbeat's, and a
 	// reader that asks how long a check has run reads this (novox/hq ADR 0433 §6).
 	Started time.Time `json:"started"`
+	// Seen is when the key was last put, which is the last heartbeat: read from the entry, never written (issue 1649).
+	Seen time.Time `json:"-"`
 }
 
 // RunningKeyOf is the key a check is said running under: its pull request's tokens, or `group.<name>` for a group's
-// composed check; none for a try, which no other request replaces.
+// composed check; `try.<id>` for a try, which no other request replaces.
 func RunningKeyOf(r BuildRequest) (string, bool) {
 	c := r.Check
 	switch {
-	case c == nil, c.Try:
+	case c == nil:
 		return "", false
+	case c.Try:
+		// A try says it runs under its own id, which no other request shares: so a reader of the bucket can tell a try that runs
+		// from a check left waiting for a redelivery (ADR 0483 clause 3).
+		return "try." + tokenOf(r.ID), true
 	case c.Group != "":
 		return "group." + tokenOf(c.Group), true
 	case c.Number <= 0:
@@ -319,6 +339,7 @@ func ReadRunning(ctx context.Context, conn *nats.Conn, r BuildRequest) (CheckRun
 	if err := json.Unmarshal(e.Value(), &running); err != nil {
 		return CheckRunning{}, false, nil // a key of a builder from before this means no match
 	}
+	running.Seen = e.Created()
 	return running, true, nil
 }
 

internal/link/checks_nats_integration_test.go

@@ -510,3 +510,113 @@ func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) {
 		stop()
 	}
 }
+
+// An ask handed out a second time to another builder while the first still runs it is acknowledged and not run: the
+// running key names the other builder and this same ask (issue 1649).
+func TestAnAskHandedOutAgainWhileAnotherBuilderRunsItIsNotRunTwice(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	aKeyBucket(t, js)
+	aRunningBucket(t, js, 2*time.Minute)
+	ctx, stop := context.WithCancel(t.Context())
+	defer stop()
+	asks := BuildsOn(js, TheBuildMachine)
+	running := make(chan string, 4)
+	release := make(chan struct{})
+	a := MachineOverNATS(js, "builder-a")
+	defer a.Close()
+	go func() {
+		_ = a.Take(ctx, func(ctx context.Context, work Build) {
+			running <- work.Request().ID
+			<-release
+			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
+			_ = work.Done()
+		})
+	}()
+	check := aPullCheck("build-twice", 7, "head-7-000000000", "trunk-1-000000", false)
+	if err := asks.Ask(ctx, check); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case <-running:
+	case <-time.After(10 * time.Second):
+		t.Fatal("builder a took nothing")
+	}
+	// The bus's redelivery, as it reaches the other machine: the same ask, the same id.
+	if err := asks.Ask(ctx, check); err != nil {
+		t.Fatal(err)
+	}
+	b := MachineOverNATS(js, "builder-b")
+	defer b.Close()
+	took := make(chan string, 4)
+	go func() {
+		_ = b.Take(ctx, func(ctx context.Context, work Build) {
+			took <- work.Request().ID
+			_ = work.Done()
+		})
+	}()
+	select {
+	case id := <-took:
+		t.Fatalf("builder b ran %s, which builder a is running", id)
+	case <-time.After(3 * time.Second):
+	}
+	if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 {
+		t.Errorf("%d request(s) still wait; builder b was to acknowledge the duplicate", n)
+	}
+	close(release)
+}
+
+// A busy machine fetches nothing, so no ask waiting in the queue is taken by it, whatever status message sits in its pulls
+// (issue 1649). A pull request of our own that expires at once leaves such a message in the other checks' pull.
+func TestABusyMachineTakesNoWaitingAskWhateverItsPullsHold(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	aKeyBucket(t, js)
+	aRunningBucket(t, js, 2*time.Minute)
+	ctx, stop := context.WithCancel(t.Context())
+	defer stop()
+	asks := BuildsOn(js, TheBuildMachine)
+	busy := make(chan string, 4)
+	release := make(chan struct{})
+	a := MachineOverNATS(js, "builder-a").(*natsMachine)
+	defer a.Close()
+	go func() {
+		_ = a.Take(ctx, func(ctx context.Context, work Build) {
+			busy <- work.Request().ID
+			<-release
+			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
+			_ = work.Done()
+		})
+	}()
+	if err := asks.Ask(ctx, BuildRequest{ID: "busy", Repository: "/d"}); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case <-busy:
+	case <-time.After(10 * time.Second):
+		t.Fatal("builder a took nothing")
+	}
+	// **A busy machine holds no pull at all** (issues 1649 and 1747): the queues the running ask did not come from are let go of,
+	// so no message, a status message or a late delivery, can sit in one while it runs.
+	if kinds := a.boundKinds(); len(kinds) != 1 || kinds[0] != queueBuilds {
+		t.Fatalf("builder a holds %v while it builds; want only the queue its build came from", kinds)
+	}
+	other := &takeQueue{kind: queueOther, worker: broker.CheckWorkers()[1]}
+	time.Sleep(500 * time.Millisecond)
+	for i, id := range []string{"wait-1", "wait-2"} {
+		if err := asks.Ask(ctx, aPullCheck(id, 20+i, "head-x-000000000", "trunk-1-000000", false)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	time.Sleep(5 * time.Second)
+	info, err := js.Context().ConsumerInfo(other.worker.Stream, other.worker.Name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.NumAckPending != 0 || info.NumRedelivered != 0 || info.Delivered.Consumer != 0 {
+		t.Fatalf("a busy machine took from the queue: %d delivered, %d pending, %d redelivered", info.Delivered.Consumer,
+			info.NumAckPending, info.NumRedelivered)
+	}
+	if n := len(queueOf(t, js).Of(AskWaiting)); n != 2 {
+		t.Fatalf("%d asks wait; want the 2 nobody took", n)
+	}
+	close(release)
+}

internal/link/checks_wait_integration_test.go

@@ -0,0 +1,479 @@
+//go:build integration
+
+package link
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/nats-io/nats.go"
+	"reflect"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1652, 1687, 1688), against a real bus.
+
+// aBusWithTheBuckets is the bus with the build role and the controller's buckets, the bucket of the checks running with an age
+// of two seconds so a key a holder stopped refreshing is seen to expire.
+func aBusWithTheBuckets(t *testing.T) (*broker.JetStream, jetstream.KeyValue) {
+	t.Helper()
+	js := aBusWithTheBuildRole(t)
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := t.Context()
+	if _, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.CheckRunningBucket, History: 1, TTL: 2 * time.Second}); err != nil {
+		t.Fatal(err)
+	}
+	plans, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.PlansBuildingBucket, History: 1, TTL: broker.PlansBuildingKeptFor})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return js, plans
+}
+
+func aCheckAskedAgo(ago time.Duration, number int, ready bool) BuildRequest {
+	r := aPullCheck(fmt.Sprintf("build-%d", time.Now().Add(-ago).UnixNano()), number, fmt.Sprintf("head-%d-0000000", number), "trunk-0-000000", ready)
+	return r
+}
+
+// 1652/AC6: a plan's builds asked in two tiers with a check waiting, the check is handed out only after the last tier's builds,
+// and not in the gap between the tiers.
+func TestNatsACheckIsHandedOutOnlyAfterTheLastTierOfAPlan(t *testing.T) {
+	js, plans := aBusWithTheBuckets(t)
+	ctx, stop := context.WithCancel(t.Context())
+	defer stop()
+	asks := BuildsOn(js, TheBuildMachine)
+
+	if _, err := plans.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
+		t.Fatal(err)
+	}
+	if err := asks.Ask(ctx, aCheckAskedAgo(time.Minute, 7, true)); err != nil {
+		t.Fatal(err)
+	}
+	if err := asks.Ask(ctx, BuildRequest{ID: "tier-1-a", Repository: "/a"}); err != nil {
+		t.Fatal(err)
+	}
+	took := make(chan string, 16)
+	machine := MachineOverNATS(js, "anchor")
+	defer machine.Close()
+	go func() {
+		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
+			took <- work.Request().ID
+			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
+			_ = work.Done()
+		})
+	}()
+	expect := func(want string) {
+		t.Helper()
+		select {
+		case got := <-took:
+			if got != want {
+				t.Fatalf("taken %s, want %s", got, want)
+			}
+		case <-time.After(15 * time.Second):
+			t.Fatalf("%s was not taken", want)
+		}
+	}
+	expect("tier-1-a")
+	// The gap between two tiers: the delivery's queue is empty, the plan's key is there, and the check is not taken.
+	select {
+	case got := <-took:
+		t.Fatalf("%s was taken in the gap between two tiers", got)
+	case <-time.After(3 * time.Second):
+	}
+	if err := asks.Ask(ctx, BuildRequest{ID: "tier-2-a", Repository: "/b"}); err != nil {
+		t.Fatal(err)
+	}
+	expect("tier-2-a")
+	select {
+	case got := <-took:
+		t.Fatalf("%s was taken before the plan's last build was built and its key deleted", got)
+	case <-time.After(2 * time.Second):
+	}
+	// The phase ends: the key is deleted, and the check is handed out.
+	if err := plans.Delete(ctx, "plan-1"); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case got := <-took:
+		if got == "tier-1-a" || got == "tier-2-a" {
+			t.Fatalf("a build was taken twice: %s", got)
+		}
+	case <-time.After(15 * time.Second):
+		t.Fatal("the check was not handed out once the plan's key was deleted")
+	}
+}
+
+// 1688: two holders racing for checks that waited past the bound: exactly one is let through at a time, by a create that fails
+// while the key is there, and the key is deleted when its check ends, after which the next oldest goes.
+func TestNatsTwoHoldersRacingForOldChecksLetExactlyOneThrough(t *testing.T) {
+	js, plans := aBusWithTheBuckets(t)
+	ctx, stop := context.WithCancel(t.Context())
+	defer stop()
+	// The buckets' age is two seconds here, so the keys are refreshed well within it.
+	was := heartbeatWhileWorking
+	heartbeatWhileWorking = 400 * time.Millisecond
+	t.Cleanup(func() { heartbeatWhileWorking = was })
+	asks := BuildsOn(js, TheBuildMachine)
+	if _, err := plans.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
+		t.Fatal(err)
+	}
+	for i, ago := range []time.Duration{50 * time.Minute, 40 * time.Minute, 35 * time.Minute} {
+		if err := asks.Ask(ctx, aCheckAskedAgo(ago, i+1, false)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	var mu sync.Mutex
+	running, most := 0, 0
+	started := make(chan string, 8)
+	release := make(chan struct{})
+	do := func(ctx context.Context, work Build) {
+		mu.Lock()
+		running++
+		most = max(most, running)
+		mu.Unlock()
+		started <- work.Request().Check.Repo + fmt.Sprint(work.Request().Check.Number)
+		<-release
+		mu.Lock()
+		running--
+		mu.Unlock()
+		_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "x", Failed: "no"})
+		_ = work.Done()
+	}
+	for _, on := range []string{"ace", "shanks"} {
+		machine := MachineOverNATS(js, on)
+		defer machine.Close()
+		go func() { _ = machine.Take(ctx, do) }()
+	}
+	var order []string
+	for range 3 {
+		select {
+		case id := <-started:
+			order = append(order, id)
+		case <-time.After(20 * time.Second):
+			t.Fatalf("started so far: %v", order)
+		}
+		// While it runs, no second check starts and the key is there.
+		select {
+		case id := <-started:
+			t.Fatalf("%s started beside a check let through", id)
+		case <-time.After(2500 * time.Millisecond):
+		}
+		kv := runningKV(t, js)
+		if _, err := kv.Get(ctx, LetThroughKey); err != nil {
+			t.Fatalf("the let-through key is not there while the check runs: %v", err)
+		}
+		release <- struct{}{}
+		// It is deleted when the check ends: the next check let through creates the key again at once, and a create fails
+		// while the key is there, so its start (the next round) is the proof; after the last one the key stays gone.
+		if len(order) == 3 {
+			deadline := time.Now().Add(10 * time.Second)
+			for {
+				if _, err := kv.Get(ctx, LetThroughKey); err != nil {
+					break
+				}
+				if time.Now().After(deadline) {
+					t.Fatal("the let-through key was not deleted when the last check ended")
+				}
+				time.Sleep(50 * time.Millisecond)
+			}
+		}
+	}
+	if want := []string{"a1", "a2", "a3"}; !reflect.DeepEqual(order, want) {
+		t.Fatalf("checks went %v, want the oldest first %v", order, want)
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if most != 1 {
+		t.Fatalf("%d checks ran at once while a delivery built", most)
+	}
+}
+
+func runningKV(t *testing.T, js *broker.JetStream) jetstream.KeyValue {
+	t.Helper()
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		t.Fatal(err)
+	}
+	kv, err := api.KeyValue(t.Context(), broker.CheckRunningBucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return kv
+}
+
+// 1688: the create of the let-through key succeeds for exactly one of holders asking at once, and a key its holder stopped
+// refreshing expires, after which the next create succeeds.
+func TestNatsOneCreateOfTheLetThroughKeySucceedsAndAStoppedHoldersKeyExpires(t *testing.T) {
+	js, _ := aBusWithTheBuckets(t)
+	gate := busGate{conn: js.Conn()}
+	ctx := t.Context()
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	won := 0
+	for i := range 8 {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			ok, err := gate.TakeLetThrough(ctx, fmt.Sprint("holder-", i))
+			if err != nil {
+				t.Error(err)
+				return
+			}
+			if ok {
+				mu.Lock()
+				won++
+				mu.Unlock()
+			}
+		}()
+	}
+	wg.Wait()
+	if won != 1 {
+		t.Fatalf("%d of 8 holders created the let-through key, want exactly one", won)
+	}
+	// The holder stopped: nothing refreshes the key, and the bucket's age (two seconds here) removes it.
+	deadline := time.Now().Add(15 * time.Second)
+	for {
+		ok, err := gate.TakeLetThrough(ctx, "next")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if ok {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("the key of a stopped holder did not expire")
+		}
+		time.Sleep(250 * time.Millisecond)
+	}
+	// And a refreshed key does not expire.
+	for range 6 {
+		if err := gate.KeepLetThrough(ctx, "next"); err != nil {
+			t.Fatal(err)
+		}
+		time.Sleep(700 * time.Millisecond)
+	}
+	if ok, _ := gate.TakeLetThrough(ctx, "other"); ok {
+		t.Fatal("a refreshed key expired")
+	}
+	var v LetThrough
+	e, err := runningKV(t, js).Get(ctx, LetThroughKey)
+	if err != nil || json.Unmarshal(e.Value(), &v) != nil || v.Builder != "next" {
+		t.Fatalf("the key says %+v (%v)", v, err)
+	}
+}
+
+// 1687: the oldest waiting ask is read without being taken: no delivery is counted against it.
+func TestNatsTheOldestWaitingAskIsReadWithoutTakingIt(t *testing.T) {
+	js, _ := aBusWithTheBuckets(t)
+	ctx := t.Context()
+	asks := BuildsOn(js, TheBuildMachine)
+	old := aCheckAskedAgo(45*time.Minute, 1, false)
+	if err := asks.Ask(ctx, old); err != nil {
+		t.Fatal(err)
+	}
+	gate := busGate{conn: js.Conn()}
+	q := &takeQueue{kind: queueOther, worker: broker.CheckWorkers()[1]}
+	at, found, err := gate.OldestWaiting(ctx, q)
+	if err != nil || !found {
+		t.Fatalf("the first waiting ask was not read: %v %v", found, err)
+	}
+	if want, _ := BuildAskedAt(old.ID); !at.Equal(want) {
+		t.Fatalf("read %v, asked %v", at, want)
+	}
+	for range 3 {
+		if _, _, err := gate.OldestWaiting(ctx, q); err != nil {
+			t.Fatal(err)
+		}
+	}
+	info, err := js.Context().ConsumerInfo(q.worker.Stream, q.worker.Name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.Delivered.Consumer != 0 || info.NumAckPending != 0 || info.NumPending != 1 {
+		t.Fatalf("reading the age took the ask: delivered %d, pending acks %d, waiting %d", info.Delivered.Consumer, info.NumAckPending, info.NumPending)
+	}
+	// And an empty queue reads as none.
+	if _, found, err := gate.OldestWaiting(ctx, &takeQueue{kind: queueReady, worker: broker.CheckWorkers()[0]}); err != nil || found {
+		t.Fatalf("an empty queue read as %v (%v)", found, err)
+	}
+}
+
+// Issue 1747: a message that reaches a builder's subscription outside a fetch is not held while the builder runs another ask. A
+// stray check lands in the other queue's subscription of builder A, which then runs a long check; the server hands the stray
+// out again when its wait for an acknowledgement ends (two seconds here), builder B runs it, and A, finished, does not run it a
+// second time from the subscription it held.
+func TestNatsAMessageDeliveredOutsideAFetchIsNotHeldWhileTheBuilderRunsAnotherAsk(t *testing.T) { // 1747
+	js := aBusWithTheBuildRole(t)
+	ctx, stop := context.WithCancel(t.Context())
+	defer stop()
+	for _, w := range broker.CheckWorkers() {
+		w.AckWaitSeconds = 2
+		if err := js.EnsureConsumer(w); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// The long check says it is alive well within the ack wait of two seconds.
+	was := heartbeatWhileWorking
+	heartbeatWhileWorking = 400 * time.Millisecond
+	t.Cleanup(func() { heartbeatWhileWorking = was })
+	asks := BuildsOn(js, TheBuildMachine)
+	long := aPullCheck("build-"+fmt.Sprint(time.Now().UnixNano()), 1, "head-1-0000000", "trunk-0-000000", true)
+	stray := aPullCheck("build-"+fmt.Sprint(time.Now().UnixNano()+1), 2, "head-2-0000000", "trunk-0-000000", false)
+	for _, r := range []BuildRequest{long, stray} {
+		if err := asks.Ask(ctx, r); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	var mu sync.Mutex
+	runs := map[string][]string{} // ask id -> the machines that ran it
+	release := make(chan struct{})
+	started := make(chan string, 8)
+	do := func(on string) func(context.Context, Build) {
+		return func(ctx context.Context, work Build) {
+			id := work.Request().ID
+			mu.Lock()
+			runs[id] = append(runs[id], on)
+			mu.Unlock()
+			started <- on + ":" + id
+			if id == long.ID {
+				<-release
+			}
+			_ = work.Announce(ctx, BuildResult{ID: id, On: on, Failed: "no"})
+			_ = work.Done()
+		}
+	}
+
+	var a *natsMachine
+	strayLanded := make(chan error, 1)
+	a = MachineOverNATSWith(js, "ace", TheBuildMachine, MachineOptions{Claim: func() bool {
+		// The moment the long check is claimed: the server hands the other queue's waiting check to A's subscription there, as
+		// one that arrives after a fetch gave up does. Done once.
+		select {
+		case <-strayLanded:
+			return true
+		default:
+		}
+		other := a.subs[2]
+		inbox := strings.TrimSuffix(other.sub.Subject, "*") + "stray"
+		body, _ := json.Marshal(map[string]any{"batch": 1, "expires": int64(30 * time.Second)})
+		if err := js.Conn().PublishRequest("$JS.API.CONSUMER.MSG.NEXT."+other.worker.Stream+"."+other.worker.Name, inbox, body); err != nil {
+			strayLanded <- err
+			return true
+		}
+		_ = js.Conn().Flush()
+		for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
+			if n, _, _ := other.sub.Pending(); n > 0 {
+				break
+			}
+		}
+		strayLanded <- nil
+		return true
+	}}).(*natsMachine)
+	defer a.Close()
+	go func() { _ = a.Take(ctx, do("ace")) }()
+
+	select {
+	case got := <-started:
+		if got != "ace:"+long.ID {
+			t.Fatalf("started %s, want the long check on ace first", got)
+		}
+	case <-time.After(15 * time.Second):
+		t.Fatal("the long check did not start")
+	}
+	if err := <-strayLanded; err != nil {
+		t.Fatal(err)
+	}
+
+	// B, free, runs the stray once the server hands it out again.
+	b := MachineOverNATS(js, "shanks")
+	defer b.Close()
+	go func() { _ = b.Take(ctx, do("shanks")) }()
+	select {
+	case got := <-started:
+		if got != "shanks:"+stray.ID {
+			t.Fatalf("started %s, want the stray on shanks", got)
+		}
+	case <-time.After(20 * time.Second):
+		t.Fatal("the stray was held by the builder that ran the long check and nobody ran it")
+	}
+	// A finishes, and does not run the stray from the subscription it held.
+	release <- struct{}{}
+	select {
+	case got := <-started:
+		t.Fatalf("%s started after the long check: the stray ran twice", got)
+	case <-time.After(6 * time.Second):
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if len(runs[stray.ID]) != 1 {
+		t.Fatalf("the stray ran on %v, want once", runs[stray.ID])
+	}
+}
+
+// Review of 1688: a young check pending redelivery (handed back by a Nak) stands ahead of a check older than 30 minutes in the
+// same queue while a plan builds. The let-through fetch would return the young one first, so nothing is fetched: the young
+// check's delivery count stays 1, it is not handed back every second until it is dead-lettered, and it is not run.
+func TestNatsAYoungRedeliveryAheadOfAnOldCheckIsNeverFetchedByTheLetThrough(t *testing.T) {
+	js, plans := aBusWithTheBuckets(t)
+	ctx, stop := context.WithCancel(t.Context())
+	defer stop()
+	asks := BuildsOn(js, TheBuildMachine)
+	young := aCheckAskedAgo(time.Minute, 1, false)
+	if err := asks.Ask(ctx, young); err != nil {
+		t.Fatal(err)
+	}
+	// Someone took it and handed it back: it is pending redelivery.
+	worker := broker.CheckWorkers()[1]
+	sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := sub.Fetch(1, nats.MaxWait(5*time.Second))
+	if err != nil || len(got) != 1 {
+		t.Fatalf("the young check was not fetched to be handed back: %v", err)
+	}
+	if err := got[0].Nak(); err != nil {
+		t.Fatal(err)
+	}
+	_ = sub.Unsubscribe()
+	old := aCheckAskedAgo(45*time.Minute, 2, false)
+	if err := asks.Ask(ctx, old); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := plans.Put(ctx, "plan-1", []byte(`{"plan":"plan-1"}`)); err != nil {
+		t.Fatal(err)
+	}
+	started := make(chan string, 4)
+	machine := MachineOverNATS(js, "ace")
+	defer machine.Close()
+	go func() {
+		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
+			started <- work.Request().ID
+			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "ace", Failed: "no"})
+			_ = work.Done()
+		})
+	}()
+	select {
+	case id := <-started:
+		t.Fatalf("%s was run while the young redelivery stood ahead of it", id)
+	case <-time.After(6 * time.Second):
+	}
+	info, err := js.Context().ConsumerInfo(worker.Stream, worker.Name)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if info.Delivered.Consumer != 1 || info.NumRedelivered != 0 {
+		t.Fatalf("the young check was handed out %d time(s), %d redelivered: the let-through fetched it", info.Delivered.Consumer, info.NumRedelivered)
+	}
+}

internal/link/declare.go

@@ -41,13 +41,54 @@ func Declare(ctx context.Context, bus Bus, signer Signer, node string,
 		return fmt.Errorf("cannot sign a declaration for %s: %w", node, err)
 	}
 
-	body, err := json.Marshal(Signed{Declaration: declaration, Signature: signature})
+	body, err := Envelope(declaration, signature)
 	if err != nil {
 		return err
 	}
 
+	// **Measured before it is sent** (novox/hq issue 1689): the bare bus error said only "maximum payload
+	// exceeded", on 2026-10-21 for the control node's declaration, 2 KB over. The limit is the connection's own.
+	if limit := payloadLimitOf(bus); limit > 0 {
+		size := int64(len(body))
+		if DeclarationMeasured != nil {
+			DeclarationMeasured(ctx, node, size, limit)
+		}
+		if size > limit {
+			return fmt.Errorf("%s's declaration is %d bytes on the wire (%d bytes of declaration, signed and encoded), "+
+				"over the bus's largest message of %d bytes by %d: it cannot be sent (novox/hq issue 1689)",
+				node, size, len(declaration), limit, size-limit)
+		}
+	}
+
 	publish, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 
 	return bus.PublishDeclaration(publish, node, body)
 }
+
+// Envelope is the message a declaration goes on the bus as: the declaration and its signature, each as base64 in one JSON
+// document. Its length is what the bus's largest message bounds.
+func Envelope(declaration, signature []byte) ([]byte, error) {
+	return json.Marshal(Signed{Declaration: declaration, Signature: signature})
+}
+
+// NearTheLimit is the share of the bus's largest message at which a declaration is said to be near it: three quarters, so that
+// a declaration which should stay far from the limit raises the condition while there is still room to act (novox/hq ADR 0486
+// §8; the first fix of issue 1689 warned at nine tenths while every file was still inline).
+const NearTheLimit = 0.75
+
+// DeclarationMeasured is told, at every send of a declaration, its size on the wire and the bus's largest message,
+// so the controller raises a condition when it nears the limit and clears it when it is back under (issue 1689).
+// Set by the controller; nil in a test or a tool.
+var DeclarationMeasured func(ctx context.Context, node string, size, limit int64)
+
+// payloadLimiter is a bus that knows its largest message.
+type payloadLimiter interface{ MaxPayload() int64 }
+
+// payloadLimitOf is the bus's largest message in bytes, or 0 when the bus does not say.
+func payloadLimitOf(bus Bus) int64 {
+	if l, ok := bus.(payloadLimiter); ok {
+		return l.MaxPayload()
+	}
+	return 0
+}

internal/link/declare_room_integration_test.go

@@ -0,0 +1,52 @@
+//go:build integration
+
+package link
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/novox/mesh-controller/internal/testbus"
+)
+
+// Against a real bus whose largest message is small: the limit is read from the connection, the one that fits goes
+// through, the one over is refused by us with its size and the limit, not by the client's bare error (issue 1689).
+func TestADeclarationIsMeasuredAgainstTheRealBusLimit(t *testing.T) {
+	s := testbus.StartWithMaxPayload(t, 2048)
+	conn, err := nats.Connect(s.ClientURL())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	js, err := conn.JetStream()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := js.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"}}); err != nil {
+		t.Fatal(err)
+	}
+	bus := OverNATS{Conn: conn, JS: js}
+	if bus.MaxPayload() != 2048 {
+		t.Fatalf("the limit read from the connection is %d, expected 2048", bus.MaxPayload())
+	}
+	var told []int64
+	was := DeclarationMeasured
+	DeclarationMeasured = func(_ context.Context, _ string, size, limit int64) { told = append(told, size, limit) }
+	t.Cleanup(func() { DeclarationMeasured = was })
+
+	fits := []byte(`{"a":"` + strings.Repeat("x", 600) + `"}`)
+	if err := Declare(t.Context(), bus, fixedSigner{}, "novox", fits, 5*time.Second); err != nil {
+		t.Fatalf("a declaration under the limit: %v", err)
+	}
+	over := []byte(`{"a":"` + strings.Repeat("x", 1600) + `"}`)
+	err = Declare(t.Context(), bus, fixedSigner{}, "novox", over, 5*time.Second)
+	if err == nil || !strings.Contains(err.Error(), "2048 bytes") || strings.Contains(err.Error(), "maximum payload exceeded") {
+		t.Fatalf("a declaration over the limit: %v", err)
+	}
+	if len(told) != 4 || told[1] != 2048 || told[2] <= 2048 {
+		t.Fatalf("measured %v", told)
+	}
+}

internal/link/declare_room_test.go

@@ -0,0 +1,57 @@
+package link
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+)
+
+type roomBus struct {
+	Bus
+	limit int64
+	sent  [][]byte
+}
+
+func (b *roomBus) MaxPayload() int64 { return b.limit }
+func (b *roomBus) PublishDeclaration(_ context.Context, _ string, body []byte) error {
+	b.sent = append(b.sent, body)
+	return nil
+}
+
+type fixedSigner struct{}
+
+func (fixedSigner) Sign(context.Context, []byte) ([]byte, error) { return []byte("sig"), nil }
+
+// A declaration is measured as it goes on the wire against the bus's own limit; one over it is refused saying size and
+// limit, and what the measure sees is told to whoever raises a condition (issue 1689).
+func TestADeclarationOverTheBusLimitIsRefusedSayingSizeAndLimit(t *testing.T) {
+	var seen []int64
+	was := DeclarationMeasured
+	DeclarationMeasured = func(_ context.Context, node string, size, limit int64) {
+		if node != "novox" || limit != 400 {
+			t.Errorf("measured %s against %d", node, limit)
+		}
+		seen = append(seen, size)
+	}
+	t.Cleanup(func() { DeclarationMeasured = was })
+
+	bus := &roomBus{limit: 400}
+	small := []byte(`{"a":"` + strings.Repeat("x", 100) + `"}`)
+	if err := Declare(t.Context(), bus, fixedSigner{}, "novox", small, time.Second); err != nil || len(bus.sent) != 1 {
+		t.Fatalf("a small declaration: %v, %d sent", err, len(bus.sent))
+	}
+	if int64(len(bus.sent[0])) != seen[0] {
+		t.Fatalf("measured %d bytes, sent %d", seen[0], len(bus.sent[0]))
+	}
+
+	big := []byte(`{"a":"` + strings.Repeat("x", 300) + `"}`)
+	err := Declare(t.Context(), bus, fixedSigner{}, "novox", big, time.Second)
+	if err == nil || !strings.Contains(err.Error(), "400 bytes") || !strings.Contains(err.Error(), "novox") ||
+		!strings.Contains(err.Error(), "over") || strings.Contains(err.Error(), "maximum payload exceeded") {
+		t.Fatalf("an oversize declaration: %v", err)
+	}
+	if len(bus.sent) != 1 || len(seen) != 2 || seen[1] <= 400 {
+		t.Fatalf("it was sent (%d) or not measured (%v)", len(bus.sent), seen)
+	}
+}

internal/link/held_asks_test.go

@@ -0,0 +1,43 @@
+package link
+
+import (
+	"testing"
+	"time"
+)
+
+// An ask a machine holds but does not start gets no heartbeat, and the bus hands it to another machine after the ack
+// wait, so two machines run one check (issue 1649).
+
+// A fetch's server-side pull request ends clearly before the fetch does, so the server's answer and any message it could still
+// deliver come inside the fetch and never into a buffer nobody reads while the machine builds.
+func TestAFetchsPullRequestEndsClearlyBeforeTheFetchDoes(t *testing.T) {
+	for name, wait := range map[string]time.Duration{"a check queue": checkTakeWait, "the idle wait": takeIdleWait} {
+		if margin := wait - fetchExpiryOf(wait); margin < minFetchMargin {
+			t.Errorf("%s: the pull request ends %v before a fetch of %v does; want at least %v", name, margin, wait, minFetchMargin)
+		}
+	}
+}
+
+func TestAnAskAnotherBuilderIsRunningIsADuplicateAndYoursIsNot(t *testing.T) {
+	r := aPullCheck("build-1", 423, "head-4771ee53aa", "trunk-0123456789", false)
+	run := CheckRunning{Head: r.Ref, Trunk: "trunk-0123456789", Builder: "shanks", Build: "build-1", Started: time.Now().Add(-time.Minute), Seen: time.Now().Add(-4 * time.Second)}
+	dead := run
+	dead.Seen = time.Now().Add(-60 * time.Second)
+	for name, c := range map[string]struct {
+		running CheckRunning
+		found   bool
+		me      string
+		dup     bool
+	}{
+		"another builder runs this very ask":                {run, true, "ace", true},
+		"its builder stopped renewing the key a minute ago": {dead, true, "ace", false},
+		"this builder runs it itself":                       {run, true, "shanks", false},
+		"nobody runs it":                                    {CheckRunning{}, false, "ace", false},
+		"another ask of the same head":                      {CheckRunning{Head: r.Ref, Trunk: "trunk-0123456789", Builder: "shanks", Build: "build-0"}, true, "ace", false},
+		"the same ask at another head":                      {CheckRunning{Head: "head-other-0000", Trunk: "trunk-0123456789", Builder: "shanks", Build: "build-1"}, true, "ace", false},
+	} {
+		if got := duplicateOf(c.running, c.found, r, c.me) != ""; got != c.dup {
+			t.Errorf("%s: duplicate = %v; want %v", name, got, c.dup)
+		}
+	}
+}

internal/link/holds.go

@@ -0,0 +1,384 @@
+package link
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1652, 1687 and 1688).
+//
+// **The layout other parts read** (ADR 0484 clause 12 has the checker read the same key; a layout shared between modules
+// belongs in mesh-sdk, and this one is not there yet):
+//
+//   - bucket `mesh-controller_plans-building` (broker.PlansBuildingBucket): one key per delivery plan in its build phase, named
+//     by the plan's id (`plan-<digits>`), value a PlanBuilding as JSON. Written by the controller (lease holder) alone: put when
+//     the plan's first build is asked, put again about every thirty seconds, deleted when the phase ends; the bucket's age
+//     (two minutes) removes the key of a controller that stopped. A reader lists the keys and needs no value to decide.
+//   - key `let-through` in bucket `mesh-controller_checks-running` (broker.CheckRunningBucket), value a LetThrough as JSON:
+//     created by the holder that takes a check which waited LetThroughAfter, refreshed at each heartbeat of that check, deleted
+//     when it ends. At most one such check runs at a time on the build seat, because the create fails while the key is there.
+
+// LetThroughAfter is how long a check waits, while a delivery builds, before it is let through (ADR 0483 clause 3).
+const LetThroughAfter = 30 * time.Minute
+
+// LetThroughKey is the key of the one check let through, in the bucket of the checks running. It has no dot, so it is no
+// pull request's key (`<owner>.<repo>.<number>`) and no group's (`group.<name>`).
+const LetThroughKey = "let-through"
+
+// PlanBuilding is the value of a plan's key in the bucket of the plans building.
+type PlanBuilding struct {
+	Plan  string    `json:"plan"`
+	Since time.Time `json:"since"`
+	Seen  time.Time `json:"seen"`
+}
+
+// LetThrough is the value of the let-through key: the builder that runs the check and when it took it.
+type LetThrough struct {
+	Builder string    `json:"builder"`
+	Since   time.Time `json:"since"`
+}
+
+// checkGate is what a taking loop asks of the bus to hold checks while a delivery builds; a variable of the machine so a test
+// hands out against fakes.
+type checkGate interface {
+	// Building is the ids of the plans in their build phase: the keys of the bucket. An error is a bucket that cannot be read.
+	Building(ctx context.Context) ([]string, error)
+	// OldestWaiting is when the first waiting ask of a check queue was asked, read without taking it.
+	OldestWaiting(ctx context.Context, q *takeQueue) (time.Time, bool, error)
+	// TakeLetThrough creates the let-through key for a builder: false when it is there (another builder runs a check let through).
+	TakeLetThrough(ctx context.Context, builder string) (bool, error)
+	// KeepLetThrough puts the key again, and DropLetThrough deletes it.
+	KeepLetThrough(ctx context.Context, builder string) error
+	DropLetThrough(ctx context.Context) error
+}
+
+// busGate is the gate over a connection.
+type busGate struct{ conn *nats.Conn }
+
+func (g busGate) Building(ctx context.Context) ([]string, error) {
+	return BuildingPlans(ctx, g.conn)
+}
+
+// BuildingPlans is the ids of the delivery plans in their build phase, from the controller's bucket. A reader that is not the
+// build seat's (the checker, ADR 0484 clause 12) calls this. A bucket that cannot be read is an error; none is none building.
+func BuildingPlans(ctx context.Context, conn *nats.Conn) ([]string, error) {
+	api, err := jetstream.New(conn)
+	if err != nil {
+		return nil, err
+	}
+	kv, err := api.KeyValue(ctx, broker.PlansBuildingBucket)
+	if err != nil {
+		return nil, fmt.Errorf("the plans building: %w", err)
+	}
+	keys, err := kv.Keys(ctx)
+	if errors.Is(err, jetstream.ErrNoKeysFound) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("the plans building: %w", err)
+	}
+	return keys, nil
+}
+
+// OldestWaiting is when the check a fetch of this queue would return was asked, read without taking anything.
+//
+// **It sees redeliveries** (review of issue 1688). A work queue keeps only what is not acknowledged: what the worker handed out
+// (sequence up to its delivered mark) is running or waiting to be handed out again, and a fetch returns those before anything
+// new. An ask there that is not said running (a key of the checks running names its build) is such a redelivery. If one is
+// younger than the bound, the queue's next fetch would return it, and a young check is never let through: the queue is not a
+// candidate, and nothing is fetched, so nothing is handed back and no delivery is spent. Otherwise the candidate is the oldest
+// of the redeliveries and the first ask after the delivered mark.
+func (g busGate) OldestWaiting(ctx context.Context, q *takeQueue) (time.Time, bool, error) {
+	info, err := jsOf(g.conn).ConsumerInfo(q.worker.Stream, q.worker.Name, nats.Context(ctx))
+	if err != nil {
+		return time.Time{}, false, fmt.Errorf("the %s worker: %w", q.kind, err)
+	}
+	var oldest time.Time
+	note := func(at time.Time) {
+		if oldest.IsZero() || at.Before(oldest) {
+			oldest = at
+		}
+	}
+	for seq := uint64(1); seq <= info.Delivered.Stream; {
+		data, stored, got, err := g.directGet(ctx, q, seq)
+		if err != nil {
+			return time.Time{}, false, err
+		}
+		if got == 0 || got > info.Delivered.Stream {
+			break
+		}
+		seq = got + 1
+		var r BuildRequest
+		if json.Unmarshal(data, &r) != nil {
+			continue
+		}
+		running, ok, err := ReadRunning(ctx, g.conn, r)
+		if err != nil {
+			return time.Time{}, false, fmt.Errorf("the checks running: %w", err)
+		}
+		if ok && running.Build == r.ID {
+			continue // running now
+		}
+		at := askedAtOf(data, stored)
+		if time.Since(at) < LetThroughAfter {
+			return time.Time{}, false, nil // a young redelivery is next in line
+		}
+		note(at)
+	}
+	data, stored, got, err := g.directGet(ctx, q, info.Delivered.Stream+1)
+	if err != nil {
+		return time.Time{}, false, err
+	}
+	if got != 0 {
+		note(askedAtOf(data, stored))
+	}
+	return oldest, !oldest.IsZero(), nil
+}
+
+// directGet reads the first ask at or after a sequence of a check queue's stream, without taking it: its body, the time the
+// stream stored it, and its sequence (0 when there is none).
+func (g busGate) directGet(ctx context.Context, q *takeQueue, seq uint64) ([]byte, string, uint64, error) {
+	body, _ := json.Marshal(map[string]any{"seq": seq, "next_by_subj": q.worker.Filters[0]})
+	reply, err := g.conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET."+q.worker.Stream, body)
+	if err != nil {
+		return nil, "", 0, fmt.Errorf("the asks of the %s queue: %w", q.kind, err)
+	}
+	if st := reply.Header.Get("Status"); st == "404" || st == "408" {
+		return nil, "", 0, nil
+	}
+	got, _ := strconv.ParseUint(reply.Header.Get("Nats-Sequence"), 10, 64)
+	return reply.Data, reply.Header.Get("Nats-Time-Stamp"), got, nil
+}
+
+// askedAtOf is when a request was asked: its id says (NewBuildID), the stream's own time otherwise, and a request that says
+// neither is taken as asked now, so it is never let through on a guess.
+func askedAtOf(data []byte, stored string) time.Time {
+	var r BuildRequest
+	if json.Unmarshal(data, &r) == nil {
+		if at, ok := BuildAskedAt(r.ID); ok {
+			return at
+		}
+	}
+	if at, err := time.Parse(time.RFC3339Nano, stored); err == nil {
+		return at
+	}
+	return time.Now()
+}
+
+func jsOf(conn *nats.Conn) nats.JetStreamContext {
+	js, _ := conn.JetStream()
+	return js
+}
+
+func (g busGate) TakeLetThrough(ctx context.Context, builder string) (bool, error) {
+	kv, err := runningBucket(ctx, g.conn)
+	if err != nil {
+		return false, err
+	}
+	body, _ := json.Marshal(LetThrough{Builder: builder, Since: time.Now().UTC()})
+	if _, err := kv.Create(ctx, LetThroughKey, body); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+func (g busGate) KeepLetThrough(ctx context.Context, builder string) error {
+	kv, err := runningBucket(ctx, g.conn)
+	if err != nil {
+		return err
+	}
+	body, _ := json.Marshal(LetThrough{Builder: builder, Since: time.Now().UTC()})
+	_, err = kv.Put(ctx, LetThroughKey, body)
+	return err
+}
+
+func (g busGate) DropLetThrough(ctx context.Context) error {
+	kv, err := runningBucket(ctx, g.conn)
+	if err != nil {
+		return err
+	}
+	if err := kv.Delete(ctx, LetThroughKey); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
+		return err
+	}
+	return nil
+}
+
+// CheckHold is what a reader of the queue (the controller's own verb) says of why checks wait: the plans building and the check
+// let through now, if one.
+type CheckHold struct {
+	Plans      []string
+	LetThrough *LetThrough
+}
+
+// ReadCheckHold reads the plans building and the let-through key. A bucket that cannot be read is an error the caller says.
+func ReadCheckHold(ctx context.Context, conn *nats.Conn) (CheckHold, error) {
+	var h CheckHold
+	plans, err := BuildingPlans(ctx, conn)
+	if err != nil {
+		return h, err
+	}
+	h.Plans = plans
+	kv, err := runningBucket(ctx, conn)
+	if err != nil {
+		return h, err
+	}
+	e, err := kv.Get(ctx, LetThroughKey)
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return h, nil
+	}
+	if err != nil {
+		return h, err
+	}
+	var l LetThrough
+	if json.Unmarshal(e.Value(), &l) == nil {
+		h.LetThrough = &l
+	}
+	return h, nil
+}
+
+// gateOf is the machine's gate: the one a test set, or the bus's.
+func (m *natsMachine) gateOf() checkGate {
+	if m.gate != nil {
+		return m.gate
+	}
+	return busGate{conn: m.js.Conn()}
+}
+
+// sayOnce says a fault of the hold the first time it happens and again once it has recovered and happened anew, so a bucket
+// that cannot be read is in the log once and not every second.
+func (m *natsMachine) sayOnce(kind string, err error) {
+	if m.said == nil {
+		m.said = map[string]bool{}
+	}
+	if err == nil {
+		delete(m.said, kind)
+		return
+	}
+	if !m.said[kind] {
+		m.said[kind] = true
+		fmt.Fprintf(os.Stderr, "%s; read as empty, so no check is held for it: %v\n", kind, err)
+	}
+}
+
+// building says whether a delivery builds: any key in the bucket of the plans building. **A bucket that cannot be read is said
+// and read as empty** (ADR 0483 clause 2): a check wrongly held costs a pull request its turn, a fault that held every check
+// would stop the mesh checking.
+func (m *natsMachine) building(ctx context.Context) []string {
+	plans, err := m.gateOf().Building(ctx)
+	m.sayOnce("the bucket of the plans building cannot be read", err)
+	if err != nil {
+		// empty-on-error: ADR 0483 clause 2 — a bucket that cannot be read is said (sayOnce) and read as empty, because a check wrongly held costs a pull request its turn and a fault that held every check would stop the mesh checking.
+		return nil
+	}
+	return plans
+}
+
+// letOneThrough takes the oldest waiting check of the two check queues once it waited LetThroughAfter, by the one let-through
+// key: nil when none waited that long, when another holder runs the check let through, or when the check was taken by
+// another holder in the moment. The caller owns the key while it returns a message and releases it with dropLetThrough.
+func (m *natsMachine) letOneThrough(ctx context.Context) (*nats.Msg, *takeQueue) {
+	gate := m.gateOf()
+	var oldest *takeQueue
+	var oldestAt time.Time
+	for _, q := range m.subs[1:] {
+		if !q.bound {
+			continue
+		}
+		at, found, err := gate.OldestWaiting(ctx, q)
+		m.sayOnce("the first waiting check of the "+q.kind+" queue cannot be read", err)
+		if err != nil || !found {
+			continue
+		}
+		if oldest == nil || at.Before(oldestAt) {
+			oldest, oldestAt = q, at
+		}
+	}
+	if oldest == nil || time.Since(oldestAt) < m.letThroughAfter() {
+		return nil, nil
+	}
+	created, err := gate.TakeLetThrough(ctx, m.on)
+	m.sayOnce("the let-through key cannot be taken", err)
+	if err != nil || !created {
+		return nil, nil
+	}
+	trying, end := context.WithTimeout(ctx, checkTakeWait)
+	defer end()
+	fetched, err := oldest.pull.Fetch(1, nats.Context(trying))
+	if len(fetched) == 0 {
+		if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.Canceled) {
+			m.sayOnce("the "+oldest.kind+" check queue cannot be taken from", err)
+		}
+		m.dropLetThrough(ctx)
+		return nil, nil
+	}
+	// Another holder took the one that was oldest in the moment between the read and the fetch: what came is a younger check,
+	// handed back and the key released, never run during a delivery.
+	if askedAtOf(fetched[0].Data, "").Add(m.letThroughAfter()).After(time.Now()) {
+		_ = fetched[0].NakWithDelay(time.Minute) // a delay keeps it off the queue of redeliveries a fetch reads first
+		m.dropLetThrough(ctx)
+		return nil, nil
+	}
+	m.letKey = true
+	return fetched[0], oldest
+}
+
+func (m *natsMachine) letThroughAfter() time.Duration {
+	if m.opts.LetThroughAfter > 0 {
+		return m.opts.LetThroughAfter
+	}
+	return LetThroughAfter
+}
+
+// dropLetThrough deletes the let-through key this holder created.
+func (m *natsMachine) dropLetThrough(ctx context.Context) {
+	m.letKey = false
+	if err := m.gateOf().DropLetThrough(context.WithoutCancel(ctx)); err != nil {
+		fmt.Fprintf(os.Stderr, "could not take the let-through key away; it ages out: %v\n", err)
+	}
+}
+
+// keepLetThrough refreshes the key at every heartbeat while the check it was created for runs, and returns the end of that: it
+// stops the refreshing and deletes the key.
+func (m *natsMachine) keepLetThrough(ctx context.Context, every time.Duration) (end func()) {
+	done := make(chan struct{})
+	finished := make(chan struct{})
+	go func() {
+		defer close(finished)
+		tick := time.NewTicker(every)
+		defer tick.Stop()
+		for {
+			select {
+			case <-done:
+				return
+			case <-tick.C:
+				putting, stop := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
+				if err := m.gateOf().KeepLetThrough(putting, m.on); err != nil {
+					fmt.Fprintf(os.Stderr, "could not keep the let-through key: %v\n", err)
+				}
+				stop()
+			}
+		}
+	}()
+	return func() {
+		close(done)
+		<-finished
+		m.dropLetThrough(ctx)
+	}
+}
+
+// whyHeld words the plans a check is held for, for the log.
+func whyHeld(plans []string) string { return strings.Join(plans, ", ") }

internal/link/holds_test.go

@@ -0,0 +1,307 @@
+package link
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+// Checks wait while a delivery builds (novox/hq ADR 0483, issues 1687 and 1688): the order the taking loop hands out in, against
+// fake queues and a fake bus.
+
+// fakeQueue is a queue the loop fetches from: the asks waiting, handed out in order, and how often it was fetched from.
+type fakeQueue struct {
+	mu      sync.Mutex
+	waiting []*nats.Msg
+	fetched int
+}
+
+func (f *fakeQueue) Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.fetched++
+	if len(f.waiting) == 0 {
+		return nil, nats.ErrTimeout
+	}
+	m := f.waiting[0]
+	f.waiting = f.waiting[1:]
+	return []*nats.Msg{m}, nil
+}
+
+func (f *fakeQueue) fetches() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.fetched
+}
+
+// askedAgo is a request asked this long ago, as the id says, for a check of this pull request.
+func askedAgo(ago time.Duration, number int, ready bool) *nats.Msg {
+	id := fmt.Sprintf("build-%d", time.Now().Add(-ago).UnixNano())
+	body, _ := json.Marshal(BuildRequest{ID: id, Repository: "r", Check: &CheckRequest{Owner: "o", Repo: "r", Number: number, Ready: ready}})
+	return &nats.Msg{Data: body}
+}
+
+func buildAsk() *nats.Msg {
+	body, _ := json.Marshal(BuildRequest{ID: fmt.Sprintf("build-%d", time.Now().UnixNano()), Repository: "r"})
+	return &nats.Msg{Data: body}
+}
+
+// fakeGate is the bus as the loop sees it for the hold.
+type fakeGate struct {
+	mu         sync.Mutex
+	plans      []string
+	plansErr   error
+	letKey     bool
+	taken      int
+	kept       int
+	dropped    int
+	oldestFrom map[*takeQueue]*fakeQueue
+}
+
+func (g *fakeGate) Building(context.Context) ([]string, error) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	return g.plans, g.plansErr
+}
+
+// OldestWaiting reads the first waiting ask of the fake queue without taking it.
+func (g *fakeGate) OldestWaiting(_ context.Context, q *takeQueue) (time.Time, bool, error) {
+	f := g.oldestFrom[q]
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if len(f.waiting) == 0 {
+		return time.Time{}, false, nil
+	}
+	return askedAtOf(f.waiting[0].Data, ""), true, nil
+}
+
+func (g *fakeGate) TakeLetThrough(context.Context, string) (bool, error) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	if g.letKey {
+		return false, nil
+	}
+	g.letKey = true
+	g.taken++
+	return true, nil
+}
+
+func (g *fakeGate) KeepLetThrough(context.Context, string) error {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	g.kept++
+	return nil
+}
+
+func (g *fakeGate) DropLetThrough(context.Context) error {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	g.letKey = false
+	g.dropped++
+	return nil
+}
+
+func (g *fakeGate) setPlans(plans ...string) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	g.plans = plans
+}
+
+// aMachine is a loop on the build seat with a delivery queue and the two check queues, all fake.
+func aMachine() (m *natsMachine, builds, ready, other *fakeQueue, gate *fakeGate) {
+	builds, ready, other = &fakeQueue{}, &fakeQueue{}, &fakeQueue{}
+	subs := []*takeQueue{
+		{kind: queueBuilds, pull: builds, bound: true},
+		{kind: queueReady, pull: ready, bound: true},
+		{kind: queueOther, pull: other, bound: true},
+	}
+	gate = &fakeGate{oldestFrom: map[*takeQueue]*fakeQueue{subs[1]: ready, subs[2]: other}}
+	m = &natsMachine{on: "ace", seat: broker.TheBuildSeat, subs: subs, gate: gate}
+	return m, builds, ready, other, gate
+}
+
+func nextAsk(t *testing.T, m *natsMachine) (*nats.Msg, *takeQueue) {
+	t.Helper()
+	msg, q, err := m.next(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	return msg, q
+}
+
+// 1652/AC6, 1687: a waiting check is not taken while a plan key is there, also when the delivery's queue is empty for the moment
+// (between two tiers).
+func TestNoCheckIsTakenWhileAPlanBuilds(t *testing.T) {
+	m, builds, ready, other, gate := aMachine()
+	gate.setPlans("plan-1")
+	ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
+	other.waiting = []*nats.Msg{askedAgo(5*time.Minute, 2, false)}
+	// The delivery's queue is empty: between two tiers.
+	if msg, _ := nextAsk(t, m); msg != nil {
+		t.Fatalf("a check was taken while plan-1 builds: %s", msg.Data)
+	}
+	if ready.fetches() != 0 || other.fetches() != 0 {
+		t.Fatalf("the check queues were fetched from while plan-1 builds: ready %d, other %d", ready.fetches(), other.fetches())
+	}
+	if builds.fetches() == 0 {
+		t.Fatal("the delivery's queue was not looked at")
+	}
+}
+
+// A delivery build is still taken first, and the check waits behind it.
+func TestADeliveryBuildIsTakenWhileAPlanBuilds(t *testing.T) {
+	m, builds, ready, _, gate := aMachine()
+	gate.setPlans("plan-1")
+	builds.waiting = []*nats.Msg{buildAsk()}
+	ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
+	msg, q := nextAsk(t, m)
+	if msg == nil || q.kind != queueBuilds {
+		t.Fatalf("the delivery build was not taken first: %v %v", msg, q)
+	}
+	if ready.fetches() != 0 {
+		t.Fatal("a check queue was fetched from")
+	}
+}
+
+// It is taken once the key is gone (deleted at the phase's end, or expired).
+func TestACheckIsTakenOnceTheKeyIsGone(t *testing.T) {
+	m, _, ready, _, gate := aMachine()
+	gate.setPlans("plan-1")
+	ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
+	if msg, _ := nextAsk(t, m); msg != nil {
+		t.Fatal("taken while the key is there")
+	}
+	gate.setPlans()
+	msg, q := nextAsk(t, m)
+	if msg == nil || q.kind != queueReady {
+		t.Fatalf("the check was not taken once the key was gone: %v", q)
+	}
+	if m.letKey {
+		t.Fatal("a check taken without a delivery building took the let-through key")
+	}
+}
+
+// 1687: a bucket that cannot be read is said and read as empty, so the checks are taken.
+func TestAnUnreadableBucketHoldsNoCheck(t *testing.T) {
+	m, _, ready, _, gate := aMachine()
+	gate.plansErr = errors.New("permissions violation")
+	gate.plans = []string{"plan-1"}
+	ready.waiting = []*nats.Msg{askedAgo(2*time.Minute, 1, true)}
+	msg, q := nextAsk(t, m)
+	if msg == nil || q.kind != queueReady {
+		t.Fatal("a check was held because the bucket could not be read")
+	}
+	if !m.said["the bucket of the plans building cannot be read"] {
+		t.Fatalf("the fault was not said: %v", m.said)
+	}
+}
+
+// 1688: a check asked 30 minutes ago is let through while a key is there; the oldest across both queues by ask time, ahead of a
+// waiting delivery build; a younger one is never taken before it.
+func TestTheOldestCheckPastTheBoundGoesFirstAcrossBothQueues(t *testing.T) {
+	m, builds, ready, other, gate := aMachine()
+	gate.setPlans("plan-1")
+	builds.waiting = []*nats.Msg{buildAsk()}
+	ready.waiting = []*nats.Msg{askedAgo(31*time.Minute, 1, true)}
+	other.waiting = []*nats.Msg{askedAgo(50*time.Minute, 2, false)}
+	msg, q := nextAsk(t, m)
+	if msg == nil || q.kind != queueOther || !strings.Contains(string(msg.Data), `"number":2`) {
+		t.Fatalf("the oldest check (50 minutes, other queue) was not let through first: %v", q)
+	}
+	if !m.letKey || gate.taken != 1 {
+		t.Fatalf("the let-through key was not created for it: held %v, created %d", m.letKey, gate.taken)
+	}
+}
+
+// 1688: several checks past the bound go one at a time, oldest first: while the key is there every other holder takes builds.
+func TestChecksPastTheBoundGoOneAtATimeOldestFirst(t *testing.T) {
+	m, builds, ready, other, gate := aMachine()
+	gate.setPlans("plan-1")
+	ready.waiting = []*nats.Msg{askedAgo(33*time.Minute, 1, true), askedAgo(31*time.Minute, 3, true)}
+	other.waiting = []*nats.Msg{askedAgo(40*time.Minute, 2, false)}
+	var order []string
+	for range 3 {
+		msg, q := nextAsk(t, m)
+		if msg == nil {
+			t.Fatalf("no check let through after %v", order)
+		}
+		var r BuildRequest
+		_ = json.Unmarshal(msg.Data, &r)
+		order = append(order, fmt.Sprint(r.Check.Number, q.kind))
+		// While it runs another holder (the same fake gate, key held) takes builds only.
+		builds.waiting = []*nats.Msg{buildAsk()}
+		other2 := &natsMachine{on: "shanks", seat: broker.TheBuildSeat, subs: m.subs, gate: gate}
+		if next, nq := nextAsk(t, other2); next == nil || nq.kind != queueBuilds {
+			t.Fatalf("a second holder did not take a delivery build while the let-through key is held: %v", nq)
+		}
+		m.dropLetThrough(context.Background())
+	}
+	if want := []string{"2other", "1ready", "3ready"}; strings.Join(order, " ") != strings.Join(want, " ") {
+		t.Fatalf("checks went %v, want %v (oldest first, one at a time)", order, want)
+	}
+}
+
+// 1688: a check younger than the bound is never let through, and no key is created for it.
+func TestACheckYoungerThanTheBoundIsHeld(t *testing.T) {
+	m, _, ready, _, gate := aMachine()
+	gate.setPlans("plan-1")
+	ready.waiting = []*nats.Msg{askedAgo(29*time.Minute, 1, true)}
+	if msg, _ := nextAsk(t, m); msg != nil {
+		t.Fatal("a check of 29 minutes was let through")
+	}
+	if gate.taken != 0 {
+		t.Fatal("a let-through key was created for a young check")
+	}
+	if ready.fetches() != 0 {
+		t.Fatal("the age was read by a fetch; it must be read without taking")
+	}
+}
+
+// 1688: the holder that lost the race for the key, or the check, takes builds only; and a younger check that arrives in the
+// moment between the read and the fetch is handed back with the key released.
+func TestAYoungerCheckFetchedInTheRaceIsHandedBackAndTheKeyReleased(t *testing.T) {
+	m, _, ready, _, gate := aMachine()
+	gate.setPlans("plan-1")
+	// The read sees the old check; before the fetch another holder takes it and a young one is first.
+	gate.oldestFrom[m.subs[1]] = &fakeQueue{waiting: []*nats.Msg{askedAgo(40*time.Minute, 1, true)}}
+	ready.waiting = []*nats.Msg{askedAgo(time.Minute, 2, true)}
+	if msg, _ := nextAsk(t, m); msg != nil {
+		t.Fatalf("a young check was run during a delivery: %s", msg.Data)
+	}
+	if gate.letKey || m.letKey {
+		t.Fatal("the let-through key was left held")
+	}
+}
+
+// The key of a check let through is refreshed while it runs and deleted when it ends.
+func TestTheLetThroughKeyLivesAsLongAsTheCheckRuns(t *testing.T) {
+	m, _, ready, _, gate := aMachine()
+	gate.setPlans("plan-1")
+	ready.waiting = []*nats.Msg{askedAgo(40*time.Minute, 1, true)}
+	if msg, _ := nextAsk(t, m); msg == nil {
+		t.Fatal("not let through")
+	}
+	end := m.keepLetThrough(context.Background(), 20*time.Millisecond)
+	time.Sleep(120 * time.Millisecond)
+	gate.mu.Lock()
+	held, kept := gate.letKey, gate.kept
+	gate.mu.Unlock()
+	if !held || kept == 0 {
+		t.Errorf("while the check ran the key was held %v and refreshed %d times", held, kept)
+	}
+	end()
+	gate.mu.Lock()
+	defer gate.mu.Unlock()
+	if gate.letKey || m.letKey {
+		t.Error("the key was not deleted when the check ended")
+	}
+}

internal/link/queue.go

@@ -88,6 +88,40 @@ type Queue struct {
 	AckPending int         `json:"ack-pending"`
 	MaxDeliver int         `json:"max-deliver"`
 	Asks       []QueuedAsk `json:"asks"`
+	// Hold is why the checks wait: the delivery plans building and the check let through now (ADR 0483, issues 1687 and 1688);
+	// HoldFault says the bucket could not be read, in which case Hold is empty.
+	Hold      CheckHold `json:"hold"`
+	HoldFault string    `json:"hold-fault,omitempty"`
+}
+
+// LetThroughOf says which ask is let through under ADR 0483 clause 3: the check that waited LetThroughAfter or longer and is next
+// (the oldest waiting check across the two check queues, while a delivery builds), and the check in flight that was let through
+// (the let-through key names the machine that runs it). Empty when none.
+func (q Queue) LetThroughOf(now time.Time) (next, running string) {
+	if len(q.Hold.Plans) == 0 && q.Hold.LetThrough == nil {
+		return "", ""
+	}
+	var oldest *QueuedAsk
+	for i := range q.Asks {
+		a := &q.Asks[i]
+		if a.State != AskWaiting || a.Request.Check == nil || a.AskedAt.IsZero() {
+			continue
+		}
+		if oldest == nil || a.AskedAt.Before(oldest.AskedAt) {
+			oldest = a
+		}
+	}
+	if oldest != nil && len(q.Hold.Plans) > 0 && now.Sub(oldest.AskedAt) >= LetThroughAfter {
+		next = oldest.ID
+	}
+	if q.Hold.LetThrough != nil {
+		for _, a := range q.Asks {
+			if a.State == AskInFlight && a.Request.Check != nil && a.On == q.Hold.LetThrough.Builder {
+				running = a.ID
+			}
+		}
+	}
+	return next, running
 }
 
 // Of is the asks in one state, in queue order.
@@ -319,6 +353,16 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
 	for _, r := range reads {
 		q.Asks = append(q.Asks, ClassifyQueue(r.asks, r.delivered, r.ackPending, heard)...)
 	}
+	// And why the checks wait, for the build seat that has them (ADR 0483 clause 5). A bucket that cannot be read is said by the
+	// queue, which then names no plan.
+	if seat == broker.TheBuildSeat {
+		hold, err := ReadCheckHold(ctx, js.Conn())
+		if err != nil {
+			q.HoldFault = err.Error()
+		} else {
+			q.Hold = hold
+		}
+	}
 	return q, nil
 }
 

internal/link/stray_test.go

@@ -0,0 +1,54 @@
+package link
+
+import (
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+)
+
+// Issue 1747: a builder never holds a message it does not start. A message the server sends after a fetch gave up lands in that
+// queue's subscription; while an ask runs, the queues it does not come from are let go of, so such a message is not held
+// unheartbeated for the whole run, and the queues are bound again before the next fetch.
+
+func TestARunningAskLetsGoOfTheQueuesItDidNotComeFrom(t *testing.T) {
+	m, builds, _, other, _ := aMachine()
+	// A delivery build reached the builds queue's subscription after its fetch gave up: it sits in the buffer.
+	builds.waiting = []*nats.Msg{buildAsk()}
+	took := m.subs[1] // the running ask came from the ready queue
+
+	m.quiesce(took)
+
+	if !took.bound || took.pull == nil {
+		t.Error("the queue the running ask came from was let go of; its acknowledgements need it")
+	}
+	for _, q := range []*takeQueue{m.subs[0], m.subs[2]} {
+		if q.bound || q.pull != nil || q.sub != nil {
+			t.Errorf("the %s queue is still held while an ask runs", q.kind)
+		}
+	}
+	// Nothing is fetched from a queue that was let go of: a message in it cannot be taken unheartbeated.
+	if msg, _, _ := m.next(t.Context()); msg != nil {
+		t.Fatalf("a message was taken from a queue that was let go of: %s", msg.Data)
+	}
+	if builds.fetches() != 0 || other.fetches() != 0 {
+		t.Errorf("fetched from a queue that was let go of: builds %d, other %d", builds.fetches(), other.fetches())
+	}
+}
+
+// Once the ask is settled the queue it came from is let go of too: a second message the server delivered into its subscription
+// on a pull request still open is not run after the ask ends (review of 1747).
+func TestTheQueueTheAskCameFromIsLetGoOfWhenTheAskEnds(t *testing.T) {
+	m, _, ready, _, _ := aMachine()
+	ready.waiting = []*nats.Msg{askedAgo(time.Minute, 2, true)}
+	m.quiesce(nil)
+	for _, q := range m.subs {
+		if q.bound || q.pull != nil {
+			t.Errorf("the %s queue is still held after the ask ended", q.kind)
+		}
+	}
+	m.subs[0].bound, m.subs[0].pull = true, &fakeQueue{}
+	if msg, _, _ := m.next(t.Context()); msg != nil || ready.fetches() != 0 {
+		t.Fatal("a message held in the kept queue's subscription was taken after the ask ended")
+	}
+}

internal/testbus/testbus.go

@@ -56,10 +56,14 @@ var buses sync.Map
 
 // Start is a server with JetStream, for this test alone: on loopback, on a port the system chooses, its
 // store in the test's own directory.
-func Start(t testing.TB) *server.Server {
+func Start(t testing.TB) *server.Server { return StartWithMaxPayload(t, 0) }
+
+// StartWithMaxPayload is Start with the largest message the server takes set to maxPayload bytes; zero keeps the
+// server's default (1 MiB, as the mesh's bus has).
+func StartWithMaxPayload(t testing.TB, maxPayload int32) *server.Server {
 	t.Helper()
 	opts := &server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(),
-		NoLog: true, NoSigs: true}
+		NoLog: true, NoSigs: true, MaxPayload: maxPayload}
 	s, err := server.NewServer(opts)
 	if err != nil {
 		t.Fatalf("a bus for this test could not be made: %v", err)