Merge pull request 'The controller's check stops waiting on fixtures and fixed intervals (issue 591)' (#279) from fix/591-the-controller-check-runs-in-minutes into main

A renewal timed out at the renew interval, so the lease test's 1 s renewals
could lose the lease on a slow build machine (novox/hq issue 591, review of #279).

cmd/mesh-builder/main.go

@@ -66,6 +66,11 @@ broker to take work from or a registry to publish into:
 
 Without --registry the artifacts stay in this machine's container runtime, named by the
 digest of their own configuration. The result is printed as JSON.
+
+  mesh-builder ready
+
+exits 0 once the agent running in this container is bound to its build work and pulls it, and
+1 before: its container's health check.
 `
 
 func run() error {
@@ -76,12 +81,18 @@ func run() error {
 			return nil
 		case "build":
 			return buildOnce(context.Background(), os.Args[2:])
+		case "ready":
+			// The container's health check (novox/hq issue 544): exits non-zero until the agent takes work.
+			return ready()
 		default:
 			fmt.Print(usage)
 			return nil
 		}
 	}
 
+	if err := notReady(); err != nil {
+		return err
+	}
 	credential, err := brokerFrom()
 	if err != nil {
 		return err
@@ -134,7 +145,8 @@ func run() error {
 	// **What an earlier holder here left goes before anything is taken** (novox/hq issue 306): a holder
 	// recreated mid-check left the check's store and bus running, and the ask went to another machine.
 	h.removeLeftBehind()
-	machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
+	machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused,
+		Taking: func() { taking(seat) }})
 	defer machine.Close()
 
 	// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired

cmd/mesh-builder/ready.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// **The build agent says it is ready** (novox/hq issue 544). It listens on nothing, so no http or tcp
+// check can see it work; it writes a mark once it is bound to its worker and pulls build work, and its
+// container's health check runs `mesh-builder ready`, which passes only while the mark is there. Without
+// it the node-engine could judge the agent only alive, and every walk that moved it waited out the whole
+// grace on each machine.
+//
+// The mark lives in the container's own temporary directory, never in the workspace the machine keeps:
+// a new container starts without it, and a restarted one removes it before it binds again.
+func readyMark() string { return filepath.Join(os.TempDir(), "mesh-builder.ready") }
+
+// notReady removes the mark: a holder starting is not taking work until it says so again.
+func notReady() error {
+	if err := os.Remove(readyMark()); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("cannot remove the mark that this agent takes build work (%s): %w", readyMark(), err)
+	}
+	return nil
+}
+
+// taking writes the mark. A mark that cannot be written is said, and the agent goes on building: its
+// check then says it is not ready, which is the truth the gate should hear.
+func taking(seat string) {
+	if err := os.WriteFile(readyMark(), []byte(seat+"\n"), 0o644); err != nil {
+		fmt.Fprintf(os.Stderr, "cannot mark that this agent takes build work (%s): %v\n", readyMark(), err)
+		return
+	}
+	fmt.Fprintf(os.Stderr, "taking build work from %s\n", seat)
+}
+
+// ready is the health check: nil when this container's agent is bound and pulling build work.
+func ready() error {
+	if _, err := os.Stat(readyMark()); err != nil {
+		return fmt.Errorf("not taking build work yet: %s is not there", readyMark())
+	}
+	return nil
+}

cmd/mesh-builder/ready_test.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"os"
+	"testing"
+)
+
+// The health check passes only between the agent saying it takes work and its next start (novox/hq
+// issue 544).
+func TestTheAgentIsReadyOnlyOnceItTakesWork(t *testing.T) {
+	t.Setenv("TMPDIR", t.TempDir())
+	if err := ready(); err == nil {
+		t.Fatal("an agent that never took work is ready")
+	}
+	taking("mesh-build-machine")
+	if err := ready(); err != nil {
+		t.Fatalf("an agent taking work is not ready: %v", err)
+	}
+	// Started again, it is not ready until it binds again.
+	if err := notReady(); err != nil {
+		t.Fatal(err)
+	}
+	if err := ready(); err == nil {
+		t.Fatal("an agent starting again is ready before it takes work")
+	}
+	if err := notReady(); err != nil {
+		t.Fatalf("removing a mark that is not there failed: %v", err)
+	}
+	if _, err := os.Stat(readyMark()); err == nil {
+		t.Fatal("the mark is still there")
+	}
+}

cmd/mesh-controller/acting.go

@@ -64,6 +64,11 @@ type actor struct {
 	// the highest epoch issued, and what was said of it; zero when it was not (S12).
 	reset     time.Time
 	resetSaid string
+	// renewEvery and poll, when set, are how often the serving controller renews the lease it holds and
+	// looks again at one another holds, in place of the lease's own: a test's, so two controllers hand the
+	// lease over in a second rather than in a renewal of five (novox/hq issue 591). A renewal is given
+	// the lease's own RenewEvery to answer however often it is made, so a slow bus does not lose it.
+	renewEvery, poll time.Duration
 }
 
 // theLease is this process's standing to the lease.
@@ -223,7 +228,7 @@ func (a *actor) serveUnderTheLease(ctx context.Context, inv *inventory.Inventory
 	}
 	say := func(format string, args ...any) { fmt.Printf(format+"\n", args...) }
 	l, err := lease.Open(ctx, api, broker.LeaseBucket, lease.Options{Holder: holderOf(instance),
-		Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
+		RenewEvery: a.renewEvery, RenewWithin: lease.RenewEvery, Poll: a.poll, Floor: inv.HighestEpoch, Say: say, Health: controllerHealth, Moved: func(was, floor uint64) {
 			a.mu.Lock()
 			defer a.mu.Unlock()
 			a.reset = time.Now()

cmd/mesh-controller/delivery.go

@@ -1042,13 +1042,13 @@ func deliveryCommand(ctx context.Context, args []string) error {
 	return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub)
 }
 
-// appliedFrom answers a machine's first report after a send from the controller's `apply` durations; a lookup
-// that fails is a report not read, which leaves the walk's end unknown rather than wrong.
+// appliedFrom answers a machine's own report of the send a walk made to it from the controller's `apply` durations
+// (novox/hq issue 582); a lookup that fails is a report not read, which leaves the walk's end unknown rather than wrong.
 func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.AppliedLookup {
-	return func(node string, sent time.Time) (inventory.AppliedReport, bool) {
-		r, ok, err := inv.FirstAppliedAfter(ctx, node, sent)
+	return func(node string, send inventory.SentDeclaration, walkSent time.Time) (inventory.AppliedReport, bool) {
+		r, ok, err := inv.FirstAppliedAfter(ctx, node, send, walkSent)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, sent.Format(time.RFC3339), err)
+			fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, walkSent.Format(time.RFC3339), err)
 			return inventory.AppliedReport{}, false
 		}
 		return r, ok

cmd/mesh-controller/lease_test.go

@@ -55,8 +55,14 @@ func TestTwoControllersOneActs(t *testing.T) {
 	inv := inventory.ForTest(t)
 	ctx := t.Context()
 
+	// Each renews five times and looks again ten times as often as a serving controller does, so the
+	// handover and the loss below are seen in a second, not in a renewal of five (novox/hq issue 591): what
+	// is checked is the order of events, which does not change with the interval. Each renewal still has a
+	// serving controller's five seconds to answer (acting.go), so a slow bus does not lose A the lease.
+	const renewEvery, poll = lease.RenewEvery / 5, lease.Poll / 10
+
 	// Controller A takes the lease.
-	a := &actor{}
+	a := &actor{renewEvery: renewEvery, poll: poll}
 	aCtx, stopA := context.WithCancel(ctx)
 	defer stopA()
 	lostA, err := a.serveUnderTheLease(aCtx, inv, url)
@@ -69,7 +75,7 @@ func TestTwoControllersOneActs(t *testing.T) {
 	}
 
 	// Controller B starts while A holds it, and waits — acting on nothing meanwhile.
-	b := &actor{}
+	b := &actor{renewEvery: renewEvery, poll: poll}
 	bCtx, stopB := context.WithCancel(ctx)
 	defer stopB()
 	tookB := make(chan (<-chan struct{}), 1)
@@ -82,10 +88,11 @@ func TestTwoControllersOneActs(t *testing.T) {
 		}
 		tookB <- lost
 	}()
+	// B looks fifteen times, across one of A's renewals at least.
 	select {
 	case <-tookB:
 		t.Fatal("B took the lease while A held it")
-	case <-time.After(3 * time.Second):
+	case <-time.After(15 * poll):
 	}
 	if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {
 		t.Fatalf("B, waiting, may act: %v", err)
@@ -125,7 +132,7 @@ func TestTwoControllersOneActs(t *testing.T) {
 	}
 	select {
 	case <-lostB:
-	case <-time.After(2 * lease.RenewEvery):
+	case <-time.After(2 * renewEvery):
 		t.Fatal("B was not told it lost the lease")
 	}
 	if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) {

cmd/mesh-controller/machine_lines.go

@@ -0,0 +1,135 @@
+package main
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// machineLines is what a machine said it is made of (novox/hq issue 160), as `node` shows it: every fact in
+// one block with its age, the architecture and kernel the profile always carried among them, and each fact
+// the machine could not determine said as unknown with its why — never as a zero, and never left out.
+func machineLines(a inventory.MachineAccount, had bool, now time.Time) []string {
+	if !had {
+		return []string{"  what it is made of: never said — this machine has not reported a profile"}
+	}
+	m := a.Machine
+	if m == nil {
+		return []string{
+			"  what it is made of: not said — its node-engine is older than the one that says it",
+			"    " + factLine("architecture", orUnsaid(a.Architecture)),
+			"    " + factLine("kernel", orUnsaid(a.Kernel)),
+		}
+	}
+	age := roughly(now.Sub(m.ObservedAt)) + " ago"
+	if m.ObservedAt.IsZero() {
+		// Said with no time: its age is not known, and is never read as two thousand years.
+		age = "at a time it did not say (age unknown)"
+	}
+	out := []string{fmt.Sprintf("  what it is made of, as it said %s:", age)}
+	add := func(name, value string) { out = append(out, "    "+factLine(name, value)) }
+	unknown := map[string]string{}
+	for k, v := range m.Unknown {
+		unknown[k] = v
+	}
+	// said is the value, or the fact's own reason it is unknown; a fact neither said nor unknown is a
+	// node-engine that broke its own rule, and is said so rather than shown blank.
+	said := func(fact, value string) string {
+		if value != "" {
+			delete(unknown, fact)
+			return value
+		}
+		if why, ok := unknown[fact]; ok {
+			delete(unknown, fact)
+			return "unknown: " + why
+		}
+		return "unknown: the machine said neither it nor why"
+	}
+
+	add("architecture", orUnsaid(a.Architecture))
+	release := said("kernel release", m.KernelRelease)
+	if m.KernelRelease == "" {
+		release = "(release " + release + ")"
+	}
+	add("kernel", orUnsaid(a.Kernel)+" "+release)
+	dist := said("distribution", m.Distribution)
+	if m.Distribution != "" {
+		dist += " " + said("version", m.Version)
+	}
+	add("distribution", dist)
+	add("processor", said("processor", m.Processor))
+	add("cores", said("cores", countWords(m.Cores)))
+	add("threads", said("threads", countWords(m.Threads)))
+	add("memory", said("memory", bytesWords(m.MemoryBytes)))
+	add("virtual", said("virtual or physical", m.Virtual))
+	up := ""
+	if m.BootedAt != nil {
+		up = fmt.Sprintf("%s, since %s", roughly(now.Sub(*m.BootedAt)), m.BootedAt.Local().Format("2006-01-02 15:04"))
+	}
+	add("up", said("uptime", up))
+	add("timezone", said("timezone", m.Timezone))
+	if len(m.Filesystems) == 0 {
+		add("filesystems", said("filesystems", ""))
+	} else {
+		for i, f := range m.Filesystems {
+			name := ""
+			if i == 0 {
+				name = "filesystems"
+			}
+			add(name, fmt.Sprintf("%s  %s, %s free (%s %s)", f.Mount, bytesWords(f.SizeBytes),
+				bytesWords(f.FreeBytes), f.Type, f.Device))
+		}
+		if why, ok := unknown["filesystems"]; ok {
+			add("", "and some unknown: "+why)
+			delete(unknown, "filesystems")
+		}
+	}
+	add("public address", said("public address", ""))
+	// Any other fact a newer node-engine named unknown, so nothing it said is dropped.
+	rest := make([]string, 0, len(unknown))
+	for k := range unknown {
+		rest = append(rest, k)
+	}
+	sort.Strings(rest)
+	for _, k := range rest {
+		add(k, "unknown: "+unknown[k])
+	}
+	return out
+}
+
+func factLine(name, value string) string { return fmt.Sprintf("%-15s %s", name, value) }
+
+func orUnsaid(s string) string {
+	if strings.TrimSpace(s) == "" {
+		return "not said"
+	}
+	return s
+}
+
+// countWords is a number a machine said, or nothing when it said none (its unknown says why).
+func countWords(n int) string {
+	if n <= 0 {
+		return ""
+	}
+	return fmt.Sprint(n)
+}
+
+// bytesWords is an amount of bytes a person reads, in binary units; nothing when none was said.
+func bytesWords(n int64) string {
+	if n <= 0 {
+		return ""
+	}
+	units := []string{"B", "KiB", "MiB", "GiB", "TiB", "PiB"}
+	v, u := float64(n), 0
+	for v >= 1024 && u < len(units)-1 {
+		v /= 1024
+		u++
+	}
+	if u == 0 {
+		return fmt.Sprintf("%d B", n)
+	}
+	return fmt.Sprintf("%.1f %s", v, units[u])
+}

cmd/mesh-controller/machine_lines_test.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// Defends novox/hq issue 160: `node` shows every fact a machine said it is made of, in one block with its
+// age, the architecture and kernel among them; and a fact it could not determine as unknown with why.
+func TestNodeShowsWhatAMachineIsMadeOfWithItsAge(t *testing.T) {
+	now := time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC)
+	booted := now.Add(-75 * time.Hour)
+	a := inventory.MachineAccount{Architecture: "amd64", Kernel: "linux", Machine: &inventory.MadeOf{
+		ObservedAt: now.Add(-3 * time.Hour), KernelRelease: "7.2.8-arch1-2", Distribution: "Arch Linux",
+		Version: "rolling", Processor: "AMD Ryzen 9 8945HS", Cores: 8, Threads: 16, MemoryBytes: 32891494400,
+		Virtual: "physical", BootedAt: &booted, Timezone: "Europe/Brussels",
+		Filesystems: []inventory.MountedFilesystem{{Mount: "/", Device: "/dev/nvme0n1p3", Type: "ext4",
+			SizeBytes: 987978522624, FreeBytes: 31830110208}},
+		Unknown: map[string]string{"public address": "how it learns it is not decided yet"},
+	}}
+	got := strings.Join(machineLines(a, true, now), "\n")
+	t.Log("\n" + got)
+	for _, want := range []string{
+		"what it is made of, as it said 3h ago:",
+		"architecture    amd64",
+		"kernel          linux 7.2.8-arch1-2",
+		"distribution    Arch Linux rolling",
+		"processor       AMD Ryzen 9 8945HS",
+		"cores           8",
+		"threads         16",
+		"memory          30.6 GiB",
+		"virtual         physical",
+		"up              3d, since",
+		"timezone        Europe/Brussels",
+		"filesystems     /  920.1 GiB, 29.6 GiB free (ext4 /dev/nvme0n1p3)",
+		"public address  unknown: how it learns it is not decided yet",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("missing %q", want)
+		}
+	}
+}
+
+func TestAFactAMachineCouldNotReadIsShownUnknownNeverZero(t *testing.T) {
+	now := time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC)
+	a := inventory.MachineAccount{Architecture: "arm64", Kernel: "linux", Machine: &inventory.MadeOf{
+		ObservedAt: now, Unknown: map[string]string{"memory": "MemTotal not found in /proc/meminfo",
+			"cores": "/proc/cpuinfo says no core id", "kernel release": "permission denied"}}}
+	got := strings.Join(machineLines(a, true, now), "\n")
+	t.Log("\n" + got)
+	for _, want := range []string{
+		"memory          unknown: MemTotal not found in /proc/meminfo",
+		"cores           unknown: /proc/cpuinfo says no core id",
+		"kernel          linux (release unknown: permission denied)",
+		// Neither said nor named unknown: the node-engine broke its rule, and that is shown.
+		"timezone        unknown: the machine said neither it nor why",
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("missing %q", want)
+		}
+	}
+	if strings.Contains(got, " 0 ") || strings.Contains(got, "0 B") {
+		t.Error("a fact the machine could not read was shown as a zero")
+	}
+}
+
+func TestAnOlderNodeEngineStillShowsArchitectureAndKernel(t *testing.T) {
+	got := strings.Join(machineLines(inventory.MachineAccount{Architecture: "amd64", Kernel: "linux"}, true,
+		time.Now()), "\n")
+	if !strings.Contains(got, "older than the one that says it") || !strings.Contains(got, "architecture    amd64") ||
+		!strings.Contains(got, "kernel          linux") {
+		t.Errorf("%s", got)
+	}
+	if got := strings.Join(machineLines(inventory.MachineAccount{}, false, time.Now()), "\n"); !strings.Contains(got, "never said") {
+		t.Errorf("%s", got)
+	}
+}
+
+func TestAnAccountWithoutItsTimeSaysAgeUnknown(t *testing.T) {
+	got := machineLines(inventory.MachineAccount{Architecture: "amd64", Kernel: "linux",
+		Machine: &inventory.MadeOf{KernelRelease: "7.2"}}, true, time.Now())[0]
+	if !strings.Contains(got, "age unknown") || strings.Contains(got, "d ago") {
+		t.Errorf("%s", got)
+	}
+}

cmd/mesh-controller/nodes.go

@@ -875,6 +875,17 @@ func showNode(ctx context.Context, stored *stores, name string) error {
 		}
 	}
 
+	// What it is made of, in one block with its age (novox/hq issue 160): collected and shown in the same
+	// breath, so no fact the machine says is kept and read by nothing.
+	if account, had, err := inv.MachineOf(ctx, name); err != nil {
+		fmt.Printf("\n  what it is made of could NOT be read: %v\n", err)
+	} else {
+		fmt.Println()
+		for _, line := range machineLines(account, had, time.Now()) {
+			fmt.Println(line)
+		}
+	}
+
 	held, err := inv.Profile(ctx, name)
 	if err != nil {
 		return err

cmd/mesh-controller/plans_follow.go

@@ -30,8 +30,9 @@ var maxPlanWait = link.LongestAnswer - planWaitStart
 // answered that the call is still running, and `calls` with its id says what came of it.
 const planWaitStart = 5 * time.Second
 
-// planWaitEvery is how often a wait reads the plan again.
-const planWaitEvery = time.Second
+// planWaitEvery is how often a wait reads the plan again. A variable so a test reads it more often
+// (novox/hq issue 591).
+var planWaitEvery = time.Second
 
 // The things a wait waits until.
 var planWaitUntil = []string{"moved", "tier", "done"}

cmd/mesh-controller/plans_follow_test.go

@@ -138,6 +138,12 @@ func TestThePlansVerbComposesAWait(t *testing.T) {
 func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) {
 	open := aMesh(t)
 	ctx := t.Context()
+	// Read every 50ms rather than every second, so the wait answers as soon as the plan moves and the bound
+	// below passes in a fraction of a second (novox/hq issue 591); the move still waits 1.5s, long enough
+	// for the wait to have read the plan before it.
+	was := planWaitEvery
+	planWaitEvery = 50 * time.Millisecond
+	t.Cleanup(func() { planWaitEvery = was })
 	now := time.Now().UTC()
 	walk := inventory.Plan{ID: "plan-229", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c229c229",
 		Created: now, State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
@@ -169,11 +175,11 @@ func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) {
 	if got.Came != "tier" || got.Plan.Tier != 1 || got.Plan.State != inventory.PlanRolling {
 		t.Fatalf("the wait answered %+v", got)
 	}
-	if waited := time.Since(began); waited > 10*time.Second {
+	if waited := time.Since(began); waited > 3*time.Second {
 		t.Errorf("it answered after %s, not when the plan moved", waited)
 	}
 	out = printed(t, func() error {
-		return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "2"})
+		return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "300ms"})
 	})
 	if err := json.Unmarshal([]byte(out), &got); err != nil || got.Came != "bound" || got.Until != "moved" {
 		t.Fatalf("a plan that did not move answered %+v (%v)", got, err)

cmd/mesh-controller/queue.go

@@ -14,7 +14,6 @@ import (
 	"time"
 
 	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
 
 	"github.com/novox/mesh-controller/internal/broker"
 	"github.com/novox/mesh-controller/internal/inventory"
@@ -81,6 +80,30 @@ func listQueue(ctx context.Context, args []string, w io.Writer) error {
 	return nil
 }
 
+// keepingTheQueueInOrder runs the build queue's passes while this controller acts (novox/hq issue 558):
+// the one process that puts the queue in order, so no two passes pause and resume it over each other.
+// The build seat is read from the catalogue at most every half minute.
+func keepingTheQueueInOrder(ctx context.Context, open *stores, js *broker.JetStream) {
+	if js == nil {
+		return
+	}
+	seat, readAt := link.TheBuildMachine, time.Time{}
+	seatNow := func() string {
+		if time.Since(readAt) < watchEvery {
+			return seat
+		}
+		if entries, err := open.inventory.Catalogued(ctx); err == nil {
+			seat, readAt = buildSeatAmong(entries), time.Now()
+		} else {
+			fmt.Printf("the build seat is read as %s: the catalogue cannot be read: %v\n", seat, err)
+		}
+		return seat
+	}
+	link.KeepingInOrder(ctx, js, seatNow, link.Holding, func(format string, args ...any) {
+		fmt.Printf(format+"\n", args...)
+	})
+}
+
 // queueAnswer is the queue as data (novox/hq issue 455): what a caller reads to say how far back an ask
 // waits — "waiting in the build queue behind 2" — which the table alone could not tell it. Text is the
 // table, written from the same read: the verb answers it as its output and the rest as its answer, so the
@@ -100,7 +123,8 @@ type queueAnswer struct {
 type queuedAnswer struct {
 	ID    string `json:"id"`
 	State string `json:"state"`
-	// Kind is "check" for a pull request's merge check, "build" for anything else.
+	// Kind is "delivery" for a build a walk waits for, "check" for a pull request's check and any ask
+	// whose kind cannot be told (link.KindOf, novox/hq issue 558).
 	Kind       string `json:"kind"`
 	Repository string `json:"repository"`
 	Path       string `json:"path,omitempty"`
@@ -129,10 +153,7 @@ func queueAnswerOf(q link.Queue, now time.Time) queueAnswer {
 	out := queueAnswer{Seat: q.Seat, Waiting: len(waiting), InFlight: len(running), Dead: len(dead),
 		MaxDeliver: q.MaxDeliver, Asks: []queuedAnswer{}, Text: queueText(q, now)}
 	one := func(a link.QueuedAsk, position int) queuedAnswer {
-		kind := "build"
-		if a.Request.Check != nil {
-			kind = "check"
-		}
+		kind := link.KindOf(a.Request)
 		since := a.AskedAt
 		if a.State == link.AskInFlight && a.On != "" && !a.Started.IsZero() {
 			since = a.Started
@@ -189,19 +210,20 @@ func queueText(q link.Queue, now time.Time) string {
 			case a.Was != "":
 				where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
 			}
-			fmt.Fprintf(&b, "  %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq)
+			fmt.Fprintf(&b, "  %-26s %-8s %s — %s, %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), where,
+				ago(a.AskedAt), a.Seq)
 		}
 	}
 	if len(waiting) > 0 {
-		fmt.Fprintln(&b, "\nwaiting:")
+		fmt.Fprintln(&b, "\nwaiting, in the order they are handed out:")
 		for _, a := range waiting {
-			fmt.Fprintf(&b, "  %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
+			fmt.Fprintf(&b, "  %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
 		}
 	}
 	if len(dead) > 0 {
 		fmt.Fprintf(&b, "\ndead — handed out as often as the worker allows (%d) and never settled, still held:\n", q.MaxDeliver)
 		for _, a := range dead {
-			fmt.Fprintf(&b, "  %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
+			fmt.Fprintf(&b, "  %-26s %-8s %s — %s (seq %d)\n", a.ID, link.KindOf(a.Request), what(a), ago(a.AskedAt), a.Seq)
 		}
 	}
 	switch {
@@ -275,6 +297,15 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
 		}
 	}
 	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
+	// **Where it waits now** (novox/hq issue 558): moved into order since it was read, a waiting ask has a
+	// later sequence, and the worker handing out what was behind its old one is not it being taken.
+	if ask.State == link.AskWaiting {
+		if q, err := link.ReadQueue(ctx, js, seat); err == nil {
+			if now, ok := q.Find(ask.ID); ok && now.State == link.AskWaiting {
+				ask.Seq = now.Seq
+			}
+		}
+	}
 	switch ask.State {
 	case link.AskWaiting:
 		if taken, err := takenSince(js, worker, ask.Seq); err != nil {
@@ -296,14 +327,25 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str
 				"ends it there", ask.ID, started, ask.ID)
 		}
 	}
-	if err := js.Context().DeleteMsg(worker.Stream, ask.Seq); err != nil && !errors.Is(err, nats.ErrMsgNotFound) &&
-		!errors.Is(err, jetstream.ErrMsgNotFound) && !strings.Contains(err.Error(), "no message found") {
+	// Found by its id when it was moved into order since it was read (novox/hq issue 558).
+	removed, err := link.DeleteAsk(ctx, js, seat, ask)
+	if err != nil {
 		return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
 			"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
 	}
 	recordCancelled(ctx, open, ask)
-	return fmt.Sprintf("cancelled %s (%s, %s): deleted from the %s queue and recorded failed, %s — a plan "+
-		"that asked for it fails with that", ask.ID, ask.Repository, ask.State, seat, link.CancelledByHand), nil
+	return cancelSaid(ask, seat, removed), nil
+}
+
+// cancelSaid is what a cancel says: deleted from the queue only when it was.
+func cancelSaid(ask link.QueuedAsk, seat string, removed uint64) string {
+	where := fmt.Sprintf("deleted from the %s queue", seat)
+	if removed == 0 {
+		where = fmt.Sprintf("no longer in the %s queue to delete, and marked cancelled, so a holder taking it "+
+			"ends it", seat)
+	}
+	return fmt.Sprintf("cancelled %s (%s, %s): %s and recorded failed, %s — a plan "+
+		"that asked for it fails with that", ask.ID, ask.Repository, ask.State, where, link.CancelledByHand)
 }
 
 // holderLooks is how long a cancel waits for a holder that took the ask before the mark to say it

cmd/mesh-controller/queue_answer_test.go

@@ -24,7 +24,7 @@ func aQueueOfEachKind(now time.Time) link.Queue {
 			State: link.AskInFlight, On: "ace", Started: started,
 			Request: link.BuildRequest{ID: "build-running", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host"}}},
 		{Seq: 3, ID: "build-first", Repository: "ssh://forge/novox/mesh-catalog", Ref: "main", AskedAt: asked,
-			State: link.AskWaiting},
+			State: link.AskWaiting, Request: link.BuildRequest{ID: "build-first", Repository: "ssh://forge/novox/mesh-catalog"}},
 		{Seq: 4, ID: "build-second", Repository: "ssh://forge/novox/mesh-catalog", Path: "modules/gitea", Ref: aCommit,
 			AskedAt: now.Add(-time.Minute), State: link.AskWaiting,
 			Request: link.BuildRequest{ID: "build-second", Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog"}}},
@@ -51,7 +51,7 @@ func TestTheQueueVerbAnswersEachAskAsData(t *testing.T) {
 		t.Errorf("in flight: %+v", running)
 	}
 	first, second := byID["build-first"], byID["build-second"]
-	if first.Position != 1 || first.Kind != "build" || first.Commit != "" || first.Ref != "main" || first.Machine != "" ||
+	if first.Position != 1 || first.Kind != "delivery" || first.Commit != "" || first.Ref != "main" || first.Machine != "" ||
 		!first.Since.Equal(now.Add(-5*time.Minute)) {
 		t.Errorf("first waiting: %+v", first)
 	}
@@ -62,6 +62,13 @@ func TestTheQueueVerbAnswersEachAskAsData(t *testing.T) {
 	if dead := byID["build-dead"]; dead.Position != 0 || dead.State != link.AskDead {
 		t.Errorf("dead: %+v", dead)
 	}
+	// And the table says each ask's kind too (novox/hq issue 558).
+	for _, line := range []string{"build-running              check    ", "build-first                delivery ",
+		"build-second               check    ", "build-dead                 check    "} {
+		if !strings.Contains(got.Text, line) {
+			t.Errorf("the table does not say %q:\n%s", line, got.Text)
+		}
+	}
 	// In the table's order: in flight, then the waiting by position, then the dead.
 	var order []string
 	for _, a := range got.Asks {
@@ -135,3 +142,16 @@ func TestTheQueueNeverShowsARepositorysPassword(t *testing.T) {
 		}
 	}
 }
+
+// A cancel says it deleted the ask only when it did: moved into order and gone since, it says so (novox/hq
+// issue 558).
+func TestACancelSaysDeletedOnlyWhenItDeleted(t *testing.T) {
+	ask := link.QueuedAsk{ID: "build-1", Repository: "novox/x", State: link.AskWaiting, Seq: 4}
+	if said := cancelSaid(ask, link.TheBuildMachine, 9); !strings.Contains(said, "deleted from the") {
+		t.Errorf("removed, it says: %s", said)
+	}
+	if said := cancelSaid(ask, link.TheBuildMachine, 0); strings.Contains(said, "deleted from") ||
+		!strings.Contains(said, "no longer in the") {
+		t.Errorf("gone already, it says: %s", said)
+	}
+}

cmd/mesh-controller/replay201_test.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+	"github.com/novox/mesh-controller/internal/inventory"
+)
+
+// TestReplay201ThePlansSend replays novox/hq issue 201 (2026-10-02): two merges a minute apart made two plans; the
+// controller's plan rolled the newer controller onto the control node, and the node-engine's plan then sent that
+// machine the controller build before it, whose binary refused the seat row the newer one had widened. Here the
+// control node, anchor, was last sent the newer controller c2 while the mesh's record of the controller is the
+// build before it, c1, which passed its gate. The outcome asserted: a send that is no person's push — composed as a
+// plan's send composes it (keepingRecorded, what sendKeeps gives every send through sendToEach) — carries the
+// controller at c2, the build the machine was last sent, and not the older record; a person's push still carries
+// the record. Written with only what the controller had before its fix (mesh-controller #273) — the inventory's
+// build, gate, assignment and send records, keepingRecorded, planFor and declarationFor — so it is laid over the
+// commit before; the fix's own TestReplay201 reads what the fix added. Needs the store (MESH_TEST_POSTGRES), and
+// is named in testdata/store-tests.
+func TestReplay201ThePlansSend(t *testing.T) {
+	open := aMesh(t)
+	ctx := t.Context()
+	inv := open.inventory
+	const older, newer = "c1aaaaaaaaaa", "c2bbbbbbbbbb"
+	old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
+	record := func(commit string, asked time.Time) inventory.Build {
+		t.Helper()
+		manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"})
+		b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit,
+			Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked,
+			Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}}
+		if err := inv.RecordBuild(ctx, b); err != nil {
+			t.Fatal(err)
+		}
+		return b
+	}
+
+	// The older controller rolled out and passed its gate; the newer one was asked after it and registered.
+	before := record(older, old)
+	if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: older,
+		Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
+		t.Fatal(err)
+	}
+	record(newer, now)
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{
+		Repository: "novox/mesh-controller", BuiltFrom: newer, Head: newer, Asked: now}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil {
+		t.Fatal(err)
+	}
+	// The controller's plan sent anchor the newer controller.
+	if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": newer}); err != nil {
+		t.Fatal(err)
+	}
+	// And the mesh's record of the controller is the build before it — whatever made it so.
+	if err := inv.RestoreModule(ctx, before); err != nil {
+		t.Fatal(err)
+	}
+
+	// The controller each send to anchor carries: a plan's send keeps recorded builds (keepingRecorded, what
+	// sendKeeps gives every send but a person's push); a person's push composes the mesh's record.
+	controllerSent := func(sends string, keeping bool) string {
+		t.Helper()
+		c := ctx
+		if keeping {
+			c = keepingRecorded(ctx)
+		}
+		plan, settings, err := planFor(c, open, "anchor")
+		if err != nil {
+			t.Fatalf("%s to anchor did not compose: %v", sends, err)
+		}
+		declared, err := declarationFor(c, open, "anchor", plan, settings)
+		if err != nil {
+			t.Fatalf("%s to anchor did not compose: %v", sends, err)
+		}
+		return declared.Builds["mesh-controller"]
+	}
+
+	if got := controllerSent("a plan's send", true); got != newer {
+		t.Fatalf("a plan's send carried anchor the controller %s, older than the %s anchor was last sent: the "+
+			"older binary would read the row its successor widened", short(got), short(newer))
+	}
+	if got := controllerSent("a person's push", false); got != older {
+		t.Fatalf("a person's push carried the controller %s, not the mesh's record %s", short(got), short(older))
+	}
+}

cmd/mesh-controller/sayable_test.go

@@ -26,6 +26,14 @@ import (
 // shape fails the suite, naming its source. The keeper would say such a summary in words at run time;
 // this is where the producer is made to say it rightly in the first place.
 func TestMain(m *testing.M) {
+	// **A command line this suite runs ends when it is done** (novox/hq issue 591). Many tests run this test
+	// binary as a verb's command line, and a binary built with the race detector sleeps a second before it
+	// exits (GORACE's atexit_sleep_ms), which was more than the rest of each such test. The children still
+	// run under the race detector and still fail on a race they find; they only stop waiting at their end.
+	// Set before any child starts, and only where nobody chose GORACE.
+	if _, chosen := os.LookupEnv("GORACE"); !chosen {
+		_ = os.Setenv("GORACE", "atexit_sleep_ms=0")
+	}
 	// The process a mesh-cli test runs as a command line: it says the verb and the caller it was given, and
 	// ends (meshcli_test.go).
 	// The process a mesh-cli test runs as `secret accept`: it reads its value exactly as `secret accept` does

cmd/mesh-controller/watchdogs.go

@@ -739,6 +739,9 @@ func watchTheMesh(ctx context.Context, open *stores, server *link.Server, bus li
 	// and the answer chosen is performed on its warrant.
 	startAsking(watching, open, server, bus.Conn, keeper)
 	go forgettingOldHeals(watching, open.inventory)
+	// And the build queue's order (novox/hq issue 558): a check that has waited its while goes ahead of the
+	// deliveries asked after it even when nothing new is asked.
+	go keepingTheQueueInOrder(watching, open, server.JetStream())
 	fmt.Printf("watching the mesh: %d signal(s) every %s, %d probe(s) every %s; what is wrong is kept in %s "+
 		"and said as %s events\n", len(watchedRows()), watchEvery, len(runnableProbes()), doctorEvery,
 		broker.ConditionsBucket, conditions.Seat)

internal/catalogue/manifest.go

@@ -1693,6 +1693,8 @@ func ParseManifest(raw []byte) (Manifest, error) {
 	problems = append(problems, LogProblems(m)...)
 	// And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262).
 	problems = append(problems, SettingProblems(m)...)
+	// And which setting keys each mergeable file takes (setting_takes.go, novox/hq ADR 0326).
+	problems = append(problems, TakenSettingsProblems(m)...)
 	wellFormed := true
 	for _, c := range m.Claims {
 		if !name.MatchString(c.Name) {

internal/catalogue/setting_takes.go

@@ -0,0 +1,176 @@
+package catalogue
+
+import (
+	"fmt"
+	"strings"
+	"unicode"
+)
+
+// A mergeable file names the setting keys it takes (novox/hq ADR 0326, issue 168).
+//
+// Before this, every key of a module's layers merged into every one of its mergeable files: the
+// mesh's own words, a key meant for a `${setting:<key>}` in another file, a key for a second
+// configuration file. So a module could not own two configurable files, and every receiver had to
+// ignore what it did not know. ADR 0164 §11 decided that a setting names the files that read it; this
+// is that relation, stated from the file's side, so the operator's own values stay declared nowhere
+// (ADR 0262 §2).
+
+// TakesSettingsField is the key a mergeable file carries to name the top-level setting keys it takes.
+const TakesSettingsField = "takes-settings"
+
+// takenKeys is the keys a resource names in takes-settings, and whether it names any list at all. An
+// empty list is a list: that file takes no setting.
+func takenKeys(resource map[string]any) (map[string]bool, bool) {
+	raw, said := resource[TakesSettingsField]
+	if !said {
+		return nil, false
+	}
+	keys := map[string]bool{}
+	for _, k := range stringsOf(raw) {
+		keys[k] = true
+	}
+	return keys, true
+}
+
+// onlyTaken is the layers with every key the file does not name left out. The file's own defaults and
+// what the mesh reads itself are not touched here: settle already keeps both out of a file.
+func onlyTaken(layers []Layer, taken map[string]bool) []Layer {
+	kept := make([]Layer, 0, len(layers))
+	for _, layer := range layers {
+		values := map[string]any{}
+		for key, value := range layer.Values {
+			if taken[key] {
+				values[key] = value
+			}
+		}
+		kept = append(kept, Layer{From: layer.From, Values: values, Default: layer.Default})
+	}
+	return kept
+}
+
+// namesItsKeys says whether a module's mergeable files name the keys they take (ADR 0326 §2): true
+// only when it has one and every one carries takes-settings. A module with a silent mergeable file
+// keeps today's form (§3), where every key reaches every mergeable file.
+func namesItsKeys(m Manifest) bool {
+	merging := 0
+	for _, r := range m.Resources {
+		if how, _ := r["merge"].(string); how == "" {
+			continue
+		}
+		merging++
+		if _, said := r[TakesSettingsField]; !said {
+			return false
+		}
+	}
+	return merging > 0
+}
+
+// TakenSettingsProblems is every way a definition's takes-settings is wrong (ADR 0326 §2 and §4), in
+// its own words: on a file that does not merge, not a list of keys, a key named twice, one the mesh
+// reads itself, one the file keeps for itself, or one mergeable file naming its keys beside another
+// that names none.
+func TakenSettingsProblems(m Manifest) []string {
+	// Not yet here: refusing a module that declares a setting and owns a mergeable file naming no keys
+	// (ADR 0326 §3). betterbird is one, and the catalogue cannot name its keys before this controller
+	// runs, so that refusal follows once the catalogue does (novox/hq issue 574).
+	return takenSettingsProblems(m)
+}
+
+// takenSettingsProblems is every rule of ADR 0326 §2 and §4, judged where a definition is checked and
+// where a stored one is composed.
+func takenSettingsProblems(m Manifest) []string {
+	var problems []string
+	var naming, silent []string
+	for _, r := range m.Resources {
+		id := fmt.Sprint(r["id"])
+		how, _ := r["merge"].(string)
+		raw, said := r[TakesSettingsField]
+		if how == "" {
+			if said {
+				problems = append(problems, fmt.Sprintf("%s's %s says %s and does not merge: only a mergeable "+
+					"file takes settings by key (novox/hq ADR 0326)", m.Module, id, TakesSettingsField))
+			}
+			continue
+		}
+		if !said {
+			silent = append(silent, id)
+			continue
+		}
+		naming = append(naming, id)
+		list, isList := raw.([]any)
+		if !isList {
+			problems = append(problems, fmt.Sprintf("%s's %s: %s is a list of setting keys, and this is %T",
+				m.Module, id, TakesSettingsField, raw))
+			continue
+		}
+		protected := map[string]bool{}
+		for _, k := range stringsOf(r["protected"]) {
+			protected[k] = true
+		}
+		seen := map[string]bool{}
+		for _, item := range list {
+			key, isString := item.(string)
+			switch {
+			case !isString || !usableKey(key):
+				problems = append(problems, fmt.Sprintf("%s's %s: %v in %s is not a usable setting key",
+					m.Module, id, item, TakesSettingsField))
+			case seen[key]:
+				problems = append(problems, fmt.Sprintf("%s's %s names %q twice in %s",
+					m.Module, id, key, TakesSettingsField))
+			case meshWords[key]:
+				problems = append(problems, fmt.Sprintf("%s's %s names %q in %s, and the mesh reads %q itself "+
+					"(novox/hq ADR 0164 §14)", m.Module, id, key, TakesSettingsField, key))
+			case protected[key]:
+				problems = append(problems, fmt.Sprintf("%s's %s names %q in %s and keeps it as protected: a "+
+					"key is the file's or the setting's, never both", m.Module, id, key, TakesSettingsField))
+			}
+			if isString {
+				seen[key] = true
+			}
+		}
+	}
+	if len(naming) > 0 {
+		for _, id := range silent {
+			problems = append(problems, fmt.Sprintf("%s's %s merges and names no %s while %s does: once one "+
+				"mergeable file of a module names the keys it takes, every one does, an empty list for none "+
+				"(novox/hq ADR 0326)", m.Module, id, TakesSettingsField, strings.Join(naming, ", ")))
+		}
+	}
+	return problems
+}
+
+// usableKey is a key a file can be said to take: not empty, and no space, line end or control
+// character. Not the lower-case rule of a `${setting:<key>}`: a file's own keys are its software's
+// spelling, such as `timeZone`.
+func usableKey(key string) bool {
+	if key == "" {
+		return false
+	}
+	for _, r := range key {
+		if unicode.IsSpace(r) || unicode.IsControl(r) || !unicode.IsPrint(r) {
+			return false
+		}
+	}
+	return true
+}
+
+// takenSaid is what a stray key's message adds for a module whose files name their keys: which keys
+// each file takes, so the person sees where the key would have had to be named.
+func takenSaid(m Manifest) string {
+	if !namesItsKeys(m) {
+		return ""
+	}
+	var files []string
+	for _, r := range m.Resources {
+		if how, _ := r["merge"].(string); how == "" {
+			continue
+		}
+		keys := stringsOf(r[TakesSettingsField])
+		if len(keys) == 0 {
+			files = append(files, fmt.Sprintf("%v takes none", r["id"]))
+			continue
+		}
+		files = append(files, fmt.Sprintf("%v takes %s", r["id"], strings.Join(keys, ", ")))
+	}
+	return " (" + strings.Join(files, "; ") + ")"
+}

internal/catalogue/setting_takes_test.go

@@ -0,0 +1,196 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+// twoFiles is a module that owns two configurable files, which issue 168 said no module could:
+// every key of every layer reached both.
+func twoFiles(app, sidecar []any) Manifest {
+	appFile := map[string]any{"id": "app", "type": "file", "path": "/etc/app.json", "merge": MergeJSON,
+		"content": `{"server":{"port":8080},"search":{"safe":1}}`}
+	sideFile := map[string]any{"id": "sidecar", "type": "file", "path": "/etc/sidecar.json", "merge": MergeJSON,
+		"content": `{"url":"http://127.0.0.1:8080"}`}
+	if app != nil {
+		appFile[TakesSettingsField] = app
+	}
+	if sidecar != nil {
+		sideFile[TakesSettingsField] = sidecar
+	}
+	return Manifest{Module: "search", Resources: []map[string]any{
+		appFile, sideFile,
+		{"id": "env", "type": "file", "path": "/etc/search.env", "content": "SITE=${setting:site-name}\n"},
+	}}
+}
+
+func settledContent(t *testing.T, resource map[string]any, layers []Layer) map[string]any {
+	t.Helper()
+	out, err := ApplySettings(resource, layers)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, kept := out[TakesSettingsField]; kept {
+		t.Errorf("%v still carries %s; the host knows no such field", out["id"], TakesSettingsField)
+	}
+	var parsed map[string]any
+	if err := json.Unmarshal([]byte(out["content"].(string)), &parsed); err != nil {
+		t.Fatalf("%v is not JSON after settling: %v", out["id"], err)
+	}
+	return parsed
+}
+
+func TestEachMergeableFileTakesOnlyTheKeysItNames(t *testing.T) {
+	// novox/hq ADR 0326 §1, issue 168: two configurable files, each taking its own keys. A key for
+	// the other file, for a ${setting:…} elsewhere or for the mesh itself lands in neither.
+	m := twoFiles([]any{"server", "search"}, []any{"url"})
+	layers := []Layer{
+		{From: "the mesh", Values: map[string]any{
+			"server": map[string]any{"base_url": "https://search.example.tld/"},
+			"url":    "http://127.0.0.1:9090", "site-name": "Search",
+			"endpoints": map[string]any{"web": map[string]any{"reach": "public"}}}},
+		{From: "node-a", Values: map[string]any{"search": map[string]any{"safe": 0.0}}},
+	}
+
+	app := settledContent(t, m.Resources[0], layers)
+	for _, stray := range []string{"url", "site-name", "endpoints"} {
+		if _, in := app[stray]; in {
+			t.Errorf("app took %q, which it does not name: %v", stray, app)
+		}
+	}
+	server, _ := app["server"].(map[string]any)
+	if server["base_url"] != "https://search.example.tld/" || server["port"] != 8080.0 {
+		t.Errorf("app's server block is %v; the setting should merge into the file's own", server)
+	}
+	if search, _ := app["search"].(map[string]any); search["safe"] != 0.0 {
+		t.Errorf("app's search block is %v; the node's layer should win", search)
+	}
+
+	side := settledContent(t, m.Resources[1], layers)
+	if side["url"] != "http://127.0.0.1:9090" {
+		t.Errorf("sidecar's url is %v; it names url", side["url"])
+	}
+	if len(side) != 1 {
+		t.Errorf("sidecar took keys it does not name: %v", side)
+	}
+}
+
+func TestAnEmptyListTakesNoSetting(t *testing.T) {
+	// ADR 0164 §12: a file that names no setting takes none.
+	m := twoFiles([]any{"server"}, []any{})
+	side := settledContent(t, m.Resources[1], []Layer{{From: "the mesh", Values: map[string]any{
+		"server": map[string]any{"port": 1.0}, "url": "x"}}})
+	if len(side) != 1 || side["url"] != "http://127.0.0.1:8080" {
+		t.Errorf("a file taking none was changed: %v", side)
+	}
+}
+
+func TestAFileThatNamesNothingKeepsTodaysForm(t *testing.T) {
+	// ADR 0326 §3, ADR 0164 §15: a module none of whose files names keys is left as it was.
+	m := twoFiles(nil, nil)
+	app := settledContent(t, m.Resources[0], []Layer{{From: "the mesh", Values: map[string]any{"url": "x"}}})
+	if app["url"] != "x" {
+		t.Errorf("a silent file should still take every key until its module names them: %v", app)
+	}
+	if unused := UnusedSettings(m, []Layer{{From: "the mesh", Values: map[string]any{"anything": 1}}}); len(unused) != 0 {
+		t.Errorf("today's form has nothing stray, and said: %v", unused)
+	}
+}
+
+func TestAKeyNoFileNamesIsStray(t *testing.T) {
+	// ADR 0326 §5: once the files name their keys, a key none of them names — and nothing else
+	// takes — is stray, and is said with what each file takes.
+	m := twoFiles([]any{"server", "search"}, []any{"url"})
+	unused := UnusedSettings(m, []Layer{{From: "the mesh", Values: map[string]any{
+		"server": 1, "url": "x", "site-name": "Search", "ui": map[string]any{"theme": "dark"}}}})
+	if len(unused) != 1 || !strings.Contains(unused[0], `"ui"`) {
+		t.Fatalf("only ui reaches nothing; named: %v", unused)
+	}
+	if !strings.Contains(unused[0], "app takes server, search") || !strings.Contains(unused[0], "sidecar takes url") {
+		t.Errorf("the stray key's message does not say what each file takes: %s", unused[0])
+	}
+}
+
+func TestOneNamingFileMakesEveryMergeableFileName(t *testing.T) {
+	// ADR 0326 §2: a module with one naming and one silent mergeable file is refused, naming the
+	// silent one — the silent one would otherwise still take every key.
+	m := twoFiles([]any{"server"}, nil)
+	problems := TakenSettingsProblems(m)
+	if len(problems) != 1 || !strings.Contains(problems[0], "sidecar merges and names no takes-settings") {
+		t.Fatalf("problems: %v", problems)
+	}
+	if err := JudgeSettings(m, nil, false); err == nil || !strings.Contains(err.Error(), "sidecar") {
+		t.Errorf("composing it should refuse it too, and said: %v", err)
+	}
+	raw, err := json.Marshal(map[string]any{"module": m.Module, "version": "1", "resources": m.Resources})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ParseManifest(raw); err == nil || !strings.Contains(err.Error(), "sidecar merges and names no") {
+		t.Errorf("the module check does not say it: %v", err)
+	}
+}
+
+func TestTakesSettingsIsRefusedWhereItCannotMeanAnything(t *testing.T) {
+	// ADR 0326 §4.
+	cases := map[string]struct {
+		resource map[string]any
+		says     string
+	}{
+		"on a file that does not merge": {
+			map[string]any{"id": "plain", "type": "file", "path": "/x", "content": "x",
+				TakesSettingsField: []any{"a"}},
+			"does not merge"},
+		"not a list": {
+			map[string]any{"id": "conf", "type": "file", "path": "/x", "merge": MergeJSON, "content": "{}",
+				TakesSettingsField: "server"},
+			"is a list of setting keys"},
+		"a mesh word": {
+			map[string]any{"id": "conf", "type": "file", "path": "/x", "merge": MergeJSON, "content": "{}",
+				TakesSettingsField: []any{"endpoints"}},
+			"the mesh reads"},
+		"a protected key": {
+			map[string]any{"id": "conf", "type": "file", "path": "/x", "merge": MergeJSON, "content": "{}",
+				"protected": []any{"secret"}, TakesSettingsField: []any{"secret"}},
+			"keeps it as protected"},
+		"named twice": {
+			map[string]any{"id": "conf", "type": "file", "path": "/x", "merge": MergeJSON, "content": "{}",
+				TakesSettingsField: []any{"a", "a"}},
+			"twice"},
+		"not a key": {
+			map[string]any{"id": "conf", "type": "file", "path": "/x", "merge": MergeJSON, "content": "{}",
+				TakesSettingsField: []any{"two words"}},
+			"not a usable setting key"},
+	}
+	for name, c := range cases {
+		problems := TakenSettingsProblems(Manifest{Module: "m", Resources: []map[string]any{c.resource}})
+		if !containsText(problems, c.says) {
+			t.Errorf("%s: want a problem saying %q, got %v", name, c.says, problems)
+		}
+	}
+	// And a file that does not merge is refused where it is settled, not handed to the host.
+	plain := map[string]any{"id": "plain", "type": "file", "path": "/x", "content": "x",
+		TakesSettingsField: []any{"a"}}
+	if _, err := ApplySettings(plain, nil); err == nil {
+		t.Error("a file that does not merge and names keys was settled")
+	}
+}
+
+func TestASoftwareSpelledKeyIsUsable(t *testing.T) {
+	// A file's keys are its software's: nodered's timeZone is not lower case and is still a key.
+	m := Manifest{Module: "flows", Resources: []map[string]any{{"id": "settings", "type": "file",
+		"path": "/x", "merge": MergeJSON, "content": "{}", TakesSettingsField: []any{"timeZone"}}}}
+	if problems := TakenSettingsProblems(m); len(problems) != 0 {
+		t.Errorf("timeZone was refused: %v", problems)
+	}
+}
+
+func containsText(list []string, text string) bool {
+	for _, s := range list {
+		if strings.Contains(s, text) {
+			return true
+		}
+	}
+	return false
+}

internal/catalogue/settings.go

@@ -46,11 +46,24 @@ type Layer struct {
 // rather than by repeating everything the rest already say.
 func ApplySettings(resource map[string]any, layers []Layer) (map[string]any, error) {
 	how, _ := resource["merge"].(string)
+	taken, names := takenKeys(resource)
 	if how == "" {
+		if names {
+			// Refused by the module check too (TakenSettingsProblems); said here as well, for a
+			// definition stored by an older binary, rather than handed to the host, which knows no
+			// such field.
+			return nil, fmt.Errorf("%v says %s and does not merge: only a mergeable file takes "+
+				"settings by key (novox/hq ADR 0326)", resource["id"], TakesSettingsField)
+		}
 		// Not a mergeable file. Settings for it are ignored rather than silently doing nothing
 		// somewhere else — see UnusedSettings, which is what says so.
 		return resource, nil
 	}
+	if names {
+		// **The file takes the keys it names, and no other** (novox/hq ADR 0326, issue 168): a key
+		// for another file, for a `${setting:<key>}` or for the mesh itself does not land here.
+		layers = onlyTaken(layers, taken)
+	}
 	if how != MergeJSON {
 		return nil, fmt.Errorf(
 			"%v says it merges as %q, and this control plane knows how to merge %q",
@@ -91,6 +104,7 @@ func ApplySettings(resource map[string]any, layers []Layer) (map[string]any, err
 	delete(out, "merge")
 	delete(out, "protected")
 	delete(out, TrustedField)
+	delete(out, TakesSettingsField)
 	return out, nil
 }
 
@@ -324,8 +338,11 @@ func sortedKeysAny(m map[string]any) []string {
 // (novox/hq 04-ISSUES/173); and the mesh's own words — `expose`, `ports`, `reach`, `endpoints` —
 // are read by the mesh. A key none of those takes is stray, and is said so rather than dropped.
 func UnusedSettings(m Manifest, layers []Layer) []string {
+	named := namesItsKeys(m)
 	for _, r := range m.Resources {
-		if how, _ := r["merge"].(string); how != "" {
+		if how, _ := r["merge"].(string); how != "" && !named {
+			// Today's form (novox/hq ADR 0326 §3, ADR 0164 §15): a mergeable file that names no
+			// keys takes every key, so nothing is stray.
 			return nil
 		}
 	}
@@ -337,6 +354,13 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
 	}
 
 	lands := settingKeysUsedBy(m)
+	// A mergeable file takes the keys it names (novox/hq ADR 0326 §5).
+	for _, r := range m.Resources {
+		taken, _ := takenKeys(r)
+		for key := range taken {
+			lands[key] = true
+		}
+	}
 	// A key the module gives a default reaches what asks for it (novox/hq ADR 0262): setting it
 	// overrides the default, which is how a running module gains a setting without a gap between.
 	for key := range m.Settings {
@@ -401,8 +425,8 @@ func UnusedSettings(m Manifest, layers []Layer) []string {
 			}
 			unused = append(unused, fmt.Sprintf(
 				"%s sets %q, and %s has no file that merges it, asks for no ${setting:%s}, and "+
-					"declares no %q in what it contributes or serves",
-				layer.From, key, m.Module, key, key))
+					"declares no %q in what it contributes or serves%s",
+				layer.From, key, m.Module, key, key, takenSaid(m)))
 		}
 	}
 	sort.Strings(unused)
@@ -512,6 +536,11 @@ func JudgeSettings(m Manifest, layers []Layer, adopted bool) error {
 	if err := settingsHoldOneLine(m.Module, layers); err != nil {
 		return err
 	}
+	// Which keys each mergeable file takes is the definition's to say right (novox/hq ADR 0326);
+	// judged here too for a definition stored by an older binary.
+	if problems := takenSettingsProblems(m); len(problems) > 0 {
+		return fmt.Errorf("%s", problems[0])
+	}
 	// With no layers too: a definition may ask for a setting nobody made — an access placed by
 	// nobody, a file's ${setting:…} nothing sets — and that is the same statement, missing.
 	if _, err := GivenPorts(m, layers); err != nil {

internal/identity/fortest.go

@@ -29,14 +29,13 @@ func ForTest(t *testing.T) *Identity {
 	if len(name) > 60 {
 		name = name[:60]
 	}
-	conn, err := pgx.Connect(t.Context(), admin)
+	// Created already migrated, a copy of a template migrated once (novox/hq issue 591); the Migrate
+	// below then finds nothing pending, which it does only if the copy's record matches every migration.
+	migrations, err := Migrations()
 	if err != nil {
-		t.Fatalf("cannot reach the test PostgreSQL: %v", err)
+		t.Fatal(err)
 	}
-	if _, err := conn.Exec(t.Context(), "create database "+name); err != nil {
-		t.Fatalf("cannot create %s: %v", name, err)
-	}
-	conn.Close(t.Context())
+	store.CreateMigratedForTest(t, admin, name, migrations)
 
 	cut := strings.LastIndex(admin, "/")
 	t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
@@ -57,10 +56,6 @@ func ForTest(t *testing.T) *Identity {
 	if err := ident.Ready(t.Context(), 20*time.Second); err != nil {
 		t.Fatal(err)
 	}
-	migrations, err := Migrations()
-	if err != nil {
-		t.Fatal(err)
-	}
 	if _, err := ident.store.Migrate(t.Context(), migrations); err != nil {
 		t.Fatal(err)
 	}

internal/inventory/bususers.go

@@ -6,6 +6,7 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"sync/atomic"
 
 	"github.com/jackc/pgx/v5"
 	"golang.org/x/crypto/bcrypt"
@@ -23,6 +24,25 @@ import (
 // access change would blank every module's password if the mesh kept nothing (design 25 §4, and the
 // migration beside this).
 
+// busPasswordCostForTest, when set, is the bcrypt cost a bus password is hashed at instead of the
+// library's default. Only ForTest sets it, to bcrypt's least (novox/hq issue 591): at the default
+// cost one hash takes most of a second under the race detector, and a hash at the least cost is the
+// same kind of hash, checked by the same comparison and by the bus the same way.
+var busPasswordCostForTest atomic.Int32
+
+// busPasswordCost is the bcrypt cost a bus password is hashed at: the library's default, but in a test.
+func busPasswordCost() int {
+	if c := busPasswordCostForTest.Load(); c != 0 {
+		return int(c)
+	}
+	return bcrypt.DefaultCost
+}
+
+// hashBusPassword is the hash a bus password is kept as, minted, recorded or seeded alike.
+func hashBusPassword(password string) ([]byte, error) {
+	return bcrypt.GenerateFromPassword([]byte(password), busPasswordCost())
+}
+
 // BusUser is one user of the bus, as the mesh records it.
 type BusUser struct {
 	Username string
@@ -67,7 +87,7 @@ func (i *Inventory) RecordBusPassword(ctx context.Context, u BusUser, password s
 	if password == "" {
 		return errors.New("a bus user needs a password")
 	}
-	hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
+	hash, err := hashBusPassword(password)
 	if err != nil {
 		return fmt.Errorf("cannot hash a bus password: %w", err)
 	}
@@ -101,7 +121,7 @@ func (i *Inventory) MintBusPassword(ctx context.Context, u BusUser) (string, err
 	// The cost the server will pay on every connection. Left at the library's default rather than
 	// raised: a node reconnecting after a network blip pays it, and the mesh's own links reconnect
 	// far more often than a person logs in anywhere.
-	hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
+	hash, err := hashBusPassword(password)
 	if err != nil {
 		return "", fmt.Errorf("cannot hash a bus password: %w", err)
 	}
@@ -183,7 +203,7 @@ func (i *Inventory) SeedBusUser(ctx context.Context, u BusUser, password string)
 	if u.Username == "" || u.Kind == "" || password == "" {
 		return errors.New("a bus user needs a username, a kind and the credential it is using")
 	}
-	hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
+	hash, err := hashBusPassword(password)
 	if err != nil {
 		return fmt.Errorf("cannot hash a bus password: %w", err)
 	}

internal/inventory/bususers_test.go

@@ -158,3 +158,17 @@ func TestACredentialTheMeshDidNotMintIsRecordedOnceAndNotOverwritten(t *testing.
 		t.Fatal("a restart put the bootstrap credential back over a rotated one")
 	}
 }
+
+// Outside a test a bus password is hashed at bcrypt's default cost: only ForTest lowers it (novox/hq issue 591).
+func TestABusPasswordIsHashedAtTheDefaultCostOutsideATest(t *testing.T) {
+	was := busPasswordCostForTest.Load()
+	t.Cleanup(func() { busPasswordCostForTest.Store(was) })
+	busPasswordCostForTest.Store(0)
+	hash, err := hashBusPassword("a password")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got, err := bcrypt.Cost(hash); err != nil || got != bcrypt.DefaultCost {
+		t.Fatalf("a bus password is hashed at cost %d (%v), not bcrypt's default %d", got, err, bcrypt.DefaultCost)
+	}
+}

internal/inventory/fortest.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/jackc/pgx/v5"
+	"golang.org/x/crypto/bcrypt"
 
 	"github.com/novox/mesh-controller/internal/store"
 )
@@ -30,14 +31,14 @@ func ForTest(t *testing.T) *Inventory {
 		name = name[:60]
 	}
 
-	conn, err := pgx.Connect(t.Context(), admin)
+	// Created already migrated, a copy of a template migrated once (novox/hq issue 591); the Migrate
+	// below then finds nothing pending, which it does only if the copy's record matches every migration.
+	migrations, err := Migrations()
 	if err != nil {
-		t.Fatalf("cannot reach the test PostgreSQL: %v", err)
+		t.Fatal(err)
 	}
-	if _, err := conn.Exec(t.Context(), "create database "+name); err != nil {
-		t.Fatalf("cannot create %s: %v", name, err)
-	}
-	conn.Close(t.Context())
+	store.CreateMigratedForTest(t, admin, name, migrations)
+	busPasswordCostForTest.Store(int32(bcrypt.MinCost))
 
 	cut := strings.LastIndex(admin, "/")
 	t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
@@ -59,10 +60,6 @@ func ForTest(t *testing.T) *Inventory {
 		t.Fatal(err)
 	}
 
-	migrations, err := Migrations()
-	if err != nil {
-		t.Fatal(err)
-	}
 	if _, err := inv.store.Migrate(t.Context(), migrations); err != nil {
 		t.Fatal(err)
 	}

internal/inventory/machine.go

@@ -0,0 +1,74 @@
+package inventory
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+)
+
+// MachineAccount is what a machine said it is made of (novox/hq issue 160), from the profile it last
+// reported: the architecture and kernel family the profile always carried, and the facts a node-engine that
+// knows how says beside them. Read by `node`, so what a machine says of itself is visible in one place —
+// collected and shown in the same breath, not collected and read by nothing (issue 160's pattern).
+type MachineAccount struct {
+	Architecture string `json:"architecture"`
+	Kernel       string `json:"kernel"`
+	// Machine is nil when the node-engine that reported is older than the one that says it.
+	Machine *MadeOf `json:"machine"`
+}
+
+// MadeOf is the node-engine's account of what it is made of, as it put it on the wire. A fact it could
+// not determine is absent here and named in Unknown with why — never a zero.
+type MadeOf struct {
+	ObservedAt    time.Time           `json:"observed_at"`
+	KernelRelease string              `json:"kernel_release"`
+	Distribution  string              `json:"distribution"`
+	Version       string              `json:"version"`
+	Processor     string              `json:"processor"`
+	Cores         int                 `json:"cores"`
+	Threads       int                 `json:"threads"`
+	MemoryBytes   int64               `json:"memory_bytes"`
+	Virtual       string              `json:"virtual"`
+	BootedAt      *time.Time          `json:"booted_at"`
+	Timezone      string              `json:"timezone"`
+	Filesystems   []MountedFilesystem `json:"filesystems"`
+	Unknown       map[string]string   `json:"unknown"`
+}
+
+// MountedFilesystem is one local filesystem a machine has mounted.
+type MountedFilesystem struct {
+	Mount     string `json:"mount"`
+	Device    string `json:"device"`
+	Type      string `json:"type"`
+	SizeBytes int64  `json:"size_bytes"`
+	FreeBytes int64  `json:"free_bytes"`
+}
+
+// MachineOf is what a node last said it is made of. had is false for a node that has never reported a
+// profile at all, which is a different answer from one whose node-engine does not say the facts yet.
+func (i *Inventory) MachineOf(ctx context.Context, nodeName string) (account MachineAccount, had bool, err error) {
+	var raw []byte
+	err = i.store.Pool().QueryRow(ctx, `select profile from node where name = $1`, nodeName).Scan(&raw)
+	if errors.Is(err, pgx.ErrNoRows) {
+		return MachineAccount{}, false, fmt.Errorf("%w: %s", ErrNoSuchNode, nodeName)
+	}
+	if err != nil {
+		return MachineAccount{}, false, err
+	}
+	return machineFrom(raw)
+}
+
+func machineFrom(raw []byte) (MachineAccount, bool, error) {
+	if len(raw) == 0 {
+		return MachineAccount{}, false, nil
+	}
+	var account MachineAccount
+	if err := json.Unmarshal(raw, &account); err != nil {
+		return MachineAccount{}, false, err
+	}
+	return account, true, nil
+}

internal/inventory/machine_test.go

@@ -0,0 +1,42 @@
+package inventory
+
+import (
+	"testing"
+)
+
+// The node-engine's words for a machine's facts (novox/hq issue 160), as mesh-host's profile.Machine puts
+// them on the wire. A rename on either side fails here rather than as a blank on a real machine.
+func TestAMachinesAccountIsReadInTheNodeEnginesWords(t *testing.T) {
+	raw := []byte(`{"architecture":"amd64","kernel":"linux","capabilities":[],
+		"machine":{"observed_at":"2026-10-12T15:00:00Z","kernel_release":"7.2.8-arch1-2",
+		"distribution":"Arch Linux","version":"rolling","processor":"AMD Ryzen 9","cores":8,"threads":16,
+		"memory_bytes":32891494400,"virtual":"physical","booted_at":"2026-10-09T08:50:34Z",
+		"timezone":"Europe/Brussels",
+		"filesystems":[{"mount":"/","device":"/dev/nvme0n1p3","type":"ext4","size_bytes":100,"free_bytes":40}],
+		"unknown":{"public address":"not decided"}}}`)
+	a, had, err := machineFrom(raw)
+	if err != nil || !had {
+		t.Fatalf("had %v, err %v", had, err)
+	}
+	m := a.Machine
+	if a.Architecture != "amd64" || a.Kernel != "linux" || m == nil {
+		t.Fatalf("%+v", a)
+	}
+	if m.KernelRelease != "7.2.8-arch1-2" || m.Distribution != "Arch Linux" || m.Version != "rolling" ||
+		m.Processor != "AMD Ryzen 9" || m.Cores != 8 || m.Threads != 16 || m.MemoryBytes != 32891494400 ||
+		m.Virtual != "physical" || m.BootedAt == nil || m.Timezone != "Europe/Brussels" ||
+		len(m.Filesystems) != 1 || m.Filesystems[0].FreeBytes != 40 || m.Unknown["public address"] == "" ||
+		m.ObservedAt.IsZero() {
+		t.Errorf("%+v", m)
+	}
+}
+
+func TestAProfileWithoutTheFactsIsAnOlderNodeEngineNotNothing(t *testing.T) {
+	a, had, err := machineFrom([]byte(`{"architecture":"amd64","kernel":"linux","capabilities":[]}`))
+	if err != nil || !had || a.Machine != nil || a.Architecture != "amd64" {
+		t.Errorf("%+v had %v err %v", a, had, err)
+	}
+	if _, had, _ := machineFrom(nil); had {
+		t.Error("a node that never reported read as having reported")
+	}
+}

internal/inventory/nodes.go

@@ -1064,9 +1064,14 @@ type Reported struct {
 // SentDeclaration is what one send carried to a machine, as a gate keeps it: the declaration's digest and
 // its sequence (novox/hq issue 352). A report about this declaration, or about one sequenced after it, is a
 // report on what the gate sent — whatever the machine was sent since.
+//
+// At is when the controller sent it to that machine: the node's own send time, which is where that machine's
+// `apply` duration starts. A walk measures its rest from it, not from its own record of the send, made only after
+// every machine of the rest was sent (novox/hq issue 582). Zero for a send kept before it was recorded.
 type SentDeclaration struct {
-	Digest   string `json:"digest"`
-	Sequence int64  `json:"sequence,omitempty"`
+	Digest   string    `json:"digest"`
+	Sequence int64     `json:"sequence,omitempty"`
+	At       time.Time `json:"at,omitzero"`
 }
 
 // ReportsOn says a report is about this send: the declaration itself; one the same machine was sequenced
@@ -1085,7 +1090,8 @@ func (s SentDeclaration) ReportsOn(r Reported) bool {
 func (i *Inventory) SentTo(ctx context.Context, name string) (SentDeclaration, bool, error) {
 	var digest *string
 	var seq *int64
-	err := i.store.Pool().QueryRow(ctx, `select sent, sequence from node where name = $1`, name).Scan(&digest, &seq)
+	var at *time.Time
+	err := i.store.Pool().QueryRow(ctx, `select sent, sequence, sent_at from node where name = $1`, name).Scan(&digest, &seq, &at)
 	if errors.Is(err, pgx.ErrNoRows) {
 		return SentDeclaration{}, false, fmt.Errorf("%w: %s", ErrNoSuchNode, name)
 	}
@@ -1096,6 +1102,9 @@ func (i *Inventory) SentTo(ctx context.Context, name string) (SentDeclaration, b
 	if seq != nil {
 		s.Sequence = *seq
 	}
+	if at != nil {
+		s.At = at.UTC()
+	}
 	return s, true, nil
 }
 

internal/inventory/walkphases.go

@@ -2,13 +2,10 @@ package inventory
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"sort"
 	"strings"
 	"time"
-
-	"github.com/jackc/pgx/v5"
 )
 
 // Where a walk's time went (novox/hq ADR 0282 decision 6, issue 382): from its merge to every machine of its
@@ -59,11 +56,14 @@ type WalkPhases struct {
 	TotalMS int64  `json:"total_ms,omitempty"`
 	Total   string `json:"total,omitempty"`
 	// UnknownMS is the time within the walk no phase could be measured over.
-	UnknownMS int64        `json:"unknown_ms,omitempty"`
-	Phases    []WalkPhase  `json:"phases"`
-	Silent    []string     `json:"silent,omitempty"`
-	Said      string       `json:"said"`
-	Merges    []MergeStart `json:"merges,omitempty"`
+	UnknownMS int64       `json:"unknown_ms,omitempty"`
+	Phases    []WalkPhase `json:"phases"`
+	Silent    []string    `json:"silent,omitempty"`
+	// Rest is each machine of the rest's own report of the send that carried the walk's build there (novox/hq ADR
+	// 0282 decision 6, issue 582): when it was sent, when it said, and what.
+	Rest   []RestReport `json:"rest,omitempty"`
+	Said   string       `json:"said"`
+	Merges []MergeStart `json:"merges,omitempty"`
 }
 
 // MergeStart is one merge the walk answers and when it was made: where that merge's delivery time starts.
@@ -92,8 +92,34 @@ type AppliedReport struct {
 	Outcome string
 }
 
-// AppliedLookup answers a machine's first report after a send to it; false when it has not reported.
-type AppliedLookup func(node string, sent time.Time) (AppliedReport, bool)
+// AppliedLookup answers a machine's report of the send the walk made to it: send is what the walk kept of it, and
+// walkSent the walk's own record of its send to the rest; false when it has not reported.
+type AppliedLookup func(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool)
+
+// RestReport is one machine of the rest and its own report of a send of the walk (novox/hq issue 582).
+type RestReport struct {
+	Node    string   `json:"node"`
+	Modules []string `json:"modules"`
+	// Sent is when that machine was sent, by its own send time where the walk kept it; else the walk's record of
+	// its send to the rest, and SentSaid says so.
+	Sent     time.Time  `json:"sent"`
+	SentSaid string     `json:"sent_said,omitempty"`
+	Sequence int64      `json:"sequence,omitempty"`
+	Reported *time.Time `json:"reported,omitempty"`
+	Took     string     `json:"took,omitempty"`
+	Outcome  string     `json:"outcome,omitempty"`
+	// State is applied, failed, waiting (not yet heard, within ApplySilentAfter) or silent (not heard within it:
+	// listed, never moving the walk's end).
+	State string `json:"state"`
+}
+
+// The states of a machine of the rest.
+const (
+	RestApplied = "applied"
+	RestFailed  = "failed"
+	RestWaiting = "waiting"
+	RestSilent  = "silent"
+)
 
 // point is one moment of the walk, ending the phase named.
 type point struct {
@@ -307,8 +333,8 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
 				}
 			} else {
 				rest = latest(rest, s.SentAt)
-				for node := range s.Rest {
-					lastSends = append(lastSends, restSend{module: m, node: node, at: *s.SentAt})
+				for node, what := range s.Rest {
+					lastSends = append(lastSends, restSend{module: m, node: node, walkSent: *s.SentAt, what: what})
 				}
 			}
 		}
@@ -347,22 +373,73 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
 	if applied == nil {
 		return append(out, point{phase: PhaseApply, tier: -1, said: "the rest's reports were not read"})
 	}
-	var end *time.Time
-	var waiting, failed []string
+	// Each machine of the rest, by its own report of the send that carried the build there (novox/hq issue 582):
+	// never a report of a send made after it, which another walk's send may be.
+	reports := map[string]*RestReport{}
+	var order []string
 	for _, s := range lastSends {
-		r, ok := applied(s.node, s.at)
-		switch {
-		case !ok && now.Sub(s.at) > ApplySilentAfter, ok && r.At.Sub(s.at) > ApplySilentAfter:
-			w.Silent = appendOnce(w.Silent, s.node)
-		case !ok:
-			waiting = appendOnce(waiting, s.node)
-		case r.Outcome != OutcomeApplied:
-			failed = appendOnce(failed, s.node+" ("+r.Outcome+")")
-		default:
-			at := r.At
-			end = latest(end, &at)
+		at, said := s.walkSent, ""
+		if !s.what.At.IsZero() {
+			at = s.what.At
+		} else {
+			said = "the walk's record of its send to the rest: this machine's own send time was not kept"
+		}
+		key := s.node + "\x00" + s.what.Digest + "\x00" + at.Format(time.RFC3339Nano)
+		if r := reports[key]; r != nil {
+			r.Modules = appendOnce(r.Modules, s.module)
+			continue
+		}
+		rr := &RestReport{Node: s.node, Modules: []string{s.module}, Sent: at, SentSaid: said, Sequence: s.what.Sequence}
+		if r, ok := applied(s.node, s.what, s.walkSent); ok {
+			got := r.At
+			rr.Reported, rr.Outcome, rr.Took = &got, r.Outcome, words(r.At.Sub(at))
+		}
+		reports[key] = rr
+		order = append(order, key)
+	}
+	// A machine sent twice by the walk (one tier's rest, then the next's) that said nothing of the first send has
+	// said it of the second, which carries the first's builds too: both are this walk's.
+	for _, k := range order {
+		rr := reports[k]
+		if rr.Reported != nil {
+			continue
+		}
+		for _, l := range order {
+			later := reports[l]
+			if later.Node == rr.Node && later.Sent.After(rr.Sent) && later.Reported != nil &&
+				(rr.Reported == nil || later.Reported.Before(*rr.Reported)) {
+				got := *later.Reported
+				rr.Reported, rr.Outcome, rr.Took = &got, later.Outcome, words(got.Sub(rr.Sent))
+			}
 		}
 	}
+	var end *time.Time
+	var waiting, failed []string
+	for _, k := range order {
+		rr := reports[k]
+		switch {
+		case rr.Reported == nil && now.Sub(rr.Sent) > ApplySilentAfter, rr.Reported != nil && rr.Reported.Sub(rr.Sent) > ApplySilentAfter:
+			rr.State = RestSilent
+			w.Silent = appendOnce(w.Silent, rr.Node)
+		case rr.Reported == nil:
+			rr.State = RestWaiting
+			waiting = appendOnce(waiting, rr.Node)
+		case rr.Outcome != OutcomeApplied:
+			rr.State = RestFailed
+			failed = appendOnce(failed, rr.Node+" ("+rr.Outcome+")")
+		default:
+			rr.State = RestApplied
+			end = latest(end, rr.Reported)
+		}
+		sort.Strings(rr.Modules)
+		w.Rest = append(w.Rest, *rr)
+	}
+	sort.SliceStable(w.Rest, func(a, b int) bool {
+		if w.Rest[a].Node != w.Rest[b].Node {
+			return w.Rest[a].Node < w.Rest[b].Node
+		}
+		return w.Rest[a].Sent.Before(w.Rest[b].Sent)
+	})
 	switch {
 	case len(failed) > 0:
 		return append(out, point{phase: PhaseApply, tier: -1, said: "not applied on " + strings.Join(failed, ", ")})
@@ -393,7 +470,8 @@ func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []poin
 
 type restSend struct {
 	module, node string
-	at           time.Time
+	walkSent     time.Time
+	what         SentDeclaration
 }
 
 func askedAnyOf(p Plan, tier []string) bool {
@@ -451,28 +529,79 @@ func words(d time.Duration) string {
 	return d.Round(100 * time.Millisecond).String()
 }
 
-// FirstAppliedAfter is a machine's first report of a send made at or after a walk's send to it, within
-// ApplySilentAfter of it — the controller's `apply` durations, which measure every send to its first report (to-be
-// 45 Phase 0). A declaration sent later carries the build too, so its report counts; one sent past the bound is a
-// machine that slept, listed as silent and never moving the walk's end (ADR 0282 decision 1).
-func (i *Inventory) FirstAppliedAfter(ctx context.Context, node string, sent time.Time) (AppliedReport, bool, error) {
-	var started time.Time
-	var ms int64
-	var outcome string
-	err := i.store.Pool().QueryRow(ctx,
-		`select started, took_ms, detail from duration
+// FirstAppliedAfter is a machine's report of the send a walk made to it: the controller's `apply` durations, which
+// measure every send to its first report (to-be 45 Phase 0), each row starting at the machine's own send time and
+// naming the declaration it was about. send is what the walk kept of that send; walkSent its own record of the
+// send to the rest, made after every machine of the rest was sent (novox/hq issue 582). FirstReportOf chooses.
+func (i *Inventory) FirstAppliedAfter(ctx context.Context, node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool, error) {
+	from, to := applyRowsBetween(send, walkSent)
+	rows, err := i.store.Pool().Query(ctx,
+		`select ref, started, took_ms, detail from duration
 		  where kind = $1 and subject = $2 and started >= $3 and started < $4
-		  order by started limit 1`,
-		DurationApply, node, sent.Add(-appliedSlack), sent.Add(ApplySilentAfter)).Scan(&started, &ms, &outcome)
-	if errors.Is(err, pgx.ErrNoRows) {
-		return AppliedReport{}, false, nil
-	}
+		  order by started`,
+		DurationApply, node, from, to)
 	if err != nil {
 		return AppliedReport{}, false, err
 	}
-	return AppliedReport{At: started.Add(time.Duration(ms) * time.Millisecond).UTC(), Outcome: outcome}, true, nil
+	defer rows.Close()
+	var ds []Duration
+	for rows.Next() {
+		d := Duration{Kind: DurationApply, Subject: node, Node: node}
+		var ms int64
+		if err := rows.Scan(&d.Ref, &d.Started, &ms, &d.Detail); err != nil {
+			return AppliedReport{}, false, err
+		}
+		d.Took = time.Duration(ms) * time.Millisecond
+		ds = append(ds, d)
+	}
+	if err := rows.Err(); err != nil {
+		return AppliedReport{}, false, err
+	}
+	r, ok := FirstReportOf(ds, send, walkSent)
+	return r, ok, nil
 }
 
-// appliedSlack is how much earlier than a walk's record of its send the machine's own record of it may be:
-// the send is recorded on the machine first, then on the walk.
+// applyRowsBetween is the span of a machine's `apply` durations that may hold its report of a walk's send.
+func applyRowsBetween(send SentDeclaration, walkSent time.Time) (time.Time, time.Time) {
+	switch {
+	case !send.At.IsZero():
+		return send.At.Add(-time.Millisecond), send.At.Add(time.Millisecond)
+	case send.Digest != "":
+		return walkSent.Add(-restSendBound), walkSent.Add(ApplySilentAfter)
+	}
+	return walkSent.Add(-appliedSlack), walkSent.Add(ApplySilentAfter)
+}
+
+// FirstReportOf picks, from a machine's `apply` durations ordered by their start, its report of the send a walk made
+// to it (novox/hq issue 582):
+//   - with the machine's own send time kept (every walk since issue 582): the duration of that very send, and of the
+//     declaration sent where the walk kept its digest. A report of a later send is never this walk's: a send made
+//     later, by another walk, would end this walk at that walk's send.
+//   - with only the declaration's digest kept (a walk kept before): the first duration about that declaration
+//     within restSendBound before the walk's record of its send, the machine sent first in the rest included.
+//   - with neither (a walk kept before ADR 0282's rest): the first duration started no earlier than appliedSlack
+//     before the walk's record of its send, as it was read before.
+//
+// A send past ApplySilentAfter is never read: its machine slept, and is listed silent (ADR 0282 decision 1).
+func FirstReportOf(ds []Duration, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) {
+	from, to := applyRowsBetween(send, walkSent)
+	for _, d := range ds {
+		if d.Started.Before(from) || !d.Started.Before(to) {
+			continue
+		}
+		if send.Digest != "" && !strings.HasPrefix(d.Ref, send.Digest+"@") {
+			continue
+		}
+		return AppliedReport{At: d.Started.Add(d.Took).UTC(), Outcome: d.Detail}, true
+	}
+	return AppliedReport{}, false
+}
+
+// appliedSlack is how much earlier than a walk's record of its send the machine's own record of it may be, for a
+// walk that kept neither the machine's send time nor what it sent.
 const appliedSlack = 2 * time.Second
+
+// restSendBound is how long before a walk's record of its send to the rest a machine of the rest may have been
+// sent, for a walk that kept what it sent and not when: the send to the rest goes machine by machine, and took
+// 19 to 28 seconds on 2026-10-12 (novox/hq issue 582).
+const restSendBound = 5 * time.Minute

internal/inventory/walkphases_582_test.go

@@ -0,0 +1,188 @@
+package inventory
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+// novox/hq issue 582: the walk's own record of its send to the rest is made after every machine of the rest was
+// sent, and that send took 19 to 28 s on 2026-10-12. The machine sent first reported 5 s after its own send, long
+// before the walk's record; read from the walk's record, its report was missed and the walk ended at another walk's
+// send to it, minutes later. A walk of one tier and three machines of the rest, the send to the rest taking 20 s.
+type sendRest582 struct {
+	judged, walkSent time.Time
+	walk             Plan
+	rows             map[string][]Duration
+}
+
+func walk582(t *testing.T) sendRest582 {
+	t.Helper()
+	judged := time.Date(2026, 10, 12, 17, 45, 35, 137000000, time.UTC)
+	ownSend := map[string]time.Duration{"g14": 500 * time.Millisecond, "novox": 10 * time.Second, "shanks": 20 * time.Second}
+	walkSent := judged.Add(20200 * time.Millisecond)
+	merged := judged.Add(-10 * time.Minute)
+	built, first := judged.Add(-3*time.Minute), judged.Add(-2*time.Minute)
+	rest := map[string]SentDeclaration{}
+	rows := map[string][]Duration{}
+	took := map[string]time.Duration{"g14": 5 * time.Second, "novox": 13 * time.Second, "shanks": 15 * time.Second}
+	for i, n := range []string{"g14", "novox", "shanks"} {
+		at := judged.Add(ownSend[n])
+		digest := "walk-" + n
+		rest[n] = SentDeclaration{Digest: digest, Sequence: int64(647 + i), At: at}
+		rows[n] = append(rows[n],
+			// This walk's send, and its report.
+			Duration{Kind: DurationApply, Subject: n, Node: n, Ref: digest + "@" + at.Format("2006-01-02T15:04:05.000000"),
+				Started: at, Took: took[n], Detail: OutcomeApplied},
+			// The next walk's send to the same machine, nine minutes later, and its report.
+			Duration{Kind: DurationApply, Subject: n, Node: n, Ref: "next-" + n + "@x",
+				Started: judged.Add(9*time.Minute + ownSend[n]), Took: 4 * time.Second, Detail: OutcomeApplied})
+	}
+	p := Plan{ID: "plan-582", State: PlanDone, Tier: 1, Tiers: [][]string{{"build-agent"}}, Created: merged,
+		Delivery: &PlanDelivery{Merges: []PlanMerge{{Repository: "novox/mesh-controller", Commit: "c", Merged: merged}}},
+		Times:    &PlanTimes{Cut: &merged, Class: ClassCore},
+		Modules: map[string]*PlanModule{"build-agent": {State: "built", AskedAt: &merged, BuiltAt: &built,
+			FirstAt: &first, SentAt: &walkSent, Gate: &PlanGate{JudgedAt: &judged}, Rest: rest}}}
+	return sendRest582{judged: judged, walkSent: walkSent, walk: p, rows: rows}
+}
+
+// lookup answers from the rows as the controller's durations would.
+func (w sendRest582) lookup(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) {
+	return FirstReportOf(w.rows[node], send, walkSent)
+}
+
+// The apply phase ends at the last machine's own report of this walk's send — shanks, sent at the end of the 20 s
+// and reporting 15 s later — and never at the next walk's send; every machine's report is shown.
+func TestTheApplyPhaseEndsAtEachMachinesOwnReport(t *testing.T) {
+	c := walk582(t)
+	w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
+	sumsUp(t, w)
+	want := c.judged.Add(35 * time.Second) // shanks: sent 20 s after the judgement, reported 15 s later
+	if !w.End.Equal(want) {
+		t.Fatalf("the walk ends at %s, want %s: its rest %+v", w.End, want, w.Rest)
+	}
+	if got := phase(w, PhaseApply, -1); got.State != PhaseMeasured || got.TookMS != want.Sub(c.walkSent).Milliseconds() {
+		t.Fatalf("apply: %+v", got)
+	}
+	if len(w.Rest) != 3 || len(w.Silent) != 0 {
+		t.Fatalf("every machine of the rest is shown with its report: %+v, silent %v", w.Rest, w.Silent)
+	}
+	g14 := w.Rest[0]
+	if g14.Node != "g14" || g14.State != RestApplied || g14.Reported == nil ||
+		!g14.Reported.Equal(c.judged.Add(5500*time.Millisecond)) || g14.Took != "5s" || g14.Sequence != 647 ||
+		!g14.Sent.Equal(c.judged.Add(500*time.Millisecond)) || g14.SentSaid != "" {
+		t.Fatalf("g14, sent first and reporting 5 s after its own send: %+v", g14)
+	}
+}
+
+// A walk kept before its machines' own send times were: matched by the declaration it sent, within restSendBound
+// of its record of the send, so the machine sent first is still found; a walk that kept neither is read as before.
+func TestAnOlderWalkIsReadByWhatItSent(t *testing.T) {
+	c := walk582(t)
+	for n, s := range c.walk.Modules["build-agent"].Rest {
+		s.At = time.Time{}
+		c.walk.Modules["build-agent"].Rest[n] = s
+	}
+	w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
+	sumsUp(t, w)
+	if !w.End.Equal(c.judged.Add(35 * time.Second)) {
+		t.Fatalf("by the declaration sent: end %s, rest %+v", w.End, w.Rest)
+	}
+	if !strings.Contains(w.Rest[0].SentSaid, "walk's record") || !w.Rest[0].Sent.Equal(c.walkSent) {
+		t.Fatalf("a send time not kept is said: %+v", w.Rest[0])
+	}
+	// Nothing of the send kept: the first report no earlier than appliedSlack before the walk's record, as before.
+	if r, ok := FirstReportOf(c.rows["shanks"], SentDeclaration{}, c.walkSent); !ok || !r.At.Equal(c.judged.Add(35*time.Second)) {
+		t.Fatalf("a walk that kept nothing of its send: %+v %v", r, ok)
+	}
+}
+
+// A machine that said nothing of this walk's send is waiting, then silent past ApplySilentAfter: the report of a
+// later send to it is never taken as this walk's, and never moves the walk's end.
+func TestALaterSendsReportIsNeverThisWalks(t *testing.T) {
+	c := walk582(t)
+	c.rows["g14"] = c.rows["g14"][1:] // g14 slept through this walk's send; the next walk's send it reported
+	w := c.walk.WalkPhases(c.judged.Add(5*time.Minute), c.lookup)
+	if w.End != nil || !strings.Contains(phase(w, PhaseApply, -1).Said, "waiting for g14") {
+		t.Fatalf("g14 not heard from yet: end %v, apply %+v", w.End, phase(w, PhaseApply, -1))
+	}
+	w = c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
+	sumsUp(t, w)
+	if len(w.Silent) != 1 || w.Silent[0] != "g14" || !w.End.Equal(c.judged.Add(35*time.Second)) {
+		t.Fatalf("g14 silent, the end at shanks' report: silent %v, end %s", w.Silent, w.End)
+	}
+	if w.Rest[0].State != RestSilent || w.Rest[0].Reported != nil {
+		t.Fatalf("g14's later report taken as this walk's: %+v", w.Rest[0])
+	}
+}
+
+// A machine sent by two tiers of one walk that said nothing of the first send has said it of the second, which
+// carries the first tier's build too.
+func TestAMachineSentTwiceByOneWalkIsHeardByItsLaterSend(t *testing.T) {
+	c := walk582(t)
+	second := c.judged.Add(4 * time.Minute)
+	secondSent := second.Add(time.Second)
+	c.walk.Tiers = append(c.walk.Tiers, []string{"route-proxy"})
+	c.walk.Tier = 2
+	asked, built := c.judged.Add(time.Minute), c.judged.Add(2*time.Minute)
+	first := built.Add(time.Second)
+	c.walk.Modules["route-proxy"] = &PlanModule{State: "built", AskedAt: &asked, BuiltAt: &built, FirstAt: &first,
+		SentAt: &secondSent, Gate: &PlanGate{JudgedAt: &second},
+		Rest: map[string]SentDeclaration{"novox": {Digest: "walk-novox-2", Sequence: 1554, At: second.Add(500 * time.Millisecond)}}}
+	c.rows["novox"] = []Duration{{Kind: DurationApply, Subject: "novox", Ref: "walk-novox-2@x",
+		Started: second.Add(500 * time.Millisecond), Took: 14 * time.Second, Detail: OutcomeApplied}}
+	w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
+	sumsUp(t, w)
+	want := second.Add(14500 * time.Millisecond)
+	if !w.End.Equal(want) || len(w.Silent) != 0 {
+		t.Fatalf("end %s (want %s), silent %v, rest %+v", w.End, want, w.Silent, w.Rest)
+	}
+	for _, r := range w.Rest {
+		if r.Node == "novox" && (r.State != RestApplied || !r.Reported.Equal(want)) {
+			t.Fatalf("novox, heard by its later send of this walk: %+v", r)
+		}
+	}
+}
+
+// The store: a machine's own send time is read with what it was sent, and its `apply` duration is found by it —
+// the send and the duration start at one moment, the node's sent_at.
+func TestAWalksRestIsReadFromEachMachinesOwnSend(t *testing.T) {
+	inv := ForTest(t)
+	ctx := t.Context()
+	record, err := inv.AddNode(ctx, "g14")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := inv.NextSequence(ctx, record.ID); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordSent(ctx, record.ID, "d-582", map[string]string{"build-agent": "c1"}); err != nil {
+		t.Fatal(err)
+	}
+	sent, found, err := inv.SentTo(ctx, "g14")
+	if err != nil || !found || sent.At.IsZero() || sent.Digest != "d-582" {
+		t.Fatalf("the machine's own send time: %+v %v %v", sent, found, err)
+	}
+	if _, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)); err != nil || ok {
+		t.Fatalf("no report yet: %v %v", ok, err)
+	}
+	if err := inv.RecordApplyDuration(ctx, "g14", "d-582", OutcomeApplied); err != nil {
+		t.Fatal(err)
+	}
+	// The walk's record of its send, 20 s after the machine's own: still found, by the machine's own send.
+	r, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second))
+	if err != nil || !ok || r.Outcome != OutcomeApplied || r.At.Before(sent.At) {
+		t.Fatalf("the machine's own report of its send: %+v %v %v", r, ok, err)
+	}
+	// A later send, reported: never this walk's.
+	if err := inv.RecordSent(ctx, record.ID, "d-next", map[string]string{"build-agent": "c2"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordApplyDuration(ctx, "g14", "d-next", OutcomeApplied); err != nil {
+		t.Fatal(err)
+	}
+	again, _, _ := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second))
+	if !again.At.Equal(r.At) {
+		t.Fatalf("a later send's report read as this walk's: %+v, was %+v", again, r)
+	}
+}

internal/inventory/walkphases_test.go

@@ -116,7 +116,7 @@ func TestAWalkOfTwoTiersEndsAtItsRestsLastReport(t *testing.T) {
 		}}
 	reports := map[string]AppliedReport{"ace": {At: *at("05:12"), Outcome: OutcomeApplied},
 		"shanks": {At: *at("08:15.25"), Outcome: OutcomeApplied}}
-	w := p.WalkPhases(*at("30:00"), func(node string, _ time.Time) (AppliedReport, bool) {
+	w := p.WalkPhases(*at("30:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
 		r, ok := reports[node]
 		return r, ok
 	})
@@ -135,7 +135,7 @@ func TestAWalkOfTwoTiersEndsAtItsRestsLastReport(t *testing.T) {
 	}
 	// A report not yet in: the walk has no end, and its apply is unknown — never zero.
 	delete(reports, "shanks")
-	w = p.WalkPhases(*at("10:00"), func(node string, _ time.Time) (AppliedReport, bool) {
+	w = p.WalkPhases(*at("10:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
 		r, ok := reports[node]
 		return r, ok
 	})
@@ -146,7 +146,7 @@ func TestAWalkOfTwoTiersEndsAtItsRestsLastReport(t *testing.T) {
 		t.Errorf("apply while shanks has not reported: %+v", got)
 	}
 	// Silent past the bound: listed, not waited for.
-	w = p.WalkPhases(at("08:00.5").Add(ApplySilentAfter+time.Second), func(node string, _ time.Time) (AppliedReport, bool) {
+	w = p.WalkPhases(at("08:00.5").Add(ApplySilentAfter+time.Second), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
 		r, ok := reports[node]
 		return r, ok
 	})
@@ -156,7 +156,7 @@ func TestAWalkOfTwoTiersEndsAtItsRestsLastReport(t *testing.T) {
 	}
 	// A failed report: the walk has no end, said.
 	reports["shanks"] = AppliedReport{At: *at("08:10"), Outcome: OutcomeFailed}
-	w = p.WalkPhases(*at("30:00"), func(node string, _ time.Time) (AppliedReport, bool) {
+	w = p.WalkPhases(*at("30:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
 		r, ok := reports[node]
 		return r, ok
 	})
@@ -247,7 +247,7 @@ func TestAWalksMomentsAreKeptAndItsRestsReportRead(t *testing.T) {
 		t.Fatal("a merge's time was not kept")
 	}
 	sent := time.Now().UTC().Add(-time.Minute).Truncate(time.Millisecond)
-	if _, ok, err := inv.FirstAppliedAfter(ctx, "ace", sent); err != nil || ok {
+	if _, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent); err != nil || ok {
 		t.Fatalf("no report yet: %v %v", ok, err)
 	}
 	for _, d := range []Duration{
@@ -258,7 +258,7 @@ func TestAWalksMomentsAreKeptAndItsRestsReportRead(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
-	r, ok, err := inv.FirstAppliedAfter(ctx, "ace", sent)
+	r, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent)
 	if err != nil || !ok || r.Outcome != OutcomeApplied || !r.At.Equal(sent.Add(11*time.Second)) {
 		t.Fatalf("the report after the send: %+v %v %v", r, ok, err)
 	}
@@ -281,7 +281,7 @@ func TestALateReportIsSilentNotTheEnd(t *testing.T) {
 	p := walkOf(t, recordedWalk)
 	p.Modules["mesh-delivery"].SentAt = &sent
 	p.Modules["mesh-delivery"].Rest = map[string]SentDeclaration{"laptop": {Digest: "d"}, "server": {Digest: "e"}}
-	w := p.WalkPhases(sent.Add(3*time.Hour), func(node string, _ time.Time) (AppliedReport, bool) {
+	w := p.WalkPhases(sent.Add(3*time.Hour), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) {
 		if node == "server" {
 			return AppliedReport{At: sent.Add(20 * time.Second), Outcome: OutcomeApplied}, true
 		}
@@ -296,7 +296,7 @@ func TestALateReportIsSilentNotTheEnd(t *testing.T) {
 		Started: sent.Add(time.Hour), Took: time.Second, Detail: OutcomeApplied}); err != nil {
 		t.Fatal(err)
 	}
-	if _, ok, err := inv.FirstAppliedAfter(ctx, "laptop", sent); err != nil || ok {
+	if _, ok, err := inv.FirstAppliedAfter(ctx, "laptop", SentDeclaration{}, sent); err != nil || ok {
 		t.Fatalf("a send past the bound read as the walk's: %v %v", ok, err)
 	}
 }

internal/lease/lease.go

@@ -75,6 +75,10 @@ type Options struct {
 	Holder Holder
 	// RenewEvery, Margin and Poll default to the package's.
 	RenewEvery, Margin, Poll time.Duration
+	// RenewWithin is how long one renewal may take before it counts as failed: RenewEvery when zero. Apart
+	// from it so a test that renews often still gives a slow bus a serving controller's time to answer
+	// (novox/hq issue 591).
+	RenewWithin time.Duration
 	// Floor is the highest epoch this mesh has issued, from the controller's own store. **An epoch is
 	// never issued twice**: a lease bucket raised again from nothing — a bus whose data was replaced —
 	// starts its revisions over, and every machine that heard epoch 57 would refuse the next controller
@@ -138,6 +142,9 @@ func Open(ctx context.Context, js jetstream.JetStream, bucket string, o Options)
 	if o.RenewEvery <= 0 {
 		o.RenewEvery = RenewEvery
 	}
+	if o.RenewWithin <= 0 {
+		o.RenewWithin = o.RenewEvery
+	}
 	if o.Margin <= 0 {
 		o.Margin = Margin
 	}
@@ -406,7 +413,7 @@ func (l *Lease) Renew(ctx context.Context) error {
 	if err != nil {
 		return l.lose(err)
 	}
-	renewing, cancel := context.WithTimeout(ctx, l.o.RenewEvery)
+	renewing, cancel := context.WithTimeout(ctx, l.o.RenewWithin)
 	defer cancel()
 	next, err := l.kv.Update(renewing, Key, value, revision)
 	if err != nil {

internal/licences/fortest.go

@@ -32,14 +32,13 @@ func ForTest(t *testing.T) *Licences {
 		name = name[:60]
 	}
 
-	conn, err := pgx.Connect(t.Context(), admin)
+	// Created already migrated, a copy of a template migrated once (novox/hq issue 591); the Migrate
+	// below then finds nothing pending, which it does only if the copy's record matches every migration.
+	migrations, err := Migrations()
 	if err != nil {
-		t.Fatalf("cannot reach the test PostgreSQL: %v", err)
+		t.Fatal(err)
 	}
-	if _, err := conn.Exec(t.Context(), "create database "+name); err != nil {
-		t.Fatalf("cannot create %s: %v", name, err)
-	}
-	conn.Close(t.Context())
+	store.CreateMigratedForTest(t, admin, name, migrations)
 
 	cut := strings.LastIndex(admin, "/")
 	t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable")
@@ -61,10 +60,6 @@ func ForTest(t *testing.T) *Licences {
 		t.Fatal(err)
 	}
 
-	migrations, err := Migrations()
-	if err != nil {
-		t.Fatal(err)
-	}
 	if _, err := held.store.Migrate(t.Context(), migrations); err != nil {
 		t.Fatal(err)
 	}

internal/link/builds_nats.go

@@ -79,6 +79,8 @@ func (b *natsBuilds) Ask(ctx context.Context, request BuildRequest) error {
 	if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
 		return fmt.Errorf("cannot submit a build: %w", err)
 	}
+	// The acting controller puts it in order (novox/hq issue 558); this only says an ask joined.
+	signalAsked()
 	return nil
 }
 
@@ -113,6 +115,8 @@ func (b *natsBuilds) Submit(ctx context.Context, request BuildRequest,
 	if _, err := b.js.Context().Publish(BuildWorkOf(b.role()), body, nats.Context(publish)); err != nil {
 		return BuildResult{}, fmt.Errorf("%w: cannot submit a build: %w", ErrNotHandedOver, err)
 	}
+	// The acting controller puts it in order (novox/hq issue 558); this only says an ask joined.
+	signalAsked()
 
 	waiting, cancelWait := context.WithTimeout(ctx, wait)
 	defer cancelWait()
@@ -167,6 +171,10 @@ type MachineOptions struct {
 	// Paused is asked before every fetch: while it says so, nothing new is taken, and a build
 	// already running finishes. Nil is never paused.
 	Paused func() bool
+	// Taking is called once, when this machine is bound to its worker and pulls work: from then on it
+	// can take a build, which is what its container's health check reads (novox/hq issue 544). Nil is
+	// told nothing.
+	Taking func()
 }
 
 // MachineOverNATS takes build work from the current build role.
@@ -232,6 +240,9 @@ func (m *natsMachine) Take(ctx context.Context, do func(context.Context, Build))
 				"the mesh's to answer, not this machine's", m.seat, err)
 	}
 	m.sub = sub
+	if m.opts.Taking != nil {
+		m.opts.Taking()
+	}
 
 	for {
 		if ctx.Err() != nil {

internal/link/builds_nats_test.go

@@ -368,3 +368,36 @@ func TestNatsAMachineOnTheRetiredBuildRoleTakesThatRolesAsks(t *testing.T) {
 		t.Errorf("the retired role's holder did not answer: %+v", result)
 	}
 }
+
+// A holder says once that it takes work, and only once it is bound to its worker (novox/hq issue 544):
+// its container's health check reads that, so a holder that could not bind is never said ready.
+func TestNatsAHolderSaysItTakesWorkOnceItIsBound(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+	said := make(chan struct{}, 4)
+	machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Taking: func() { said <- struct{}{} }})
+	defer machine.Close()
+	go func() { _ = machine.Take(ctx, func(context.Context, Build) {}) }()
+	select {
+	case <-said:
+	case <-time.After(5 * time.Second):
+		t.Fatal("a holder bound to its worker never said it takes work")
+	}
+	time.Sleep(300 * time.Millisecond)
+	if len(said) != 0 {
+		t.Fatalf("a holder said it takes work %d more time(s)", len(said))
+	}
+
+	// A role the mesh raised no worker for cannot be bound, and its holder says nothing.
+	bounded, end := context.WithTimeout(ctx, 10*time.Second)
+	defer end()
+	other := MachineOverNATSWith(js, "anchor", "mesh-no-such-seat", MachineOptions{Taking: func() { said <- struct{}{} }})
+	defer other.Close()
+	if err := other.Take(bounded, func(context.Context, Build) {}); err == nil {
+		t.Fatal("a machine with no worker took work")
+	}
+	if len(said) != 0 {
+		t.Fatal("a machine that could not bind said it takes work")
+	}
+}

internal/link/queue.go

@@ -62,7 +62,11 @@ type QueuedAsk struct {
 	Path       string    `json:"path,omitempty"`
 	Ref        string    `json:"ref,omitempty"`
 	AskedAt    time.Time `json:"asked-at,omitempty"`
-	State      string    `json:"state"`
+	// Stored is when the stream first took it, a move into order kept (novox/hq issue 558).
+	Stored time.Time `json:"stored,omitempty"`
+	State  string    `json:"state"`
+	// movedFrom is the MovedFromHeader it carries: the sequence of the message a move copied it from.
+	movedFrom string
 	// On and Started are the holder building an in-flight ask and when it started, from its started
 	// event. Was is the holder that started an in-flight ask and is no longer building it — handed
 	// back, to be handed out again — with Started its start there.
@@ -236,15 +240,7 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
 		if err != nil {
 			return q, fmt.Errorf("reading the %s work queue: %w", seat, err)
 		}
-		ask := QueuedAsk{Seq: msg.Sequence}
-		if json.Unmarshal(msg.Data, &ask.Request) == nil {
-			r := ask.Request
-			ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref
-			if at, ok := BuildAskedAt(r.ID); ok {
-				ask.AskedAt = at
-			}
-		}
-		asks = append(asks, ask)
+		asks = append(asks, askOf(msg))
 		seq = msg.Sequence + 1
 	}
 
@@ -269,6 +265,26 @@ func ReadQueue(ctx context.Context, js *broker.JetStream, seat string) (Queue, e
 	return q, nil
 }
 
+// askOf is one ask as the stream holds it: what its request says, and when the stream took it.
+func askOf(msg *jetstream.RawStreamMsg) QueuedAsk {
+	ask := QueuedAsk{Seq: msg.Sequence, Stored: msg.Time}
+	// Moved into order, it keeps when the stream first took it (novox/hq issue 558).
+	if at, err := time.Parse(time.RFC3339Nano, msg.Header.Get(QueuedAtHeader)); err == nil {
+		ask.Stored = at
+	}
+	if json.Unmarshal(msg.Data, &ask.Request) == nil {
+		r := ask.Request
+		ask.ID, ask.Repository, ask.Path, ask.Ref = r.ID, r.Repository, r.Path, r.Ref
+		if at, ok := BuildAskedAt(r.ID); ok {
+			ask.AskedAt = at
+		}
+	} else {
+		// Nothing read is nothing kept: a half-read request would pass for a delivery (novox/hq issue 558).
+		ask.Request = BuildRequest{}
+	}
+	return ask
+}
+
 // WithoutPassword is a repository as a listing may say it: a URL's password, should one ever be written
 // into it, left out (novox/hq issue 455). Only for what is shown: the ask itself keeps what it was given,
 // which a cancel's outcome and a rebuild's match are read from.
@@ -305,11 +321,23 @@ func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, sin
 		return heard, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
 	}
 	defer func() { _ = sub.Unsubscribe() }()
-	for {
-		batch, err := sub.Fetch(500, nats.MaxWait(2*time.Second))
+	// **Read what is there, then stop** (novox/hq issue 591): the consumer says how many events it holds
+	// from the moment asked, and the reading ends once they are read. A fetch for a full batch waits out
+	// its whole wait when fewer are there, so every reading of the queue took two seconds and more.
+	info, err := sub.ConsumerInfo()
+	if err != nil {
+		return heard, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
+	}
+	for remaining := info.NumPending; remaining > 0; {
+		batch, err := sub.Fetch(int(min(remaining, 500)), nats.MaxWait(2*time.Second))
 		if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
 			return heard, fmt.Errorf("reading %s: %w", subject, err)
 		}
+		// None in the whole wait: what was counted is gone from the stream since (its limits), not coming.
+		if len(batch) == 0 {
+			break
+		}
+		remaining -= uint64(len(batch))
 		for _, msg := range batch {
 			switch msg.Subject {
 			case BuildStartedOf(seat):
@@ -326,9 +354,6 @@ func ReadBuildEvents(ctx context.Context, js *broker.JetStream, seat string, sin
 				}
 			}
 		}
-		if len(batch) < 500 {
-			break
-		}
 		if ctx.Err() != nil {
 			return heard, ctx.Err()
 		}

internal/link/queue_order.go

@@ -0,0 +1,426 @@
+package link
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+// The order the build queue hands asks out in (novox/hq issue 558).
+//
+// **A build a walk waits for goes before a pull request's check.** The queue was first come, first served,
+// and a merged change's delivery build waited minutes behind checks while its walk, and every walk after
+// it, waited on the build. So:
+//
+//   - a delivery is handed out before any check;
+//   - a check that has waited CheckWaitsAtMost goes before every delivery asked after it, so no check
+//     waits for ever behind a stream of deliveries;
+//   - within a kind, first come, first served, by when it was asked;
+//   - an ask whose kind cannot be told counts as a check: a delivery taken late breaks a budget, a check
+//     taken late breaks nothing.
+//
+// **Kept in the stream, never beside it.** The queue is the seat's work queue, which the server hands out
+// in stream order to whichever holder pulls; nothing else holds the order, so a restarted controller
+// finds it as it was left. An ask that must go behind another is moved to the tail — deleted from where
+// it was and published again, its body unchanged, under the same id — while the worker is paused, so no
+// holder takes it in the moment it moves. Only waiting asks move: what a holder took is never touched.
+
+// The kinds of ask, as the queue says them.
+const (
+	KindDelivery = "delivery"
+	KindCheck    = "check"
+)
+
+// CheckWaitsAtMost is how long a check waits before it goes ahead of the deliveries asked after it.
+var CheckWaitsAtMost = 10 * time.Minute
+
+// KindOf is an ask's kind: a check for a pull request's check — a recheck the merge tool asks included —
+// a dry run, or an ask nothing could be read from; a delivery for any other build.
+func KindOf(r BuildRequest) string {
+	if r.ID == "" || r.Check != nil || r.DryRun {
+		return KindCheck
+	}
+	return KindDelivery
+}
+
+// askedOrStored is when an ask was asked, as its id says, or else when the queue took it.
+func (a QueuedAsk) askedOrStored() time.Time {
+	if !a.AskedAt.IsZero() {
+		return a.AskedAt
+	}
+	return a.Stored
+}
+
+// QueueOrder is the order the waiting asks are to be handed out in, now: the deliveries and every check
+// that has waited CheckWaitsAtMost, by when each was asked, then the other checks, by when each was asked.
+func QueueOrder(waiting []QueuedAsk, now time.Time) []QueuedAsk {
+	var first, then []QueuedAsk
+	for _, a := range waiting {
+		if KindOf(a.Request) == KindCheck && now.Sub(a.askedOrStored()) < CheckWaitsAtMost {
+			then = append(then, a)
+			continue
+		}
+		first = append(first, a)
+	}
+	byAsking := func(s []QueuedAsk) {
+		sort.SliceStable(s, func(i, j int) bool {
+			if ti, tj := s[i].askedOrStored(), s[j].askedOrStored(); !ti.Equal(tj) {
+				return ti.Before(tj)
+			}
+			return s[i].Seq < s[j].Seq
+		})
+	}
+	byAsking(first)
+	byAsking(then)
+	return append(first, then...)
+}
+
+// toMove is the asks to move to the tail, in this order, for the queue to stand as order says: everything
+// after the longest start of order that the queue already holds in that order.
+func toMove(queue, order []QueuedAsk) []QueuedAsk {
+	at := make(map[uint64]int, len(queue))
+	for i, a := range queue {
+		at[a.Seq] = i
+	}
+	kept, last := 0, -1
+	for kept < len(order) {
+		i, ok := at[order[kept].Seq]
+		if !ok || i < last {
+			break
+		}
+		last = i
+		kept++
+	}
+	return order[kept:]
+}
+
+// orderPasses keeps one pass at a time. Only the acting controller's loop runs one (KeepingInOrder), so
+// there is one; the lock holds a test's two at once to that too.
+var orderPasses sync.Mutex
+
+// pausedFor bounds a pause the pass could not end itself: a controller that stops mid-pass leaves the
+// worker paused this long at most. moveMargin is the part of it no move starts in: a pass stops moving
+// with at least this left, and the next pass moves the rest, so no move runs past the pause.
+const (
+	pausedFor    = 20 * time.Second
+	moveMargin   = 5 * time.Second
+	publishWaits = 3 * time.Second
+)
+
+// QueuedAtHeader carries, on a moved ask, when the stream first took it: a move keeps an ask's place in
+// time, its id unread or not.
+const QueuedAtHeader = "Mesh-Queued-At"
+
+// MovedFromHeader names, on a move's copy, the sequence of the message it is a copy of: with QueuedAtHeader,
+// what proves a second message of one id is a move's copy and not an ask delivered again (ADR 0293), whose
+// headers are its original's — and so never name that original's own sequence.
+const MovedFromHeader = "Mesh-Moved-From"
+
+// Pass is what one pass did: the asks it moved, and each second copy it removed, in words.
+type Pass struct {
+	Moved   []string
+	Removed []string
+}
+
+// asked is the signal an ask sends the acting controller's pass loop, in this process. An ask asked in
+// another process — a verb run as its own command — is seen by the loop as the queue's last sequence moving.
+var asked = make(chan struct{}, 1)
+
+// signalAsked tells the pass loop an ask joined the queue. Never blocks, never runs a pass.
+func signalAsked() {
+	select {
+	case asked <- struct{}{}:
+	default:
+	}
+}
+
+// PutInOrder moves the seat's waiting asks into the order QueueOrder says, and says each it moved. A queue
+// already in order is read and left alone; the worker is paused only while asks move. Run by the acting
+// controller alone (KeepingInOrder): two passes in two processes would each resume what the other paused.
+func PutInOrder(ctx context.Context, js *broker.JetStream, seat string) (Pass, error) {
+	var pass Pass
+	orderPasses.Lock()
+	defer orderPasses.Unlock()
+	api, err := jetstream.New(js.Conn())
+	if err != nil {
+		return pass, err
+	}
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
+	waiting, extra, delivered, err := readWaiting(ctx, api, worker, seat)
+	if err != nil || (len(extra) == 0 && len(toMove(waiting, QueueOrder(waiting, time.Now()))) == 0) {
+		return pass, err
+	}
+	// Paused, then read again: what the worker handed out up to the pause is not waiting any more.
+	pauseEnds := time.Now().Add(pausedFor)
+	if delivered >= 0 {
+		if _, err := api.PauseConsumer(ctx, worker.Stream, worker.Name, pauseEnds); err != nil {
+			return pass, fmt.Errorf("the %s worker could not be paused to put its queue in order, so it is left "+
+				"first come, first served: %w", seat, err)
+		}
+		defer func() {
+			resuming, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
+			defer cancel()
+			if _, err := api.ResumeConsumer(resuming, worker.Stream, worker.Name); err != nil {
+				fmt.Printf("the %s worker could not be resumed after its queue was put in order; it resumes by "+
+					"itself by %s: %v\n", seat, pauseEnds.Format(time.RFC3339), err)
+			}
+		}()
+		if waiting, extra, _, err = readWaiting(ctx, api, worker, seat); err != nil {
+			return pass, err
+		}
+	}
+	stream, err := api.Stream(ctx, worker.Stream)
+	if err != nil {
+		return pass, err
+	}
+	// A second copy of one ask — a move that published and stopped before it deleted — goes first.
+	for _, a := range extra {
+		if err := stream.DeleteMsg(ctx, a.Seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
+			return pass, fmt.Errorf("a second copy of %s (seq %d) could not be removed: %w", a.ID, a.Seq, err)
+		}
+		pass.Removed = append(pass.Removed, fmt.Sprintf("%s at seq %d, a move's second copy of the message at seq %s",
+			a.ID, a.Seq, a.movedFrom))
+	}
+	for _, a := range toMove(waiting, QueueOrder(waiting, time.Now())) {
+		if delivered >= 0 && time.Until(pauseEnds) < moveMargin {
+			// The rest waits for the next pass, under a pause of its own.
+			break
+		}
+		ok, err := moveToTheTail(ctx, js, stream, worker.Name, seat, a)
+		if err != nil {
+			return pass, err
+		}
+		if ok {
+			pass.Moved = append(pass.Moved, a.ID)
+		}
+	}
+	return pass, nil
+}
+
+// readWaiting is every ask the worker has not yet handed out, in queue order, but each second copy of one
+// ask; the second copies, to remove; and the last sequence the worker handed out, -1 when there is no
+// worker, so nothing is handed out to pause.
+//
+// **A second copy is proven, never guessed from an id**: a waiting message whose MovedFromHeader names an
+// earlier message of the same request id still in the queue, and whose QueuedAtHeader is that message's
+// first-queued time — a move that published its copy and stopped before it deleted the original. The
+// original stays, waiting or handed out (a holder's then, and the copy would build it twice). Any other
+// second message of one id — an ask a dead letter delivered again beside its original (ADR 0293) — is a
+// real ask and stays.
+func readWaiting(ctx context.Context, api jetstream.JetStream, worker broker.Consumer, seat string) (
+	waiting, extra []QueuedAsk, delivered int64, err error) {
+	stream, err := api.Stream(ctx, worker.Stream)
+	if err != nil {
+		return nil, nil, 0, fmt.Errorf("the %s work queue (%s) cannot be read: %w", seat, worker.Stream, err)
+	}
+	delivered = -1
+	if consumer, err := stream.Consumer(ctx, worker.Name); err == nil {
+		info, err := consumer.Info(ctx)
+		if err != nil {
+			return nil, nil, 0, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
+		}
+		delivered = int64(info.Delivered.Stream)
+	} else if !errors.Is(err, jetstream.ErrConsumerNotFound) {
+		return nil, nil, 0, fmt.Errorf("the worker of %s cannot be read: %w", seat, err)
+	}
+	// From the first: the stream holds only what is unsettled, so this is a handful of reads.
+	// Every message by sequence, with its id and first-queued time, to prove a copy against.
+	type held struct {
+		id     string
+		stored time.Time
+	}
+	bySeq := map[string]held{}
+	var seq uint64 = 1
+	for n := 0; n < 10000; n++ {
+		msg, err := stream.GetMsg(ctx, seq, jetstream.WithGetMsgSubject(BuildWorkOf(seat)))
+		if errors.Is(err, jetstream.ErrMsgNotFound) {
+			break
+		}
+		if err != nil {
+			return nil, nil, 0, fmt.Errorf("reading the %s work queue: %w", seat, err)
+		}
+		seq = msg.Sequence + 1
+		a := askOf(msg)
+		a.movedFrom = msg.Header.Get(MovedFromHeader)
+		from, known := bySeq[a.movedFrom]
+		again := a.ID != "" && a.movedFrom != "" && known && from.id == a.ID && from.stored.Equal(a.Stored)
+		bySeq[strconv.FormatUint(a.Seq, 10)] = held{id: a.ID, stored: a.Stored}
+		switch {
+		case int64(a.Seq) <= delivered:
+		case again:
+			extra = append(extra, a)
+		default:
+			waiting = append(waiting, a)
+		}
+	}
+	return waiting, extra, delivered, nil
+}
+
+// moveToTheTail publishes the ask's body again at the queue's tail and then deletes it where it was. **The
+// copy first**: a publish that fails deletes nothing, and a pass that stops between the two leaves two
+// copies, which the next pass sees by their one id and makes one (readWaiting). A move never loses an ask.
+// Looked at once more just before: one the worker has handed out since is a holder's now and stays. False
+// when it was not moved.
+func moveToTheTail(ctx context.Context, js *broker.JetStream, stream jetstream.Stream, workerName, seat string,
+	a QueuedAsk) (bool, error) {
+	if consumer, err := stream.Consumer(ctx, workerName); err == nil {
+		info, err := consumer.Info(ctx)
+		if err != nil {
+			return false, err
+		}
+		if info.Delivered.Stream >= a.Seq {
+			return false, nil
+		}
+	}
+	msg, err := stream.GetMsg(ctx, a.Seq)
+	if errors.Is(err, jetstream.ErrMsgNotFound) {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	again := &nats.Msg{Subject: msg.Subject, Data: msg.Data, Header: nats.Header{}}
+	for k, v := range msg.Header {
+		// The server's own headers stay behind: a message id would be taken for a duplicate and dropped.
+		if !strings.HasPrefix(k, "Nats-") {
+			again.Header[k] = v
+		}
+	}
+	// Its place in time goes with it: when the stream first took it, as askOf read it; and the message it
+	// is a copy of, which proves it one.
+	again.Header.Set(QueuedAtHeader, askOf(msg).Stored.UTC().Format(time.RFC3339Nano))
+	again.Header.Set(MovedFromHeader, strconv.FormatUint(a.Seq, 10))
+	// Inside the margin, so the publish ends while the worker is still paused.
+	publishing, cancel := context.WithTimeout(ctx, publishWaits)
+	ack, err := js.Context().PublishMsg(again, nats.Context(publishing))
+	cancel()
+	if err != nil {
+		return false, fmt.Errorf("%s (seq %d) could not be put behind, and stays where it is: %w", a.ID, a.Seq, err)
+	}
+	// **Handed out meanwhile** — a bus slow enough that the pause ended: the original is a holder's, and the
+	// copy goes instead, so it is built once.
+	if consumer, err := stream.Consumer(ctx, workerName); err == nil {
+		info, err := consumer.Info(ctx)
+		if err != nil || info.Delivered.Stream >= a.Seq {
+			if derr := stream.DeleteMsg(ctx, ack.Sequence); derr != nil && !errors.Is(derr, jetstream.ErrMsgNotFound) {
+				return false, fmt.Errorf("%s was handed out while it moved and its copy (seq %d) could not be "+
+					"removed; the next pass removes it: %w", a.ID, ack.Sequence, derr)
+			}
+			if err != nil {
+				return false, fmt.Errorf("whether %s was handed out while it moved cannot be read, so its copy was "+
+					"removed and it stays where it was: %w", a.ID, err)
+			}
+			if info.Delivered.Stream >= ack.Sequence {
+				return false, fmt.Errorf("%s was handed out while it moved, and its copy too: it may be built twice", a.ID)
+			}
+			return false, nil
+		}
+	}
+	if err := stream.DeleteMsg(ctx, a.Seq); err != nil && !errors.Is(err, jetstream.ErrMsgNotFound) {
+		return false, fmt.Errorf("%s was copied behind and its original (seq %d) not removed; the next pass "+
+			"removes the second copy: %w", a.ID, a.Seq, err)
+	}
+	return true, nil
+}
+
+// KeepingInOrder runs the passes, on the acting controller alone, until ctx ends: when an ask in this
+// process says so, when the queue's last sequence moved — an ask from another process — and every
+// passEvery, for a check that comes of age with nothing asked. seat says the build seat to order; acting
+// whether this controller acts (Holding); said what a pass did.
+func KeepingInOrder(ctx context.Context, js *broker.JetStream, seat func() string, acting func() bool,
+	said func(format string, args ...any)) {
+	tick := time.NewTicker(orderLooks)
+	defer tick.Stop()
+	var lastSeq uint64
+	var lastPass time.Time
+	for {
+		signalled := false
+		select {
+		case <-ctx.Done():
+			return
+		case <-asked:
+			signalled = true
+		case <-tick.C:
+		}
+		if !acting() {
+			continue
+		}
+		name := seat()
+		worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: name, Accepts: []string{"build"}})
+		info, err := js.Context().StreamInfo(worker.Stream)
+		if err != nil {
+			continue
+		}
+		if !signalled && info.State.LastSeq == lastSeq && time.Since(lastPass) < passEvery {
+			continue
+		}
+		lastSeq, lastPass = info.State.LastSeq, time.Now()
+		passing, cancel := context.WithTimeout(ctx, pausedFor+10*time.Second)
+		pass, err := PutInOrder(passing, js, name)
+		cancel()
+		for _, r := range pass.Removed {
+			said("the %s queue: removed %s", name, r)
+		}
+		if len(pass.Moved) > 0 {
+			said("the %s queue put in order: %s moved behind what goes first", name, strings.Join(pass.Moved, ", "))
+		}
+		if err != nil {
+			said("the %s queue could not be put in order: %v", name, err)
+		}
+		if info, err := js.Context().StreamInfo(worker.Stream); err == nil {
+			// Its own moves are not an ask.
+			lastSeq = info.State.LastSeq
+		}
+	}
+}
+
+// orderLooks is how often the loop looks whether an ask joined from another process; passEvery how often
+// it passes with nothing asked.
+var (
+	orderLooks = 2 * time.Second
+	passEvery  = 30 * time.Second
+)
+
+// DeleteAsk removes a waiting or dead ask from the queue: at its sequence, or — moved into order since it
+// was read — the copy carrying its id. Says the sequence it removed, 0 when neither was there any more.
+func DeleteAsk(ctx context.Context, js *broker.JetStream, seat string, ask QueuedAsk) (uint64, error) {
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
+	gone := func(err error) bool {
+		return errors.Is(err, nats.ErrMsgNotFound) || errors.Is(err, jetstream.ErrMsgNotFound) ||
+			(err != nil && strings.Contains(err.Error(), "no message found"))
+	}
+	err := js.Context().DeleteMsg(worker.Stream, ask.Seq)
+	if err == nil {
+		return ask.Seq, nil
+	}
+	if !gone(err) {
+		return 0, err
+	}
+	if ask.ID == "" {
+		return 0, nil
+	}
+	q, err := ReadQueue(ctx, js, seat)
+	if err != nil {
+		return 0, err
+	}
+	for _, a := range q.Asks {
+		if a.ID == ask.ID && a.Seq != ask.Seq && a.State != AskInFlight {
+			if err := js.Context().DeleteMsg(worker.Stream, a.Seq); err != nil && !gone(err) {
+				return 0, err
+			} else if err == nil {
+				return a.Seq, nil
+			}
+		}
+	}
+	return 0, nil
+}

internal/link/queue_order_test.go

@@ -0,0 +1,441 @@
+package link
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+
+	"github.com/novox/mesh-controller/internal/broker"
+)
+
+// The order the build queue hands asks out in (novox/hq issue 558): a build a walk waits for before any
+// pull request's check, a check that has waited ten minutes before every delivery asked after it, and
+// first come, first served within a kind.
+
+func anAsk(seq uint64, id string, asked time.Time, check bool) QueuedAsk {
+	r := BuildRequest{ID: id, Repository: "/r"}
+	if check {
+		r.Check = &CheckRequest{Owner: "novox", Repo: "r"}
+	}
+	return QueuedAsk{Seq: seq, ID: id, AskedAt: asked, Request: r, State: AskWaiting}
+}
+
+func idsOf(asks []QueuedAsk) []string {
+	out := []string{}
+	for _, a := range asks {
+		out = append(out, a.ID)
+	}
+	return out
+}
+
+func TestADeliveryIsHandedOutBeforeTheChecksAskedBeforeIt(t *testing.T) {
+	now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
+	waiting := []QueuedAsk{
+		anAsk(1, "check-a", now.Add(-3*time.Minute), true),
+		anAsk(2, "check-b", now.Add(-2*time.Minute), true),
+		anAsk(3, "delivery-d", now.Add(-time.Minute), false),
+	}
+	want := []string{"delivery-d", "check-a", "check-b"}
+	if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
+		t.Fatalf("handed out as %v, want %v", got, want)
+	}
+	// Only the checks ahead of it move behind it; the delivery stays where it is.
+	if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"check-a", "check-b"}) {
+		t.Fatalf("moved %v", got)
+	}
+}
+
+func TestACheckThatWaitedTenMinutesGoesBeforeALaterDelivery(t *testing.T) {
+	now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC)
+	// As the queue stands after a delivery put the young checks behind it, ten minutes ago and since.
+	waiting := []QueuedAsk{
+		anAsk(5, "delivery-early", now.Add(-12*time.Minute), false),
+		anAsk(6, "delivery-later", now.Add(-time.Minute), false),
+		anAsk(7, "check-old", now.Add(-11*time.Minute), true),
+		anAsk(8, "check-young", now.Add(-9*time.Minute), true),
+	}
+	want := []string{"delivery-early", "check-old", "delivery-later", "check-young"}
+	if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) {
+		t.Fatalf("handed out as %v, want %v", got, want)
+	}
+	if got := idsOf(toMove(waiting, QueueOrder(waiting, now))); !reflect.DeepEqual(got, []string{"delivery-later", "check-young"}) {
+		t.Fatalf("moved %v", got)
+	}
+}
+
+func TestAnAskWhoseKindCannotBeToldIsACheck(t *testing.T) {
+	for _, r := range []BuildRequest{
+		{},                                      // unreadable: nothing was read from it
+		{ID: "build-1", Check: &CheckRequest{}}, // a pull request's check, or a recheck the merge tool asked
+		{ID: "build-2", DryRun: true},           // looked at, never delivered
+	} {
+		if got := KindOf(r); got != KindCheck {
+			t.Errorf("%+v is a %s", r, got)
+		}
+	}
+	if got := KindOf(BuildRequest{ID: "build-3", Repository: "/r"}); got != KindDelivery {
+		t.Errorf("a build is a %s", got)
+	}
+}
+
+// Against a real server: two checks wait, a delivery is asked, and the next free holder takes the delivery.
+func TestNatsADeliveryIsTakenBeforeTheChecksQueuedAheadOfIt(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx, stop := context.WithCancel(context.Background())
+	defer stop()
+	asks := BuildsOn(js, TheBuildMachine)
+	now := time.Now()
+	for i, r := range []BuildRequest{
+		{ID: NewBuildID(now.Add(-3 * time.Minute)), Repository: "/a", Check: &CheckRequest{Owner: "novox", Repo: "a"}},
+		{ID: NewBuildID(now.Add(-2 * time.Minute)), Repository: "/b", Check: &CheckRequest{Owner: "novox", Repo: "b"}},
+		{ID: NewBuildID(now), Repository: "/d"},
+	} {
+		if i == 2 {
+			// And one nobody can read, whose kind cannot be told: it waits with the checks.
+			if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
+				t.Fatal(err)
+			}
+		}
+		if err := asks.Ask(ctx, r); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// The acting controller's loop puts it in order, told by the asks.
+	go KeepingInOrder(ctx, js, func() string { return TheBuildMachine }, func() bool { return true },
+		func(string, ...any) {})
+	time.Sleep(500 * time.Millisecond)
+	took := make(chan string, 4)
+	machine := MachineOverNATS(js, "anchor")
+	defer machine.Close()
+	go func() {
+		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
+			took <- work.Request().Repository
+			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
+			_ = work.Done()
+		})
+	}()
+	var order []string
+	for len(order) < 3 {
+		select {
+		case r := <-took:
+			order = append(order, r)
+		case <-time.After(10 * time.Second):
+			t.Fatalf("taken so far: %v", order)
+		}
+	}
+	if want := []string{"/d", "/a", "/b"}; !reflect.DeepEqual(order, want) {
+		t.Fatalf("taken as %v, want %v", order, want)
+	}
+}
+
+// --- the moves, against a real server ------------------------------------------------------------
+
+func publishAsk(t *testing.T, js *broker.JetStream, r BuildRequest) {
+	t.Helper()
+	body, _ := json.Marshal(r)
+	if _, err := js.Context().Publish(BuildWork(), body); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func aCheck(asked time.Time, repository string) BuildRequest {
+	return BuildRequest{ID: NewBuildID(asked), Repository: repository, Check: &CheckRequest{Owner: "novox", Repo: "r"}}
+}
+
+func aDelivery(asked time.Time, repository string) BuildRequest {
+	return BuildRequest{ID: NewBuildID(asked), Repository: repository}
+}
+
+// queued is the stream's asks, in queue order.
+func queued(t *testing.T, js *broker.JetStream) []QueuedAsk {
+	t.Helper()
+	q, err := ReadQueue(context.Background(), js, TheBuildMachine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return q.Asks
+}
+
+// takeAll runs a holder that records how often it was handed each repository.
+func takeAll(t *testing.T, ctx context.Context, js *broker.JetStream) (chan string, func() map[string]int) {
+	t.Helper()
+	took := make(chan string, 100)
+	var mu sync.Mutex
+	counts := map[string]int{}
+	machine := MachineOverNATS(js, "anchor")
+	t.Cleanup(machine.Close)
+	go func() {
+		_ = machine.Take(ctx, func(ctx context.Context, work Build) {
+			mu.Lock()
+			counts[work.Request().Repository]++
+			mu.Unlock()
+			took <- work.Request().Repository
+			_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
+			_ = work.Done()
+		})
+	}()
+	return took, func() map[string]int {
+		mu.Lock()
+		defer mu.Unlock()
+		out := map[string]int{}
+		for k, v := range counts {
+			out[k] = v
+		}
+		return out
+	}
+}
+
+// A move that published its copy and stopped before deleting the original leaves two copies of one ask;
+// the next pass makes them one, and a copy of one a holder already took is removed, never built twice.
+func TestNatsAMoveCutShortLeavesOneCopyAfterTheNextPass(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx := t.Context()
+	now := time.Now()
+	check := aCheck(now.Add(-2*time.Minute), "/c")
+	publishAsk(t, js, check)
+	publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/d"))
+	moveCopyOf(t, js, check.ID) // the copy, published; the original never deleted
+	if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Removed) != 1 {
+		t.Fatalf("removed %v: %v", pass.Removed, err)
+	}
+	var got []string
+	for _, a := range queued(t, js) {
+		got = append(got, a.Repository)
+	}
+	if want := []string{"/d", "/c"}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("after the pass the queue holds %v, want %v", got, want)
+	}
+
+	// And one a holder took before the copy was published: the copy goes, the holder's stays.
+	taken := aCheck(now, "/taken")
+	publishAsk(t, js, taken)
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
+	sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	// Take everything waiting up to and including /taken, and keep /taken unsettled.
+	for {
+		msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
+		if err != nil || len(msgs) == 0 {
+			t.Fatalf("nothing to take: %v", err)
+		}
+		var r BuildRequest
+		_ = json.Unmarshal(msgs[0].Data, &r)
+		if r.Repository == "/taken" {
+			break
+		}
+		_ = msgs[0].Ack()
+	}
+	moveCopyOf(t, js, taken.ID)
+	if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
+		t.Fatal(err)
+	}
+	n := 0
+	for _, a := range queued(t, js) {
+		if a.ID == taken.ID {
+			n++
+		}
+	}
+	if n != 1 {
+		t.Fatalf("%d copies of the ask a holder took are in the queue, want 1", n)
+	}
+}
+
+// moveCopyOf publishes what a move publishes for the ask with this id, as a pass stopped before its delete
+// would leave it.
+func moveCopyOf(t *testing.T, js *broker.JetStream, id string) {
+	t.Helper()
+	for _, a := range queued(t, js) {
+		if a.ID != id {
+			continue
+		}
+		body, _ := json.Marshal(a.Request)
+		msg := &nats.Msg{Subject: BuildWork(), Data: body, Header: nats.Header{}}
+		msg.Header.Set(QueuedAtHeader, a.Stored.UTC().Format(time.RFC3339Nano))
+		msg.Header.Set(MovedFromHeader, fmt.Sprint(a.Seq))
+		if _, err := js.Context().PublishMsg(msg); err != nil {
+			t.Fatal(err)
+		}
+		return
+	}
+	t.Fatalf("%s is not queued", id)
+}
+
+// A dead letter delivered again whose original stayed beside it (ADR 0293: its removal failed) is a real
+// ask with the same id, and is never removed as a second copy — even when the original was itself a move's
+// copy, whose headers the delivery carries.
+func TestNatsAnAskDeliveredAgainBesideItsOriginalStays(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx := t.Context()
+	ask := aCheck(time.Now(), "/again")
+	publishAsk(t, js, aCheck(time.Now().Add(-time.Minute), "/before"))
+	publishAsk(t, js, ask)
+	publishAsk(t, js, aDelivery(time.Now(), "/d"))
+	if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil { // /again becomes a move's copy
+		t.Fatal(err)
+	}
+	worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}})
+	sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	var original *nats.Msg
+	for original == nil {
+		msgs, err := sub.Fetch(1, nats.MaxWait(2*time.Second))
+		if err != nil || len(msgs) == 0 {
+			t.Fatalf("nothing to take: %v", err)
+		}
+		var r BuildRequest
+		_ = json.Unmarshal(msgs[0].Data, &r)
+		if r.ID == ask.ID {
+			original = msgs[0] // given up on, never settled: dead, and kept
+			continue
+		}
+		_ = msgs[0].Ack()
+	}
+	// Delivered again as DeliverAgain does: its headers, and its own.
+	again := &nats.Msg{Subject: BuildWork(), Data: original.Data, Header: nats.Header{}}
+	for k, v := range original.Header {
+		again.Header[k] = v
+	}
+	again.Header.Set(AgainHeader, "7")
+	if _, err := js.Context().PublishMsg(again); err != nil {
+		t.Fatal(err)
+	}
+	pass, err := PutInOrder(ctx, js, TheBuildMachine)
+	if err != nil || len(pass.Removed) != 0 {
+		t.Fatalf("removed %v: %v", pass.Removed, err)
+	}
+	n := 0
+	for _, a := range queued(t, js) {
+		if a.ID == ask.ID {
+			n++
+		}
+	}
+	if n != 2 {
+		t.Fatalf("%d messages of the ask delivered again are queued, want its original and the delivery", n)
+	}
+}
+
+// Two passes at once: nothing is built twice, and nothing is lost.
+func TestNatsTwoPassesAtOnceBuildNothingTwice(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx, stop := context.WithCancel(t.Context())
+	defer stop()
+	took, counts := takeAll(t, ctx, js)
+	now := time.Now()
+	const asks = 24
+	var passes sync.WaitGroup
+	for p := 0; p < 2; p++ {
+		passes.Add(1)
+		go func() {
+			defer passes.Done()
+			for i := 0; i < 15; i++ {
+				_, _ = PutInOrder(ctx, js, TheBuildMachine)
+			}
+		}()
+	}
+	for i := 0; i < asks; i++ {
+		repository := fmt.Sprintf("/%02d", i)
+		if i%3 == 2 {
+			publishAsk(t, js, aDelivery(now.Add(time.Duration(i)*time.Millisecond), repository))
+		} else {
+			publishAsk(t, js, aCheck(now.Add(time.Duration(i)*time.Millisecond), repository))
+		}
+	}
+	passes.Wait()
+	for got := 0; got < asks; got++ {
+		select {
+		case <-took:
+		case <-time.After(15 * time.Second):
+			t.Fatalf("taken %d of %d: %v", got, asks, counts())
+		}
+	}
+	time.Sleep(time.Second)
+	c := counts()
+	for i := 0; i < asks; i++ {
+		if n := c[fmt.Sprintf("/%02d", i)]; n != 1 {
+			t.Errorf("/%02d was built %d times", i, n)
+		}
+	}
+}
+
+// A check that has waited ten minutes goes before a delivery asked after it, even one queued ahead of it.
+func TestNatsACheckThatWaitedTenMinutesIsTakenBeforeALaterDelivery(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx, stop := context.WithCancel(t.Context())
+	defer stop()
+	now := time.Now()
+	publishAsk(t, js, aDelivery(now.Add(-time.Minute), "/delivery"))
+	publishAsk(t, js, aCheck(now.Add(-11*time.Minute), "/old-check"))
+	if _, err := PutInOrder(ctx, js, TheBuildMachine); err != nil {
+		t.Fatal(err)
+	}
+	took, _ := takeAll(t, ctx, js)
+	var order []string
+	for len(order) < 2 {
+		select {
+		case r := <-took:
+			order = append(order, r)
+		case <-time.After(10 * time.Second):
+			t.Fatalf("taken so far: %v", order)
+		}
+	}
+	if want := []string{"/old-check", "/delivery"}; !reflect.DeepEqual(order, want) {
+		t.Fatalf("taken as %v, want %v", order, want)
+	}
+}
+
+// A cancel that read an ask before it moved deletes the moved copy, found by its id, and says when
+// there was nothing left to delete.
+func TestNatsACancelRacingAMoveDeletesTheMovedCopy(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx := t.Context()
+	now := time.Now()
+	publishAsk(t, js, aCheck(now.Add(-time.Minute), "/c"))
+	read := queued(t, js)[0]
+	publishAsk(t, js, aDelivery(now, "/d"))
+	if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
+		t.Fatalf("moved %v: %v", pass.Moved, err)
+	}
+	removed, err := DeleteAsk(ctx, js, TheBuildMachine, read)
+	if err != nil || removed == 0 || removed == read.Seq {
+		t.Fatalf("removed seq %d (read at %d): %v", removed, read.Seq, err)
+	}
+	for _, a := range queued(t, js) {
+		if a.ID == read.ID {
+			t.Fatalf("the cancelled ask is still queued at seq %d", a.Seq)
+		}
+	}
+	if removed, err := DeleteAsk(ctx, js, TheBuildMachine, read); err != nil || removed != 0 {
+		t.Fatalf("an ask gone already is said removed at %d: %v", removed, err)
+	}
+}
+
+// An ask nothing can be read from keeps its place in time when it moves.
+func TestNatsAnUnreadableAskKeepsItsAgeWhenItMoves(t *testing.T) {
+	js := aBusWithTheBuildRole(t)
+	ctx := t.Context()
+	if _, err := js.Context().Publish(BuildWork(), []byte("not an ask")); err != nil {
+		t.Fatal(err)
+	}
+	first := queued(t, js)[0].Stored
+	time.Sleep(50 * time.Millisecond)
+	publishAsk(t, js, aDelivery(time.Now(), "/d"))
+	if pass, err := PutInOrder(ctx, js, TheBuildMachine); err != nil || len(pass.Moved) != 1 {
+		t.Fatalf("moved %v: %v", pass.Moved, err)
+	}
+	asks := queued(t, js)
+	if len(asks) != 2 || asks[1].ID != "" || !asks[1].Stored.Equal(first) {
+		t.Fatalf("after the move the unreadable ask was taken at %v, first at %v: %+v", asks[1].Stored, first, asks)
+	}
+}

internal/store/fortest.go

@@ -0,0 +1,138 @@
+package store
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgconn"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+// CreateMigratedForTest creates the database name on the test PostgreSQL that admin reaches, holding
+// every one of migrations already applied and recorded, as a fresh database migrated by Migrate would.
+//
+// **A copy of a template, migrated once** (novox/hq issue 591): a store test used to create an empty
+// database and run every migration into it, some 90 for the inventory, and with hundreds of store tests
+// that alone was most of the controller's check. The template is migrated by Migrate itself, once for
+// each set of migrations, and every test's database is copied from it by PostgreSQL. The caller still
+// runs Migrate on the copy, which finds nothing pending only if the copy's record matches the migrations
+// exactly, so a template that disagreed with them would fail the test rather than pass it unseen.
+//
+// The template is named after a digest of the migrations, so a changed or added migration is a new
+// template, never an old one reused. Packages tested at once in separate processes share one: it is
+// built under an advisory lock, and a template a crashed build left half done is not marked a template
+// and is built again. It takes a *testing.T, so nothing that is not a test can call it.
+func CreateMigratedForTest(t *testing.T, admin, name string, migrations []Migration) {
+	t.Helper()
+	template, err := templateFor(admin, migrations)
+	if err != nil {
+		t.Fatalf("cannot raise the migrated template: %v", err)
+	}
+	conn, err := pgx.Connect(t.Context(), admin)
+	if err != nil {
+		t.Fatalf("cannot reach the test PostgreSQL: %v", err)
+	}
+	defer conn.Close(context.Background())
+	// A copy is refused while anything is connected to its template; the template allows no connection
+	// once built, so only the builder's last connection closing can still be in the way.
+	for deadline := time.Now().Add(10 * time.Second); ; {
+		_, err = conn.Exec(t.Context(), "create database "+name+" template "+template)
+		var pgErr *pgconn.PgError
+		if err == nil || !errors.As(err, &pgErr) || pgErr.Code != "55006" || time.Now().After(deadline) {
+			break
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	if err != nil {
+		t.Fatalf("cannot create %s from %s: %v", name, template, err)
+	}
+}
+
+// templates are the templates this process has raised or found, by name.
+var templates sync.Map // name → *templateOnce
+
+type templateOnce struct {
+	once sync.Once
+	err  error
+}
+
+// templateFor is the name of the template holding migrations, raised if no process has yet.
+func templateFor(admin string, migrations []Migration) (string, error) {
+	digest := sha256.New()
+	for _, m := range migrations {
+		fmt.Fprintf(digest, "%d %s %s\n", m.Number, m.Name, m.Checksum)
+	}
+	sum := digest.Sum(nil)
+	name := "tpl_" + hex.EncodeToString(sum[:12])
+	held, _ := templates.LoadOrStore(name, &templateOnce{})
+	once := held.(*templateOnce)
+	once.once.Do(func() {
+		// Not the asking test's context: the template outlives that test, and is raised once for all.
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+		defer cancel()
+		once.err = raiseTemplate(ctx, admin, name, int64(binary.BigEndian.Uint64(sum[12:20])), migrations)
+	})
+	return name, once.err
+}
+
+// raiseTemplate builds the template name under the advisory lock key, unless it stands built.
+func raiseTemplate(ctx context.Context, admin, name string, key int64, migrations []Migration) error {
+	conn, err := pgx.Connect(ctx, admin)
+	if err != nil {
+		return err
+	}
+	defer conn.Close(context.Background())
+	if _, err := conn.Exec(ctx, `select pg_advisory_lock($1)`, key); err != nil {
+		return err
+	}
+	defer func() { _, _ = conn.Exec(context.Background(), `select pg_advisory_unlock($1)`, key) }()
+
+	var built bool
+	if err := conn.QueryRow(ctx, `select exists (select 1 from pg_database where datname = $1 and datistemplate)`,
+		name).Scan(&built); err != nil {
+		return err
+	}
+	if built {
+		return nil
+	}
+	if _, err := conn.Exec(ctx, "drop database if exists "+name+" with (force)"); err != nil {
+		return err
+	}
+	if _, err := conn.Exec(ctx, "create database "+name); err != nil {
+		return err
+	}
+	cut := strings.LastIndex(admin, "/")
+	rest := ""
+	if q := strings.Index(admin[cut:], "?"); q >= 0 {
+		rest = admin[cut+q:]
+	}
+	pool, err := pgxpool.New(ctx, admin[:cut]+"/"+name+rest)
+	if err != nil {
+		return err
+	}
+	_, err = (&Store{context: "template", pool: pool}).Migrate(ctx, migrations)
+	pool.Close()
+	if err != nil {
+		return err
+	}
+	// Marked a template only once whole, and closed to connections, so a copy is never refused for one.
+	for _, sql := range []string{
+		"alter database " + name + " with allow_connections false",
+		"select pg_terminate_backend(pid) from pg_stat_activity where datname = '" + name + "'",
+		"alter database " + name + " with is_template true",
+	} {
+		if _, err := conn.Exec(ctx, sql); err != nil {
+			return err
+		}
+	}
+	return nil
+}

testdata/store-tests

@@ -35,3 +35,5 @@
 ./cmd/mesh-controller TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend
 ./cmd/mesh-controller TestAGateJudgesTheOtherMovesBesideAKeptCore
 ./cmd/mesh-controller TestReplay401
+./cmd/mesh-controller TestReplay201ThePlansSend
+./internal/inventory TestAWalksRestIsReadFromEachMachinesOwnSend